From ffd5c1b92bb73faafc17fe611f728fca3b013560 Mon Sep 17 00:00:00 2001 From: Arnim Balzer Date: Thu, 18 Apr 2024 13:02:41 -0700 Subject: [PATCH 001/179] Fix CMake shared library build using utf8_range and utf8_valid (#14867) Instead of hardcoding static libraries, allow using the BUILD_SHARED_LIBS behaviour to allow the build of shared utf8 libraries. Closes #14867 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/14867 from Chekov2k:main 971562ba471443b93685ab9fa21a3c25ea2e2672 PiperOrigin-RevId: 626125210 --- third_party/utf8_range/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/utf8_range/CMakeLists.txt b/third_party/utf8_range/CMakeLists.txt index 8d7a6e15c6..4276b97290 100644 --- a/third_party/utf8_range/CMakeLists.txt +++ b/third_party/utf8_range/CMakeLists.txt @@ -11,13 +11,13 @@ option (utf8_range_ENABLE_INSTALL "Configure installation" ON) ## # Create the lightweight C library -add_library (utf8_range STATIC +add_library (utf8_range utf8_range.c ) ## # A heavier-weight C++ wrapper that supports Abseil. -add_library (utf8_validity STATIC utf8_validity.cc utf8_range.c) +add_library (utf8_validity utf8_validity.cc utf8_range.c) # Load Abseil dependency. if (NOT TARGET absl::strings) From 2be4364e4c6804a0891e603058d95bf811adf843 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Apr 2024 13:05:09 -0700 Subject: [PATCH 002/179] C#: Remove TODO, replacing it with an explanation. PiperOrigin-RevId: 626125905 --- csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs index f46667ed07..9d41930838 100644 --- a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs @@ -379,7 +379,12 @@ namespace Google.Protobuf.Reflection IDescriptor typeDescriptor = File.DescriptorPool.LookupSymbol(Proto.TypeName, this); - // TODO: See how much of this is actually required. + // In most cases, the type will be specified in the descriptor proto. This may be + // guaranteed in descriptor.proto in the future (with respect to spring 2024), but + // we may still see older descriptors created by old versions of protoc, and there + // may be some code creating descriptor protos directly. This code effectively + // maintains backward compatibility, but we don't expect it to be a path taken + // often at all. if (!Proto.HasType) { // Choose field type based on symbol. From 8bace9999894c0cea4f80b813694a0d7b890f3e8 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 18 Apr 2024 13:23:31 -0700 Subject: [PATCH 003/179] Publishes "Editions: Group Migration Issues" to GitHub. PiperOrigin-RevId: 626131189 --- docs/design/editions/README.md | 3 +- .../design/editions/group-migration-issues.md | 401 ++++++++++++++++++ 2 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 docs/design/editions/group-migration-issues.md diff --git a/docs/design/editions/README.md b/docs/design/editions/README.md index 546fed8605..7ad8674677 100644 --- a/docs/design/editions/README.md +++ b/docs/design/editions/README.md @@ -37,4 +37,5 @@ The following topics are in this repository: * [Edition Naming](edition-naming.md) * [Editions Feature Visibility](editions-feature-visibility.md) * [Legacy Syntax Editions](legacy-syntax-editions.md) -* [Editions: Feature Extension Layout](editions-feature-extension-layout.md) \ No newline at end of file +* [Editions: Feature Extension Layout](editions-feature-extension-layout.md) +* [Editions: Group Migration Issues](group-migration-issues.md) \ No newline at end of file diff --git a/docs/design/editions/group-migration-issues.md b/docs/design/editions/group-migration-issues.md new file mode 100644 index 0000000000..c41ffb2066 --- /dev/null +++ b/docs/design/editions/group-migration-issues.md @@ -0,0 +1,401 @@ +# Editions: Group Migration Issues + +**Authors**: [@mkruskal-google](https://github.com/mkruskal-google) + +## Summary + +Address some unexpected issues in delimited encoding in edition 2023 before its +OSS release. + +## Background + +Joshua Humphries reported some well-timed +[issues](https://github.com/protocolbuffers/protobuf/issues/16239) discovered +while experimenting with our early release of Edition 2023. He discovered that +our new message encoding feature piggybacked a bit too much on the old group +logic, and actually ended up being virtually useless in general. + +None of our testing or migrations caught this because they were heavily focused +on *preserving* old behavior (which is the primary goal of edition 2023). +Delimited messages structured exactly like proto2 groups (e.g. message and field +in the same scope with matching names) continued to work exactly as before, +making it seem like everything was fine. + +All of this is especially problematic in light of *Submessages: In Pursuit of a +More Perfect Encoding* (not available externally yet), which intends to migrate the +ecosystem to use delimited encoding everywhere. Releasing a semi-broken feature +as a migration tool to eliminate a deprecated syntax is one thing, but trying to +push the ecosystem to it is especially bad. + +## Overview + +The problems here stem from the fact that before edition 2023, the field and +type name of group fields was guaranteed to always be unique and intuitive. +Proto2 splits groups into a synthetic nested message with a type name equivalent +to the group specification (required to be capitalized), and a field name that's +fully lowercased. For example, + +``` +optional group MyGroup = 1 { ... } +``` + +would become: + +``` +message MyGroup { ... } +optional MyGroup mygroup = 1; +``` + +The casing here is very important, since the transformation is irreversible. We +can't recover the group name from the field name in general, only if the group +is a single word. + +The problem under edition 2023 is that we've removed the generation of +synchronized synthetic messages from the language. Users now explicitly define +messages, and any message field can be marked `DELIMITED`. This means that +anyone assuming that the type and field name are synchronized could now be +broken. + +### Codegen + +While using the field name for generated APIs required less special-casing in +the generators, the field name ends up producing slightly-less-readable APIs for +multi-word camelcased groups. The result is that we see a fairly random-seeming +mix in different generators. Using protoc-explorer (not available externally), +we find the following: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Language + Generated APIs + Example proto2 getter +
C++ + field + MyGroup mygroup() +
Java (all) + message + MyGroup getMyGroup() +
Python + field + mygroup +
Go (all) + field + GetMygroup() *Foo_MyGroup +
Dart V1 + field/message* + get mygroup +
upb ** + field + Foo_mygroup() +
Objective-c + message + MyGroup* myGroup +
Swift + message + MyGroup myGroup +
C# + field/message* + MyGroup Mygroup +
+ +\* This codegen difference was [caught](cl/611144002) during the implementation +and intentionally "fixed" in Edition 2023. \ +\*\* This includes all upb-based runtimes as well (e.g. Ruby, Rust, etc.) \ +† Extensions use field + +In the Dart V1 implementation, we decided to intentionally introduce a behavior +change on editions upgrades. It was determined that this only affected a handful +of protos in google3, and could probably be manually fixed as-needed. Java's +handling changes the story significantly, since over 50% of protos in google3 +produce generated Java code. Objective-C is also noteworthy since we open-source +it, and Swift because it's widely used in OSS and we don't own it. + +While the editions upgrade is still non-breaking, it means that the generated +APIs could have very surprising spellings and may not be unique. For example, +using the same type for two delimited fields in the same containing message will +create two sets of generated APIs with the same name in some languages! + +### Text Format + +Our "official" +[draft specification](https://protobuf.dev/reference/protobuf/textformat-spec/) +of text-format explicitly states that group messages are encoded by the *message +name*, rather than the lowercases field name. A group `MyGroup` will be +serialized as: + +``` +MyGroup { + ... +} +``` + +In C++, we always serialize the message name and have special handling to only +accept the message name in parsing. We also have conformance tests locking down +the positive path here (i.e. using the message name round-trip). The negative +path (i.e. failing to accept the field name) doesn't have a conformance test, +but C++/Java/Python all agree and there's no known case that doesn't. + +To make things even stranger, for *extensions* (group fields extending other +messages), we always use the field name for groups. So as far as group +extensions are concerned, there's no problem for editions. + +There are a few problems with non-extension group fields in editions: + +* Refactoring the message name will change any text-format output +* New delimited fields will have unexpected text-format output, that *could* + conflict with other fields +* Text parsers will expect the message name, which is surprising and could be + impossible to specify uniquely + +## Recommendation + +Clearly the end-state we want is for the field name to be used in all generated +APIs, and for text-format serialization/parsing. The only questions are: how do +we get there and can/should we do it in time for the 2023 release in 27.0 next +month? + +We propose a combination of the alternatives listed below. +[Smooth Extension](#smooth-extension) seems like the best short-term path +forward to unblock the delimited migration. It *mostly* solves the problem and +doesn't require any new features. The necessary changes for this approach have +already been prepared, along with new conformance tests to lock down the +behavior changes. + +[Global Feature](#global-feature) is a good long-term mitigation for tech debt +we're leaving behind with *Smooth Extension*. Ultimately we would like to remove +any labeling of fields by their type, and editions provides a good mechanism to +do this. Alternatively, we could implement [aliases](#aliases) and use that to +unify this old behavior and avoid a new feature. Either of these options will be +the next step after the release of 2023, with aliases being preferred as long as +the timing works out. + +If we hit any unexpected delays, Nerf Delimited Encoding in 2023 (not available +externally) is the quickest path forward to unblock the release of edition 2023. +It has a lot of downsides though, and will block any migration towards delimited +encoding until edition 2024 has started rolling out. + +## Alternatives + +### Smooth Extension {#smooth-extension} + +Instead of trying to change the existing behavior, we could expand the current +spec to try to cover both proto2 and editions. We would define a "group-like" +concept, which applies to all fields which: + +* Have `DELIMITED` encoding +* Have a type corresponding to a nested message directly under its containing + message +* Have a name corresponding to its lowercased type name. + +Note that proto2 groups will *always* be "group-like." + +For any group-like field we will use the old proto2 semantics, whatever they are +today. Otherwise, we will treat them as regular fields for both codegen and +text-format. This means that *most* new cases of delimited encoding will have +the desired behavior, while *all* old groups will continue to function. The main +exception here is that users will see the unexpected proto2 behavior if they +have message/field names that *happen* to match. + +While the old behavior will result in some unexpected capitalization when it's +hit, it's mostly safe. Because of 2 and 3 (and the fact that we disallow +duplicate field names), we can guarantee that in both codegen and text encoding +there will never be any conflicting symbols. There can never be two delimited +fields of the same type using the old behavior, and no other messages or fields +will exist with either spelling. + +Additionally, we will update the text parsers to accept **both** the old +message-based spelling and the new field-based spelling for group-like fields. +This will at least prevent parsing failures if users hit this unexpected change +in behavior. + +#### Pros + +* Fully supports old proto2 behavior +* Treats most new editions fields correctly +* Doesn't allow for any of the problematic cases we see today +* By updating the parsers to accept both, we have a migration path to change + the "wire"-format +* Decoupled from editions launch (since it's a non-breaking change w/o a + feature) + +#### Cons + +* Requires coordinated changes in every editions-compatible runtime (and many + generators) +* Keeps the old proto2 behavior around indefinitely, with no path to remove it +* Plants surprising edge case for users if they happen to name their + message/fields a certain way + +### Global Feature {#global-feature} + +The simplest answer here is to introduce a new global message feature +`legacy_group_handling` to control all the changes we'd like. This will only be +applicable to group-like fields (see +[Smooth Extension](?tab=t.0#heading=h.blnhard1tpyx)). With this feature enabled, +these fields will always use their message name for text-format. Each +non-conformant language could also use this feature to gate the codegen rules. + +#### Pros + +* Simple boolean to gate all the behavior changes +* Doesn't require adding language features to a bunch of languages that don't + have them yet +* Uses editions to ratchet down the bad behavior + +#### Cons + +* It's a little late in the game to be introducing new features to 2023 + (go/edition-lifetimes) +* Requires coordinated changes in every editions-compatible runtime (and many + generators) +* The migration story for users is unclear. Overriding the value of this + feature is both a "wire"-breaking and API-breaking change they may not be + able to do easily. +* With the feature set, users will still see all of the problems we have today + +### Feature Suite + +An extension of [Global feature](?tab=t.0#heading=h.mvtf74vplkdg) would be to +split the codegen changes out into separate per-language features. + +#### Pros + +* Simple booleans to gate all the distinct behavior changes +* Uses editions to ratchet down the bad behavior +* Better migration story for users, since it separates API and "wire" breaking + changes + +#### Cons + +* Requires a whole slew of new language features, which typically have a + difficult first-time setup +* Requires coordinated changes in every editions-compatible runtime (and many + generators) +* Increases the complexity of edition 2023 significantly +* With the features set, users will still see all of the problems we have + today + +### Nerf Delimited Encoding in 2023 + +A quick fix to avoid releasing a bad feature would be to simply ban the case +where the message and field names don't match. Adding this validation to protoc +would cover the majority of cases, although we might want additional checks in +every language that supports dynamic messages. + +This is a good fallback option if we can't implement anything better before 27.0 +is released. It allows us to release editions in a reasonable state, where we +can fix these issues and release a more functional `DELIMITED` feature in 2024. + +#### Pros + +* Unblocks editions rollout +* Easy and safe to implement +* Avoids rushed implementation of a proper fix +* Avoids runtime issues with text format +* Avoids unexpected build breakages post-editions (e.g. renaming the nested + message) + +#### Cons + +* We'd still be releasing a really bad feature. Instead of opening up new + possibilities, it's just "like groups but worse" +* We couldn't fix this in 2023 without potential version skew from third party + plugins. We'd likely have to wait until edition 2024 +* Might requires coordinated changes in a lot of runtimes +* Doesn't unblock our effort to roll out delimited + +### Rename Fields in Editions + +While it might be tempting to leverage the edition 2023 upgrade as a place we +can just rename the group field, that doesn't actually work (e.g. rename +`mygroup` to `my_group`). Because so many runtimes already use the *field name* +in generated APIs, they would break under this transformation. + +#### Pros + +* Works really well for text-format and some languages + +#### Cons + +* Turns 2023 upgrade into a breaking change for many languages + +### Aliases {#aliases} + +We've discussed aliases a lot mostly in the context of `Any`, but they would be +useful for any encoding scheme that locks down field/message names. If we had a +fully implemented alias system in place, it would be the perfect mitigation +here. Unfortunately, we don't yet and the timeline here is probably too tight to +implement one. + +#### Pros + +* Fixes all of the problems mentioned above +* Allows us to specify the old behavior using the proto language, which allows + it to be handled by Prototiller + +#### Cons + +* We want this to be a real fully thought-out feature, not a hack rushed into + a tight timeline + +### Do Nothing + +Doing nothing doesn't actually break anyone, but it is embarrassing. + +#### Pros + +* Easy to do + +#### Cons + +* Releases a horrible feature full of foot-guns in our first edition +* Doesn't unblock our effort to roll out delimited From 97022a9c836351df485d12b586005641ae2c508c Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 18 Apr 2024 13:54:36 -0700 Subject: [PATCH 004/179] Move xcode setup out of protobuf-ci PiperOrigin-RevId: 626140997 --- ci/macOS.bazelrc | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc index 465bf0957f..8e7eaf0fb3 100644 --- a/ci/macOS.bazelrc +++ b/ci/macOS.bazelrc @@ -2,3 +2,4 @@ import common.bazelrc build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 +common --xcode_version_config=@com_google_protobuf//.github:host_xcodes \ No newline at end of file From 0cda26d48db0241f6aca0e931ff0a31acea1a6c2 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 18 Apr 2024 14:16:48 -0700 Subject: [PATCH 005/179] Move the windows path length workarounds out of protobuf-ci PiperOrigin-RevId: 626148009 --- ci/Windows.bazelrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/Windows.bazelrc b/ci/Windows.bazelrc index 427e923213..dd3bb48b10 100644 --- a/ci/Windows.bazelrc +++ b/ci/Windows.bazelrc @@ -1,2 +1,5 @@ import common.bazelrc +# Workaround for maximum path length issues +startup --output_user_root=C:/tmp --windows_enable_symlinks +common --enable_runfiles \ No newline at end of file From b28d9d41446ee25540f3767c002437a23beb0a9e Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 19 Apr 2024 09:38:13 -0700 Subject: [PATCH 006/179] Migrate bazel tests to protobuf-ci. This will allow them to reuse our bazelrc and remote caching setup. This also silences the non-bzlmod windows test that's hitting the windows path length. PiperOrigin-RevId: 626390416 --- .github/workflows/staleness_check.yml | 4 +- .github/workflows/test_bazel.yml | 69 ++++++++++++++++++++------- .github/workflows/test_cpp.yml | 64 ++++++++++++------------- .github/workflows/test_csharp.yml | 16 +++---- .github/workflows/test_java.yml | 14 +++--- .github/workflows/test_objectivec.yml | 14 +++--- .github/workflows/test_php.yml | 30 ++++++------ .github/workflows/test_php_ext.yml | 6 +-- .github/workflows/test_python.yml | 8 ++-- .github/workflows/test_ruby.yml | 24 +++++----- .github/workflows/test_runner.yml | 8 ++++ .github/workflows/test_rust.yml | 4 +- .github/workflows/test_upb.yml | 24 +++++----- examples/.bazelrc | 10 +++- 14 files changed, 172 insertions(+), 123 deletions(-) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 4807ac6d7d..86a9848243 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -26,7 +26,7 @@ jobs: if: ${{ github.event.repository.full_name == 'protocolbuffers/protobuf' }} steps: - name: Checkout ${{ github.head_ref && 'PR' || matrix.branch }} - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout || github.head_ref || matrix.branch }} @@ -49,7 +49,7 @@ jobs: # In branches where automatic updates work as post-submits, we don't want to run staleness # tests along with user changes. Any stale files will be automatically fixed in a follow-up # commit. - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: staleness diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml index 2ba549a1e6..8b6139c35a 100644 --- a/.github/workflows/test_bazel.yml +++ b/.github/workflows/test_bazel.yml @@ -1,23 +1,56 @@ -name: Bazel +name: Bazel Tests -# Controls when the action will run. on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: [main] - pull_request: - branches: [main] - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: + workflow_call: + inputs: + safe-checkout: + required: true + description: "The SHA key for the commit we want to run over" + type: string -concurrency: - # Cancel previous actions from the same PR or branch except 'main' branch. - # See https://docs.github.com/en/actions/using-jobs/using-concurrency and https://docs.github.com/en/actions/learn-github-actions/contexts for more info. - group: concurrency-group::${{ github.workflow }}::${{ github.event.pull_request.number > 0 && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}${{ github.ref_name == 'main' && format('::{0}', github.run_id) || ''}} - cancel-in-progress: ${{ github.ref_name != 'main' }} +permissions: + contents: read jobs: - test: - uses: bazel-contrib/.github/.github/workflows/bazel.yaml@v6 - with: - folders: '["examples"]' + examples: + strategy: + fail-fast: false + matrix: + runner: [ ubuntu, windows, macos ] + bazelversion: [ '7.1.1' ] + bzlmod: [true, false ] + include: + - runner: ubuntu + bazelversion: '6.4.0' + bzlmod: true + - runner: ubuntu + bazelversion: '6.4.0' + bzlmod: false + runs-on: ${{ matrix.runner }}-latest + name: Examples ${{ matrix.runner }} ${{ matrix.bazelversion }}${{ matrix.bzlmod && ' (bzlmod)' || '' }} + steps: + - name: Checkout pending changes + uses: protocolbuffers/protobuf-ci/checkout@v3 + with: + ref: ${{ inputs.safe-checkout }} + + - name: Windows startup flags + if: runner.os == 'Windows' + working-directory: examples + shell: bash + run: echo "startup --output_user_root=C:/ --windows_enable_symlinks" >> .bazelrc + + - name: Configure Bazel version + working-directory: examples + shell: bash + run: echo "${{ matrix.bazelversion }}" > .bazelversion + + - name: Run tests + uses: protocolbuffers/protobuf-ci/bazel@v3 + # TODO Silence this until we have a fix. + if: runner.os != 'Windows' || matrix.bzlmod + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: examples + version: ${{ matrix.bazelversion }} + bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 772402f1c7..8ab24bca85 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -43,11 +43,11 @@ jobs: runs-on: ${{ matrix.config.runner || 'ubuntu-latest' }} steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -63,11 +63,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -83,24 +83,24 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} submodules: recursive - name: Cross compile protoc for ${{ matrix.arch }} id: cross-compile - uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2 + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-${{ matrix.arch }} - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-release-${{ matrix.arch }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:${{ matrix.arch }}-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -135,18 +135,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-cmake credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -160,19 +160,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} submodules: recursive - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-cmake-install credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -193,18 +193,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-cmake-examples credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -233,19 +233,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} submodules: recursive - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-cmake-gcc credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -264,19 +264,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} submodules: recursive - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-cmake-submodules credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -289,19 +289,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} submodules: recursive - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: linux-cmake-32-bit credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:8275360dc5d676f3470872d79087901c0e4153453976bea908a92c82e8d209ea platform: linux/386 @@ -348,11 +348,11 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel: ${{ matrix.bazel }} @@ -414,7 +414,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} submodules: recursive @@ -438,7 +438,7 @@ jobs: shell: bash - name: Setup sccache - uses: protocolbuffers/protobuf-ci/sccache@v2 + uses: protocolbuffers/protobuf-ci/sccache@v3 with: cache-prefix: ${{ matrix.cache-prefix }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -446,7 +446,7 @@ jobs: # Install phase. - name: Configure CMake for install if: matrix.install-flags - uses: protocolbuffers/protobuf-ci/bash@v2 + uses: protocolbuffers/protobuf-ci/bash@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON @@ -468,7 +468,7 @@ jobs: run: cmake --build . --target clean && rm CMakeCache.txt - name: Configure CMake - uses: protocolbuffers/protobuf-ci/bash@v2 + uses: protocolbuffers/protobuf-ci/bash@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index 26d974102a..f816f1f9bb 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -17,13 +17,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} # TODO Run this with Bazel once codegen is handled properly. - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -39,7 +39,7 @@ jobs: run: sudo rm -rf _build - name: Run conformance tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -51,7 +51,7 @@ jobs: runs-on: windows-2019 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} @@ -70,7 +70,7 @@ jobs: shell: bash - name: Run tests - uses: protocolbuffers/protobuf-ci/bash@v2 + uses: protocolbuffers/protobuf-ci/bash@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: | @@ -82,14 +82,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Build protobuf C# tests under x86_64 docker image # Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory # (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run) - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -106,7 +106,7 @@ jobs: # running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user # otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, # we just run map the user's home to a throwaway temporary directory - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 skip-staleness-check: true diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 0c602f3818..a9abfacaec 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -40,11 +40,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -57,11 +57,11 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Checkout pending changes - # uses: protocolbuffers/protobuf-ci/checkout@v2 + # uses: protocolbuffers/protobuf-ci/checkout@v3 # with: # ref: ${{ inputs.safe-checkout }} # - name: Run Linkage Monitor test - # uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + # uses: protocolbuffers/protobuf-ci/bazel-docker@v3 # with: # image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 # credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -75,12 +75,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Build protoc id: build-protoc - uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2 + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -94,7 +94,7 @@ jobs: mvn -e -B -Dhttps.protocols=TLSv1.2 install -Dmaven.test.skip=true working-directory: java - name: Generate pom.xml files from the template - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index 185f596fd4..1db6c10c88 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -34,18 +34,18 @@ jobs: DEVELOPER_DIR: /Applications/Xcode_14.1.app/Contents/Developer steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: protocolbuffers/protobuf-ci/ccache@v2 + uses: protocolbuffers/protobuf-ci/ccache@v3 with: cache-prefix: objectivec_${{ matrix.platform }}_${{ matrix.xc_config }} support-modules: true - name: Run tests - uses: protocolbuffers/protobuf-ci/bash@v2 + uses: protocolbuffers/protobuf-ci/bash@v3 env: CC: ${{ github.workspace }}/ci/clang_wrapper CXX: ${{ github.workspace }}/ci/clang_wrapper++ @@ -80,13 +80,13 @@ jobs: runs-on: ${{ matrix.OS }} steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Xcode version run: sudo xcode-select -switch /Applications/Xcode_${{ matrix.XCODE }}.app - name: Pod lib lint - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: cocoapods/${{ matrix.XCODE }} @@ -125,11 +125,11 @@ jobs: runs-on: macos-12 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: bazel ${{ matrix.config.bazel_action }} - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel: ${{ matrix.config.bazel_action }} ${{ matrix.config.flags }} ${{ matrix.bazel_targets }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 6909668759..86d24954b2 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -44,16 +44,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Setup composer - uses: protocolbuffers/protobuf-ci/composer-setup@v2 + uses: protocolbuffers/protobuf-ci/composer-setup@v3 with: cache-prefix: php-${{ matrix.version-short }} directory: php - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -82,26 +82,26 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:836f2cedcfe351d9a30055076630408e61994fc7d783e8333a99570968990eeb steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Cross compile protoc for i386 id: cross-compile - uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2 + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 - name: Setup composer - uses: protocolbuffers/protobuf-ci/composer-setup@v2 + uses: protocolbuffers/protobuf-ci/composer-setup@v3 with: cache-prefix: php-${{ matrix.version }} directory: php - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: ${{ env.image }} platform: linux/386 @@ -119,26 +119,26 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Cross compile protoc for aarch64 id: cross-compile - uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2 + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 - name: Setup composer - uses: protocolbuffers/protobuf-ci/composer-setup@v2 + uses: protocolbuffers/protobuf-ci/composer-setup@v3 with: cache-prefix: php-8.1 directory: php - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64@sha256:77ff9fdec867bbfb290ee0b10d8b7a3e5e434155daa5ec93de7341c7592b858d platform: linux/arm64 @@ -161,7 +161,7 @@ jobs: runs-on: macos-12 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} @@ -180,13 +180,13 @@ jobs: run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1) - name: Setup composer - uses: protocolbuffers/protobuf-ci/composer-setup@v2 + uses: protocolbuffers/protobuf-ci/composer-setup@v3 with: cache-prefix: php-${{ matrix.version }} directory: php - name: Run tests - uses: protocolbuffers/protobuf-ci/bash@v2 + uses: protocolbuffers/protobuf-ci/bash@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: | @@ -198,7 +198,7 @@ jobs: popd - name: Run conformance tests - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: php_macos/${{ matrix.version }} diff --git a/.github/workflows/test_php_ext.yml b/.github/workflows/test_php_ext.yml index cae0241f72..dfe1951b11 100644 --- a/.github/workflows/test_php_ext.yml +++ b/.github/workflows/test_php_ext.yml @@ -17,12 +17,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Package extension - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: php_ext/${{ matrix.version }} @@ -50,7 +50,7 @@ jobs: name: protobuf-php-release - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-extension:${{ matrix.version }}-a48f26c08d9a803dd0177dda63563f6ea6f7b2d4 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index a5dbe8f85b..b48da36aab 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -37,11 +37,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-63dd26c0c7a808d92673a3e52e848189d4ab0f17', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -68,7 +68,7 @@ jobs: runs-on: macos-12 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} @@ -88,7 +88,7 @@ jobs: source venv/bin/activate - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 env: KOKORO_PYTHON_VERSION: ${{ matrix.version }} with: diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index b858bbfcda..38cb5ef5b2 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -34,11 +34,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-6.3.0-a6940b1421a71325ef4c7828ec72d404f56bbf30', matrix.ruby) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -50,20 +50,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Cross compile protoc for i386 id: cross-compile - uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2 + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: i386/ruby:3.0.2-buster credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -81,20 +81,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Cross compile protoc for aarch64 id: cross-compile - uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2 + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 - name: Run tests - uses: protocolbuffers/protobuf-ci/docker@v2 + uses: protocolbuffers/protobuf-ci/docker@v3 with: image: arm64v8/ruby:3.0.2-buster credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -128,7 +128,7 @@ jobs: runs-on: macos-12 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} @@ -141,7 +141,7 @@ jobs: run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1) - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_macos/${{ matrix.version }} @@ -166,11 +166,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-6.3.0-a6940b1421a71325ef4c7828ec72d404f56bbf30 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index 0d4232bf5c..d67f45593b 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -105,6 +105,14 @@ jobs: # Note: this pattern of passing the head sha is vulnerable to PWN requests for # pull_request_target events. We carefully limit those workflows to require a # human stamp before continuing. + bazel: + name: Bazel + needs: [check-tag] + uses: ./.github/workflows/test_bazel.yml + with: + safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }} + secrets: inherit + cpp: name: C++ needs: [check-tag] diff --git a/.github/workflows/test_rust.yml b/.github/workflows/test_rust.yml index 6018eec953..cab8c4d874 100644 --- a/.github/workflows/test_rust.yml +++ b/.github/workflows/test_rust.yml @@ -17,11 +17,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.1-75f2a85ece6526cc3d54087018c0f1097d78d42b" credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index 6f9f25c6a9..e0e0d4d128 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -31,11 +31,11 @@ jobs: steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '6.3.0' }}-75f2a85ece6526cc3d54087018c0f1097d78d42b credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -50,11 +50,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17" credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -68,7 +68,7 @@ jobs: runs-on: windows-2022 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 @@ -76,7 +76,7 @@ jobs: cache: pip cache-dependency-path: 'python/requirements.txt' - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: "upb-bazel-windows" @@ -95,7 +95,7 @@ jobs: runs-on: macos-12 steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 @@ -103,7 +103,7 @@ jobs: cache: pip cache-dependency-path: 'python/requirements.txt' - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel@v2 + uses: protocolbuffers/protobuf-ci/bazel@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: "upb-bazel-macos" @@ -117,11 +117,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -138,11 +138,11 @@ jobs: if: ${{ github.event_name != 'pull_request_target' }} steps: - name: Checkout pending changes - uses: protocolbuffers/protobuf-ci/checkout@v2 + uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - name: Build Wheels - uses: protocolbuffers/protobuf-ci/bazel-docker@v2 + uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: image: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple@sha256:b3dc9b75d8e599b0e95ed245d89f44b5a4231112f975da89dd02006a484a58df credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} diff --git a/examples/.bazelrc b/examples/.bazelrc index 554440cfe3..f37d75bc21 100644 --- a/examples/.bazelrc +++ b/examples/.bazelrc @@ -1 +1,9 @@ -build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 +common --enable_platform_specific_config + +build:linux --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 +build:macos --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 + +common:windows --enable_runfiles + +build --experimental_remote_cache_eviction_retries=5 +build --remote_download_outputs=all From eeeb985a1bad97631acb547d857333e8fe89a42a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Apr 2024 10:21:55 -0700 Subject: [PATCH 007/179] Use absl::bit_cast to implicitly check presence of float and double. PiperOrigin-RevId: 626402666 --- src/google/protobuf/compiler/cpp/message.cc | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 647f804d20..b54749c245 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -208,21 +208,11 @@ bool EmitFieldNonDefaultCondition(io::Printer* p, const std::string& prefix, )cc"); } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT) { p->Emit(R"cc( - static_assert(sizeof(::uint32_t) == sizeof(float), - "Code assumes ::uint32_t and float are the same size."); - float tmp_$name$ = $prefix$_internal_$name$(); - ::uint32_t raw_$name$; - memcpy(&raw_$name$, &tmp_$name$, sizeof(tmp_$name$)); - if (raw_$name$ != 0) { + if (::absl::bit_cast<::uint32_t>($prefix$_internal_$name$()) != 0) { )cc"); } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_DOUBLE) { p->Emit(R"cc( - static_assert(sizeof(::uint64_t) == sizeof(double), - "Code assumes ::uint64_t and double are the same size."); - double tmp_$name$ = $prefix$_internal_$name$(); - ::uint64_t raw_$name$; - memcpy(&raw_$name$, &tmp_$name$, sizeof(tmp_$name$)); - if (raw_$name$ != 0) { + if (::absl::bit_cast<::uint64_t>($prefix$_internal_$name$()) != 0) { )cc"); } else { p->Emit(R"cc( From e98423c4032a42d4e6effce85d070bf27d64ff0e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Apr 2024 17:39:58 +0000 Subject: [PATCH 008/179] Auto-generate files after cl/626402666 --- src/google/protobuf/wrappers.pb.cc | 42 +++++------------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/src/google/protobuf/wrappers.pb.cc b/src/google/protobuf/wrappers.pb.cc index ee019eacfe..a47a849591 100644 --- a/src/google/protobuf/wrappers.pb.cc +++ b/src/google/protobuf/wrappers.pb.cc @@ -457,12 +457,7 @@ PROTOBUF_NOINLINE void DoubleValue::Clear() { (void)cached_has_bits; // double value = 1; - static_assert(sizeof(::uint64_t) == sizeof(double), - "Code assumes ::uint64_t and double are the same size."); - double tmp_value = this->_internal_value(); - ::uint64_t raw_value; - memcpy(&raw_value, &tmp_value, sizeof(tmp_value)); - if (raw_value != 0) { + if (::absl::bit_cast<::uint64_t>(this->_internal_value()) != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteDoubleToArray( 1, this->_internal_value(), target); @@ -486,12 +481,7 @@ PROTOBUF_NOINLINE void DoubleValue::Clear() { (void) cached_has_bits; // double value = 1; - static_assert(sizeof(::uint64_t) == sizeof(double), - "Code assumes ::uint64_t and double are the same size."); - double tmp_value = this->_internal_value(); - ::uint64_t raw_value; - memcpy(&raw_value, &tmp_value, sizeof(tmp_value)); - if (raw_value != 0) { + if (::absl::bit_cast<::uint64_t>(this->_internal_value()) != 0) { total_size += 9; } @@ -507,12 +497,7 @@ void DoubleValue::MergeImpl(::google::protobuf::MessageLite& to_msg, const ::goo ::uint32_t cached_has_bits = 0; (void) cached_has_bits; - static_assert(sizeof(::uint64_t) == sizeof(double), - "Code assumes ::uint64_t and double are the same size."); - double tmp_value = from._internal_value(); - ::uint64_t raw_value; - memcpy(&raw_value, &tmp_value, sizeof(tmp_value)); - if (raw_value != 0) { + if (::absl::bit_cast<::uint64_t>(from._internal_value()) != 0) { _this->_impl_.value_ = from._impl_.value_; } _this->_internal_metadata_.MergeFrom<::google::protobuf::UnknownFieldSet>(from._internal_metadata_); @@ -643,12 +628,7 @@ PROTOBUF_NOINLINE void FloatValue::Clear() { (void)cached_has_bits; // float value = 1; - static_assert(sizeof(::uint32_t) == sizeof(float), - "Code assumes ::uint32_t and float are the same size."); - float tmp_value = this->_internal_value(); - ::uint32_t raw_value; - memcpy(&raw_value, &tmp_value, sizeof(tmp_value)); - if (raw_value != 0) { + if (::absl::bit_cast<::uint32_t>(this->_internal_value()) != 0) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteFloatToArray( 1, this->_internal_value(), target); @@ -672,12 +652,7 @@ PROTOBUF_NOINLINE void FloatValue::Clear() { (void) cached_has_bits; // float value = 1; - static_assert(sizeof(::uint32_t) == sizeof(float), - "Code assumes ::uint32_t and float are the same size."); - float tmp_value = this->_internal_value(); - ::uint32_t raw_value; - memcpy(&raw_value, &tmp_value, sizeof(tmp_value)); - if (raw_value != 0) { + if (::absl::bit_cast<::uint32_t>(this->_internal_value()) != 0) { total_size += 5; } @@ -693,12 +668,7 @@ void FloatValue::MergeImpl(::google::protobuf::MessageLite& to_msg, const ::goog ::uint32_t cached_has_bits = 0; (void) cached_has_bits; - static_assert(sizeof(::uint32_t) == sizeof(float), - "Code assumes ::uint32_t and float are the same size."); - float tmp_value = from._internal_value(); - ::uint32_t raw_value; - memcpy(&raw_value, &tmp_value, sizeof(tmp_value)); - if (raw_value != 0) { + if (::absl::bit_cast<::uint32_t>(from._internal_value()) != 0) { _this->_impl_.value_ = from._impl_.value_; } _this->_internal_metadata_.MergeFrom<::google::protobuf::UnknownFieldSet>(from._internal_metadata_); From c413b1231c679fa16dcbe6d14b648ebb63f627d1 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Apr 2024 10:45:10 -0700 Subject: [PATCH 009/179] Change upb::Arena to be Send (but still not Sync) PiperOrigin-RevId: 626409752 --- rust/upb/arena.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/rust/upb/arena.rs b/rust/upb/arena.rs index 51e8958e4e..8610d4de56 100644 --- a/rust/upb/arena.rs +++ b/rust/upb/arena.rs @@ -22,11 +22,11 @@ const _CHECK_UPB_MALLOC_ALIGN_AT_LEAST_POINTER_ALIGNED: () = /// This is an owning type and will automatically free the arena when /// dropped. /// -/// Note that this type is neither `Sync` nor `Send`. The upb_Arena C object -/// could be understood as being Sync (at least vacuously, as there are no -/// const-ptr functions on upb_Arena's API), but the Rust Arena is -/// necessarily expressed as interior mutability (&self rather than &mut self -/// receivers) See https://doc.rust-lang.org/nomicon/lifetime-mismatch.html and +/// Note that this type is not `Sync` as it implements unsynchronized interior +/// mutability. The upb_Arena C object could be understood as being Sync (at +/// least vacuously under current API since there are not any const upb_Arena* +/// API functions), but the Rust Arena is necessarily expressed as interior +/// mutability (&self rather than &mut self receivers) See https://doc.rust-lang.org/nomicon/lifetime-mismatch.html and /// https://blog.reverberate.org/2021/12/19/arenas-and-rust.html, and the /// 'known problems' section of https://rust-lang.github.io/rust-clippy/master/index.html#/mut_from_ref. #[derive(Debug)] @@ -36,6 +36,10 @@ pub struct Arena { _not_sync: PhantomData>, } +// SAFETY: `Arena` uniquely holds the underlying RawArena and has no +// thread-local data. +unsafe impl Send for Arena {} + impl Arena { /// Allocates a fresh arena. #[inline] From 971f88aa636b5f996775cc37394bc717f17ca0b3 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Apr 2024 15:07:28 -0700 Subject: [PATCH 010/179] Improve C++, Python, Java runtime version representations to avoid merge conflicts during release. PiperOrigin-RevId: 626480269 --- .../com/google/protobuf/RuntimeVersion.java | 20 +++++++++------ python/google/protobuf/runtime_version.py | 25 ++++++++++++------- src/google/protobuf/runtime_version.h | 16 ++++++++++-- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/RuntimeVersion.java b/java/core/src/main/java/com/google/protobuf/RuntimeVersion.java index 3789a9db7b..5a51706417 100644 --- a/java/core/src/main/java/com/google/protobuf/RuntimeVersion.java +++ b/java/core/src/main/java/com/google/protobuf/RuntimeVersion.java @@ -24,12 +24,18 @@ public final class RuntimeVersion { // The version of this runtime. // Automatically updated by Protobuf release process. Do not edit manually. - - public static final RuntimeDomain DOMAIN = RuntimeDomain.PUBLIC; - public static final int MAJOR = 4; - public static final int MINOR = 28; - public static final int PATCH = 0; - public static final String SUFFIX = "-dev"; + // These OSS versions are not stripped to avoid merging conflicts. + public static final RuntimeDomain OSS_DOMAIN = RuntimeDomain.PUBLIC; + public static final int OSS_MAJOR = 4; + public static final int OSS_MINOR = 28; + public static final int OSS_PATCH = 0; + public static final String OSS_SUFFIX = "-dev"; + + public static final RuntimeDomain DOMAIN = OSS_DOMAIN; + public static final int MAJOR = OSS_MAJOR; + public static final int MINOR = OSS_MINOR; + public static final int PATCH = OSS_PATCH; + public static final String SUFFIX = OSS_SUFFIX; private static final String VERSION_STRING = versionString(MAJOR, MINOR, PATCH, SUFFIX); private static final Logger logger = Logger.getLogger(RuntimeVersion.class.getName()); @@ -88,7 +94,7 @@ public final class RuntimeVersion { } // Check that runtime version is newer than the gencode version. - if (MINOR < minor || (MINOR == minor && PATCH < patch)) { + if (MINOR < minor || (minor == MINOR && PATCH < patch)) { throw new ProtobufRuntimeVersionException( String.format( "Detected incompatible Protobuf Gencode/Runtime versions when loading %s: gencode %s," diff --git a/python/google/protobuf/runtime_version.py b/python/google/protobuf/runtime_version.py index e414e09e9e..bc4061cfeb 100644 --- a/python/google/protobuf/runtime_version.py +++ b/python/google/protobuf/runtime_version.py @@ -22,17 +22,24 @@ class Domain(Enum): PUBLIC = 2 -class VersionError(Exception): - """Exception class for version violation.""" - - # The versions of this Python Protobuf runtime to be changed automatically by # the Protobuf release process. Do not edit them manually. -DOMAIN = Domain.PUBLIC -MAJOR = 5 -MINOR = 28 -PATCH = 0 -SUFFIX = '-dev' +# These OSS versions are not stripped to avoid merging conflicts. +OSS_DOMAIN = Domain.PUBLIC +OSS_MAJOR = 5 +OSS_MINOR = 28 +OSS_PATCH = 0 +OSS_SUFFIX = '-dev' + +DOMAIN = OSS_DOMAIN +MAJOR = OSS_MAJOR +MINOR = OSS_MINOR +PATCH = OSS_PATCH +SUFFIX = OSS_SUFFIX + + +class VersionError(Exception): + """Exception class for version violation.""" def ValidateProtobufRuntimeVersion( diff --git a/src/google/protobuf/runtime_version.h b/src/google/protobuf/runtime_version.h index 50dd313c1f..9244839623 100644 --- a/src/google/protobuf/runtime_version.h +++ b/src/google/protobuf/runtime_version.h @@ -9,7 +9,19 @@ #error PROTOBUF_VERSION_SUFFIX was previously defined #endif // PROTOBUF_VERSION_SUFFIX -#define PROTOBUF_VERSION 5028000 -#define PROTOBUF_VERSION_SUFFIX "-dev" +#ifdef PROTOBUF_OSS_VERSION +#error PROTOBUF_OSS_VERSION was previously defined +#endif // PROTOBUF_OSS_VERSION + +#ifdef PROTOBUF_OSS_VERSION_SUFFIX +#error PROTOBUF_OSS_VERSION_SUFFIX was previously defined +#endif // PROTOBUF_OSS_VERSION_SUFFIX + +// The OSS versions are not stripped to avoid merging conflicts. +#define PROTOBUF_OSS_VERSION 5028000 +#define PROTOBUF_OSS_VERSION_SUFFIX "-dev" + +#define PROTOBUF_VERSION PROTOBUF_OSS_VERSION +#define PROTOBUF_VERSION_SUFFIX PROTOBUF_OSS_VERSION_SUFFIX #endif // GOOGLE_PROTOBUF_RUNTIME_VERSION_H__ From 37e484a0aa89b2d54b182c093adff9a2b8358c81 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Fri, 19 Apr 2024 15:29:58 -0700 Subject: [PATCH 011/179] Replace direct access to MiniTable subs with calls to the encapsulated functions. PiperOrigin-RevId: 626485595 --- upb/message/accessors.c | 6 +-- upb/message/accessors.h | 12 ++++-- upb/message/copy.c | 5 +-- upb/message/promote.c | 7 +--- upb/mini_descriptor/internal/encode_test.cc | 12 +++--- upb/mini_table/internal/message.h | 43 ++++++++++++--------- upb/mini_table/message.h | 12 +++++- 7 files changed, 56 insertions(+), 41 deletions(-) diff --git a/upb/message/accessors.c b/upb/message/accessors.c index a3184a5a21..de142d90d4 100644 --- a/upb/message/accessors.c +++ b/upb/message/accessors.c @@ -24,10 +24,8 @@ bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* m, const upb_MiniTableField* f, upb_Message* map_entry_message, upb_Arena* arena) { UPB_ASSERT(!upb_Message_IsFrozen(map_entry_message)); - - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_mini_table = upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* map_entry_mini_table = + upb_MiniTable_MapEntrySubMessage(m, f); UPB_ASSERT(map_entry_mini_table); const upb_MiniTableField* map_entry_key_field = upb_MiniTable_MapKey(map_entry_mini_table); diff --git a/upb/message/accessors.h b/upb/message/accessors.h index 59ec925d79..09ea66ae5b 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -22,9 +22,15 @@ #include "upb/message/internal/message.h" #include "upb/message/internal/tagged_ptr.h" #include "upb/message/map.h" +#include "upb/message/message.h" #include "upb/message/tagged_ptr.h" #include "upb/message/value.h" #include "upb/mini_table/enum.h" +#include "upb/mini_table/extension.h" +#include "upb/mini_table/field.h" +#include "upb/mini_table/internal/field.h" +#include "upb/mini_table/internal/message.h" +#include "upb/mini_table/message.h" #include "upb/mini_table/sub.h" // Must be last. @@ -354,8 +360,6 @@ UPB_API_INLINE void _upb_Message_SetTaggedMessagePtr( UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - UPB_ASSERT(upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)])); upb_Message_SetBaseField(msg, field, &sub_message); } @@ -379,8 +383,8 @@ UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( upb_Message* sub_message = *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); if (!sub_message) { - const upb_MiniTable* sub_mini_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* sub_mini_table = + upb_MiniTable_SubMessage(mini_table, field); UPB_ASSERT(sub_mini_table); sub_message = _upb_Message_New(sub_mini_table, arena); *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*) = sub_message; diff --git a/upb/message/copy.c b/upb/message/copy.c index 114b92e018..38cb8494bc 100644 --- a/upb/message/copy.c +++ b/upb/message/copy.c @@ -117,9 +117,8 @@ static upb_Map* upb_Message_Map_DeepClone(const upb_Map* map, const upb_MiniTableField* f, upb_Message* clone, upb_Arena* arena) { - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* map_entry_table = + upb_MiniTable_MapEntrySubMessage(mini_table, f); UPB_ASSERT(map_entry_table); const upb_MiniTableField* key_field = upb_MiniTable_MapKey(map_entry_table); diff --git a/upb/message/promote.c b/upb/message/promote.c index 51412e1781..d4b1ad1dee 100644 --- a/upb/message/promote.c +++ b/upb/message/promote.c @@ -327,12 +327,9 @@ upb_UnknownToMessage_Status upb_MiniTable_PromoteUnknownToMessageArray( upb_UnknownToMessage_Status upb_MiniTable_PromoteUnknownToMap( upb_Message* msg, const upb_MiniTable* mini_table, const upb_MiniTableField* field, int decode_options, upb_Arena* arena) { - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_mini_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]); - UPB_ASSERT(map_entry_mini_table); + const upb_MiniTable* map_entry_mini_table = + upb_MiniTable_MapEntrySubMessage(mini_table, field); UPB_ASSERT(upb_MiniTable_FieldCount(map_entry_mini_table) == 2); - UPB_ASSERT(upb_MiniTableField_IsMap(field)); // Find all unknowns with given field number and parse. upb_FindUnknownRet unknown; while (1) { diff --git a/upb/mini_descriptor/internal/encode_test.cc b/upb/mini_descriptor/internal/encode_test.cc index 3ae2f6f56c..a84c739bbc 100644 --- a/upb/mini_descriptor/internal/encode_test.cc +++ b/upb/mini_descriptor/internal/encode_test.cc @@ -45,7 +45,7 @@ TEST_P(MiniTableTest, Empty) { upb_MiniTable* table = _upb_MiniTable_Build(nullptr, 0, GetParam(), arena.ptr(), status.ptr()); ASSERT_NE(nullptr, table); - EXPECT_EQ(0, table->UPB_PRIVATE(field_count)); + EXPECT_EQ(0, upb_MiniTable_FieldCount(table)); EXPECT_EQ(0, table->UPB_PRIVATE(required_count)); } @@ -242,11 +242,11 @@ TEST_P(MiniTableTest, SubsInitializedToEmpty) { upb_MiniTable* table = _upb_MiniTable_Build( e.data().data(), e.data().size(), GetParam(), arena.ptr(), status.ptr()); ASSERT_NE(nullptr, table); - EXPECT_EQ(table->UPB_PRIVATE(field_count), 2); - EXPECT_TRUE(UPB_PRIVATE(_upb_MiniTable_IsEmpty)( - upb_MiniTableSub_Message(table->UPB_PRIVATE(subs)[0]))); - EXPECT_TRUE(UPB_PRIVATE(_upb_MiniTable_IsEmpty)( - upb_MiniTableSub_Message(table->UPB_PRIVATE(subs)[1]))); + EXPECT_EQ(upb_MiniTable_FieldCount(table), 2); + EXPECT_FALSE(upb_MiniTable_FieldIsLinked( + table, upb_MiniTable_GetFieldByIndex(table, 0))); + EXPECT_FALSE(upb_MiniTable_FieldIsLinked( + table, upb_MiniTable_GetFieldByIndex(table, 1))); } TEST(MiniTableEnumTest, PositiveAndNegative) { diff --git a/upb/mini_table/internal/message.h b/upb/mini_table/internal/message.h index e044c0e48c..d5b1ae4e0b 100644 --- a/upb/mini_table/internal/message.h +++ b/upb/mini_table/internal/message.h @@ -8,8 +8,10 @@ #ifndef UPB_MINI_TABLE_INTERNAL_MESSAGE_H_ #define UPB_MINI_TABLE_INTERNAL_MESSAGE_H_ +#include #include +#include "upb/base/descriptor_constants.h" #include "upb/mini_table/internal/field.h" #include "upb/mini_table/internal/sub.h" @@ -97,34 +99,44 @@ UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_GetFieldByIndex( return &m->UPB_ONLYBITS(fields)[i]; } -UPB_INLINE const union upb_MiniTableSub* UPB_PRIVATE( +UPB_INLINE const union upb_MiniTableSub UPB_PRIVATE( _upb_MiniTable_GetSubByIndex)(const struct upb_MiniTable* m, uint32_t i) { - return &m->UPB_PRIVATE(subs)[i]; + return m->UPB_PRIVATE(subs)[i]; +} + +UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_SubMessage( + const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { + if (upb_MiniTableField_CType(f) != kUpb_CType_Message) { + return NULL; + } + return m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)].UPB_PRIVATE(submsg); } UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_GetSubMessageTable( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - UPB_ASSERT(upb_MiniTableField_CType(f) == kUpb_CType_Message); - const struct upb_MiniTable* ret = upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + const struct upb_MiniTable* ret = upb_MiniTable_SubMessage(m, f); UPB_ASSUME(ret); return UPB_PRIVATE(_upb_MiniTable_IsEmpty)(ret) ? NULL : ret; } -UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_SubMessage( +UPB_API_INLINE bool upb_MiniTable_FieldIsLinked( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - if (upb_MiniTableField_CType(f) != kUpb_CType_Message) { - return NULL; - } - return upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + return upb_MiniTable_GetSubMessageTable(m, f) != NULL; +} + +UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_MapEntrySubMessage( + const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { + UPB_ASSERT(upb_MiniTable_FieldIsLinked(m, f)); // Map entries must be linked. + UPB_ASSERT(upb_MiniTableField_IsMap(f)); // Function precondition. + return upb_MiniTable_SubMessage(m, f); } UPB_API_INLINE const struct upb_MiniTableEnum* upb_MiniTable_GetSubEnumTable( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { UPB_ASSERT(upb_MiniTableField_CType(f) == kUpb_CType_Enum); - return upb_MiniTableSub_Enum( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + return m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)].UPB_PRIVATE( + subenum); } UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_MapKey( @@ -143,11 +155,6 @@ UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_MapValue( return f; } -UPB_API_INLINE bool upb_MiniTable_FieldIsLinked( - const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - return upb_MiniTable_GetSubMessageTable(m, f) != NULL; -} - // Computes a bitmask in which the |m->required_count| lowest bits are set. // // Sample output: diff --git a/upb/mini_table/message.h b/upb/mini_table/message.h index 1ce9087b0a..6589762f4e 100644 --- a/upb/mini_table/message.h +++ b/upb/mini_table/message.h @@ -34,10 +34,20 @@ UPB_API_INLINE int upb_MiniTable_FieldCount(const upb_MiniTable* m); UPB_API_INLINE const upb_MiniTable* upb_MiniTable_GetSubMessageTable( const upb_MiniTable* m, const upb_MiniTableField* f); -// Returns the MiniTable for a message field if it is a submessage. +// Returns the MiniTable for a message field if it is a submessage, otherwise +// returns NULL. +// +// WARNING: if dynamic tree shaking is in use, the return value may be the +// "empty", zero-field placeholder message instead of the real message type. +// If the message is later linked, this function will begin returning the real +// message type. UPB_API_INLINE const upb_MiniTable* upb_MiniTable_SubMessage( const upb_MiniTable* m, const upb_MiniTableField* f); +// Returns the MiniTable for a map field. The given field must refer to a map. +UPB_API_INLINE const upb_MiniTable* upb_MiniTable_MapEntrySubMessage( + const upb_MiniTable* m, const upb_MiniTableField* f); + // Returns the MiniTableEnum for a message field, NULL if the field is unlinked. UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTable_GetSubEnumTable( const upb_MiniTable* m, const upb_MiniTableField* f); From d1cf09abcff7636101fd27e94ea5aa7b9f603f5e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 19 Apr 2024 22:43:34 +0000 Subject: [PATCH 012/179] Auto-generate files after cl/626485595 --- php/ext/google/protobuf/php-upb.c | 11 ++-- php/ext/google/protobuf/php-upb.h | 72 ++++++++++++++++----------- ruby/ext/google/protobuf_c/ruby-upb.c | 11 ++-- ruby/ext/google/protobuf_c/ruby-upb.h | 72 ++++++++++++++++----------- 4 files changed, 94 insertions(+), 72 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 9f653e2c3a..d32863b2cc 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -5137,10 +5137,8 @@ bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* m, const upb_MiniTableField* f, upb_Message* map_entry_message, upb_Arena* arena) { UPB_ASSERT(!upb_Message_IsFrozen(map_entry_message)); - - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_mini_table = upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* map_entry_mini_table = + upb_MiniTable_MapEntrySubMessage(m, f); UPB_ASSERT(map_entry_mini_table); const upb_MiniTableField* map_entry_key_field = upb_MiniTable_MapKey(map_entry_mini_table); @@ -6043,9 +6041,8 @@ static upb_Map* upb_Message_Map_DeepClone(const upb_Map* map, const upb_MiniTableField* f, upb_Message* clone, upb_Arena* arena) { - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* map_entry_table = + upb_MiniTable_MapEntrySubMessage(mini_table, f); UPB_ASSERT(map_entry_table); const upb_MiniTableField* key_field = upb_MiniTable_MapKey(map_entry_table); diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 4462920cb6..fa63a63297 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -1409,6 +1409,7 @@ UPB_API_INLINE bool upb_MiniTableEnum_CheckValue(const upb_MiniTableEnum* e, #ifndef UPB_MINI_TABLE_INTERNAL_MESSAGE_H_ #define UPB_MINI_TABLE_INTERNAL_MESSAGE_H_ +#include #include @@ -1540,34 +1541,44 @@ UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_GetFieldByIndex( return &m->UPB_ONLYBITS(fields)[i]; } -UPB_INLINE const union upb_MiniTableSub* UPB_PRIVATE( +UPB_INLINE const union upb_MiniTableSub UPB_PRIVATE( _upb_MiniTable_GetSubByIndex)(const struct upb_MiniTable* m, uint32_t i) { - return &m->UPB_PRIVATE(subs)[i]; + return m->UPB_PRIVATE(subs)[i]; +} + +UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_SubMessage( + const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { + if (upb_MiniTableField_CType(f) != kUpb_CType_Message) { + return NULL; + } + return m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)].UPB_PRIVATE(submsg); } UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_GetSubMessageTable( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - UPB_ASSERT(upb_MiniTableField_CType(f) == kUpb_CType_Message); - const struct upb_MiniTable* ret = upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + const struct upb_MiniTable* ret = upb_MiniTable_SubMessage(m, f); UPB_ASSUME(ret); return UPB_PRIVATE(_upb_MiniTable_IsEmpty)(ret) ? NULL : ret; } -UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_SubMessage( +UPB_API_INLINE bool upb_MiniTable_FieldIsLinked( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - if (upb_MiniTableField_CType(f) != kUpb_CType_Message) { - return NULL; - } - return upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + return upb_MiniTable_GetSubMessageTable(m, f) != NULL; +} + +UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_MapEntrySubMessage( + const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { + UPB_ASSERT(upb_MiniTable_FieldIsLinked(m, f)); // Map entries must be linked. + UPB_ASSERT(upb_MiniTableField_IsMap(f)); // Function precondition. + return upb_MiniTable_SubMessage(m, f); } UPB_API_INLINE const struct upb_MiniTableEnum* upb_MiniTable_GetSubEnumTable( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { UPB_ASSERT(upb_MiniTableField_CType(f) == kUpb_CType_Enum); - return upb_MiniTableSub_Enum( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + return m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)].UPB_PRIVATE( + subenum); } UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_MapKey( @@ -1586,11 +1597,6 @@ UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_MapValue( return f; } -UPB_API_INLINE bool upb_MiniTable_FieldIsLinked( - const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - return upb_MiniTable_GetSubMessageTable(m, f) != NULL; -} - // Computes a bitmask in which the |m->required_count| lowest bits are set. // // Sample output: @@ -1645,10 +1651,20 @@ UPB_API_INLINE int upb_MiniTable_FieldCount(const upb_MiniTable* m); UPB_API_INLINE const upb_MiniTable* upb_MiniTable_GetSubMessageTable( const upb_MiniTable* m, const upb_MiniTableField* f); -// Returns the MiniTable for a message field if it is a submessage. +// Returns the MiniTable for a message field if it is a submessage, otherwise +// returns NULL. +// +// WARNING: if dynamic tree shaking is in use, the return value may be the +// "empty", zero-field placeholder message instead of the real message type. +// If the message is later linked, this function will begin returning the real +// message type. UPB_API_INLINE const upb_MiniTable* upb_MiniTable_SubMessage( const upb_MiniTable* m, const upb_MiniTableField* f); +// Returns the MiniTable for a map field. The given field must refer to a map. +UPB_API_INLINE const upb_MiniTable* upb_MiniTable_MapEntrySubMessage( + const upb_MiniTable* m, const upb_MiniTableField* f); + // Returns the MiniTableEnum for a message field, NULL if the field is unlinked. UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTable_GetSubEnumTable( const upb_MiniTable* m, const upb_MiniTableField* f); @@ -3047,12 +3063,6 @@ UPB_API_INLINE bool upb_Map_IsFrozen(const upb_Map* map); #endif /* UPB_MESSAGE_MAP_H_ */ -#ifndef UPB_MINI_TABLE_TAGGED_PTR_H_ -#define UPB_MINI_TABLE_TAGGED_PTR_H_ - -#include - - // Public APIs for message operations that do not depend on the schema. // // MiniTable-based accessors live in accessors.h. @@ -3108,6 +3118,12 @@ UPB_INLINE void upb_Message_LogNewMessage(const upb_MiniTable* mini_table, #endif /* UPB_MESSAGE_MESSAGE_H_ */ +#ifndef UPB_MINI_TABLE_TAGGED_PTR_H_ +#define UPB_MINI_TABLE_TAGGED_PTR_H_ + +#include + + // Must be last. // When a upb_Message* is stored in a message, array, or map, it is stored in a @@ -3501,8 +3517,6 @@ UPB_API_INLINE void _upb_Message_SetTaggedMessagePtr( UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - UPB_ASSERT(upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)])); upb_Message_SetBaseField(msg, field, &sub_message); } @@ -3526,8 +3540,8 @@ UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( upb_Message* sub_message = *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); if (!sub_message) { - const upb_MiniTable* sub_mini_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* sub_mini_table = + upb_MiniTable_SubMessage(mini_table, field); UPB_ASSERT(sub_mini_table); sub_message = _upb_Message_New(sub_mini_table, arena); *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*) = sub_message; diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index cc42c374d8..9dfbd6510e 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -4630,10 +4630,8 @@ bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* m, const upb_MiniTableField* f, upb_Message* map_entry_message, upb_Arena* arena) { UPB_ASSERT(!upb_Message_IsFrozen(map_entry_message)); - - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_mini_table = upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* map_entry_mini_table = + upb_MiniTable_MapEntrySubMessage(m, f); UPB_ASSERT(map_entry_mini_table); const upb_MiniTableField* map_entry_key_field = upb_MiniTable_MapKey(map_entry_mini_table); @@ -5536,9 +5534,8 @@ static upb_Map* upb_Message_Map_DeepClone(const upb_Map* map, const upb_MiniTableField* f, upb_Message* clone, upb_Arena* arena) { - // TODO: use a variant of upb_MiniTable_GetSubMessageTable() here. - const upb_MiniTable* map_entry_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* map_entry_table = + upb_MiniTable_MapEntrySubMessage(mini_table, f); UPB_ASSERT(map_entry_table); const upb_MiniTableField* key_field = upb_MiniTable_MapKey(map_entry_table); diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index f88cd1ff86..76b6af3e2b 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -1411,6 +1411,7 @@ UPB_API_INLINE bool upb_MiniTableEnum_CheckValue(const upb_MiniTableEnum* e, #ifndef UPB_MINI_TABLE_INTERNAL_MESSAGE_H_ #define UPB_MINI_TABLE_INTERNAL_MESSAGE_H_ +#include #include @@ -1542,34 +1543,44 @@ UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_GetFieldByIndex( return &m->UPB_ONLYBITS(fields)[i]; } -UPB_INLINE const union upb_MiniTableSub* UPB_PRIVATE( +UPB_INLINE const union upb_MiniTableSub UPB_PRIVATE( _upb_MiniTable_GetSubByIndex)(const struct upb_MiniTable* m, uint32_t i) { - return &m->UPB_PRIVATE(subs)[i]; + return m->UPB_PRIVATE(subs)[i]; +} + +UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_SubMessage( + const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { + if (upb_MiniTableField_CType(f) != kUpb_CType_Message) { + return NULL; + } + return m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)].UPB_PRIVATE(submsg); } UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_GetSubMessageTable( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - UPB_ASSERT(upb_MiniTableField_CType(f) == kUpb_CType_Message); - const struct upb_MiniTable* ret = upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + const struct upb_MiniTable* ret = upb_MiniTable_SubMessage(m, f); UPB_ASSUME(ret); return UPB_PRIVATE(_upb_MiniTable_IsEmpty)(ret) ? NULL : ret; } -UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_SubMessage( +UPB_API_INLINE bool upb_MiniTable_FieldIsLinked( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - if (upb_MiniTableField_CType(f) != kUpb_CType_Message) { - return NULL; - } - return upb_MiniTableSub_Message( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + return upb_MiniTable_GetSubMessageTable(m, f) != NULL; +} + +UPB_API_INLINE const struct upb_MiniTable* upb_MiniTable_MapEntrySubMessage( + const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { + UPB_ASSERT(upb_MiniTable_FieldIsLinked(m, f)); // Map entries must be linked. + UPB_ASSERT(upb_MiniTableField_IsMap(f)); // Function precondition. + return upb_MiniTable_SubMessage(m, f); } UPB_API_INLINE const struct upb_MiniTableEnum* upb_MiniTable_GetSubEnumTable( const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { UPB_ASSERT(upb_MiniTableField_CType(f) == kUpb_CType_Enum); - return upb_MiniTableSub_Enum( - m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)]); + return m->UPB_PRIVATE(subs)[f->UPB_PRIVATE(submsg_index)].UPB_PRIVATE( + subenum); } UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_MapKey( @@ -1588,11 +1599,6 @@ UPB_API_INLINE const struct upb_MiniTableField* upb_MiniTable_MapValue( return f; } -UPB_API_INLINE bool upb_MiniTable_FieldIsLinked( - const struct upb_MiniTable* m, const struct upb_MiniTableField* f) { - return upb_MiniTable_GetSubMessageTable(m, f) != NULL; -} - // Computes a bitmask in which the |m->required_count| lowest bits are set. // // Sample output: @@ -1647,10 +1653,20 @@ UPB_API_INLINE int upb_MiniTable_FieldCount(const upb_MiniTable* m); UPB_API_INLINE const upb_MiniTable* upb_MiniTable_GetSubMessageTable( const upb_MiniTable* m, const upb_MiniTableField* f); -// Returns the MiniTable for a message field if it is a submessage. +// Returns the MiniTable for a message field if it is a submessage, otherwise +// returns NULL. +// +// WARNING: if dynamic tree shaking is in use, the return value may be the +// "empty", zero-field placeholder message instead of the real message type. +// If the message is later linked, this function will begin returning the real +// message type. UPB_API_INLINE const upb_MiniTable* upb_MiniTable_SubMessage( const upb_MiniTable* m, const upb_MiniTableField* f); +// Returns the MiniTable for a map field. The given field must refer to a map. +UPB_API_INLINE const upb_MiniTable* upb_MiniTable_MapEntrySubMessage( + const upb_MiniTable* m, const upb_MiniTableField* f); + // Returns the MiniTableEnum for a message field, NULL if the field is unlinked. UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTable_GetSubEnumTable( const upb_MiniTable* m, const upb_MiniTableField* f); @@ -3049,12 +3065,6 @@ UPB_API_INLINE bool upb_Map_IsFrozen(const upb_Map* map); #endif /* UPB_MESSAGE_MAP_H_ */ -#ifndef UPB_MINI_TABLE_TAGGED_PTR_H_ -#define UPB_MINI_TABLE_TAGGED_PTR_H_ - -#include - - // Public APIs for message operations that do not depend on the schema. // // MiniTable-based accessors live in accessors.h. @@ -3110,6 +3120,12 @@ UPB_INLINE void upb_Message_LogNewMessage(const upb_MiniTable* mini_table, #endif /* UPB_MESSAGE_MESSAGE_H_ */ +#ifndef UPB_MINI_TABLE_TAGGED_PTR_H_ +#define UPB_MINI_TABLE_TAGGED_PTR_H_ + +#include + + // Must be last. // When a upb_Message* is stored in a message, array, or map, it is stored in a @@ -3503,8 +3519,6 @@ UPB_API_INLINE void _upb_Message_SetTaggedMessagePtr( UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - UPB_ASSERT(upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)])); upb_Message_SetBaseField(msg, field, &sub_message); } @@ -3528,8 +3542,8 @@ UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( upb_Message* sub_message = *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); if (!sub_message) { - const upb_MiniTable* sub_mini_table = upb_MiniTableSub_Message( - mini_table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]); + const upb_MiniTable* sub_mini_table = + upb_MiniTable_SubMessage(mini_table, field); UPB_ASSERT(sub_mini_table); sub_message = _upb_Message_New(sub_mini_table, arena); *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*) = sub_message; From eaee04d89719ed8e0fde96b6d37cf741d866f511 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Apr 2024 16:02:13 -0700 Subject: [PATCH 013/179] Avoid boxing when serializing primitive lists in CodedOutputStreamWriter Previously, these List.get methods returned Object, forcing boxing, and (if not in the primitive's box cache) allocation, with all the cost that entails. I've taken the approach of duplicating methods to specialise for primitives, like BinaryWriter does for non-lite protos. I initially considered checking the class of the list on every iteration of the loop, but that feels wasteful, when we can check it once at the start of the loop. This also means we have the same behaviour as serverside protos. At the cost of a few more methods, but hopefully they're trivially inlineable, so hopefully leading to a small dex increase without really increasing the method count. Given this is a public API, I don't think we can remove the List overloads either. PiperOrigin-RevId: 627183583 --- .../protobuf/CodedOutputStreamWriter.java | 417 +++++++++++++++++- 1 file changed, 416 insertions(+), 1 deletion(-) diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java index 6c060cc3ab..5368f5dede 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java @@ -167,6 +167,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeInt32List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof IntArrayList) { + writeInt32ListInternal(fieldNumber, (IntArrayList) value, packed); + } else { + writeInt32ListInternal(fieldNumber, value, packed); + } + } + + private void writeInt32ListInternal(int fieldNumber, IntArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeInt32SizeNoTag(value.getInt(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeInt32NoTag(value.getInt(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeInt32(fieldNumber, value.getInt(i)); + } + } + } + + private void writeInt32ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -191,6 +223,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeFixed32List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof IntArrayList) { + writeFixed32ListInternal(fieldNumber, (IntArrayList) value, packed); + } else { + writeFixed32ListInternal(fieldNumber, value, packed); + } + } + + private void writeFixed32ListInternal(int fieldNumber, IntArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeFixed32SizeNoTag(value.getInt(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeFixed32NoTag(value.getInt(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeFixed32(fieldNumber, value.getInt(i)); + } + } + } + + private void writeFixed32ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -214,6 +278,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeInt64List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof LongArrayList) { + writeInt64ListInternal(fieldNumber, (LongArrayList) value, packed); + } else { + writeInt64ListInternal(fieldNumber, value, packed); + } + } + + private void writeInt64ListInternal(int fieldNumber, LongArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeInt64SizeNoTag(value.getLong(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeInt64NoTag(value.getLong(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeInt64(fieldNumber, value.getLong(i)); + } + } + } + + private void writeInt64ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -234,10 +330,41 @@ final class CodedOutputStreamWriter implements Writer { } } } - @Override public void writeUInt64List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof LongArrayList) { + writeUInt64ListInternal(fieldNumber, (LongArrayList) value, packed); + } else { + writeUInt64ListInternal(fieldNumber, value, packed); + } + } + + private void writeUInt64ListInternal(int fieldNumber, LongArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeUInt64SizeNoTag(value.getLong(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeUInt64NoTag(value.getLong(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeUInt64(fieldNumber, value.getLong(i)); + } + } + } + + private void writeUInt64ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -262,6 +389,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeFixed64List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof LongArrayList) { + writeFixed64ListInternal(fieldNumber, (LongArrayList) value, packed); + } else { + writeFixed64ListInternal(fieldNumber, value, packed); + } + } + + private void writeFixed64ListInternal(int fieldNumber, LongArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeFixed64SizeNoTag(value.getLong(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeFixed64NoTag(value.getLong(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeFixed64(fieldNumber, value.getLong(i)); + } + } + } + + private void writeFixed64ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -286,6 +445,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeFloatList(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof FloatArrayList) { + writeFloatListInternal(fieldNumber, (FloatArrayList) value, packed); + } else { + writeFloatListInternal(fieldNumber, value, packed); + } + } + + private void writeFloatListInternal(int fieldNumber, FloatArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeFloatSizeNoTag(value.getFloat(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeFloatNoTag(value.getFloat(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeFloat(fieldNumber, value.getFloat(i)); + } + } + } + + private void writeFloatListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -310,6 +501,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeDoubleList(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof DoubleArrayList) { + writeDoubleListInternal(fieldNumber, (DoubleArrayList) value, packed); + } else { + writeDoubleListInternal(fieldNumber, value, packed); + } + } + + private void writeDoubleListInternal(int fieldNumber, DoubleArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeDoubleSizeNoTag(value.getDouble(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeDoubleNoTag(value.getDouble(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeDouble(fieldNumber, value.getDouble(i)); + } + } + } + + private void writeDoubleListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -358,6 +581,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeBoolList(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof BooleanArrayList) { + writeBoolListInternal(fieldNumber, (BooleanArrayList) value, packed); + } else { + writeBoolListInternal(fieldNumber, value, packed); + } + } + + private void writeBoolListInternal(int fieldNumber, BooleanArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeBoolSizeNoTag(value.getBoolean(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeBoolNoTag(value.getBoolean(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeBool(fieldNumber, value.getBoolean(i)); + } + } + } + + private void writeBoolListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -411,6 +666,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeUInt32List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof IntArrayList) { + writeUInt32ListInternal(fieldNumber, (IntArrayList) value, packed); + } else { + writeUInt32ListInternal(fieldNumber, value, packed); + } + } + + private void writeUInt32ListInternal(int fieldNumber, IntArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeUInt32SizeNoTag(value.getInt(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeUInt32NoTag(value.getInt(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeUInt32(fieldNumber, value.getInt(i)); + } + } + } + + public void writeUInt32ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -435,6 +722,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeSFixed32List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof IntArrayList) { + writeSFixed32ListInternal(fieldNumber, (IntArrayList) value, packed); + } else { + writeSFixed32ListInternal(fieldNumber, value, packed); + } + } + + private void writeSFixed32ListInternal(int fieldNumber, IntArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeSFixed32SizeNoTag(value.getInt(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeSFixed32NoTag(value.getInt(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeSFixed32(fieldNumber, value.getInt(i)); + } + } + } + + private void writeSFixed32ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -459,6 +778,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeSFixed64List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof LongArrayList) { + writeSFixed64ListInternal(fieldNumber, (LongArrayList) value, packed); + } else { + writeSFixed64ListInternal(fieldNumber, value, packed); + } + } + + private void writeSFixed64ListInternal(int fieldNumber, LongArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeSFixed64SizeNoTag(value.getLong(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeSFixed64NoTag(value.getLong(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeSFixed64(fieldNumber, value.getLong(i)); + } + } + } + + private void writeSFixed64ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -483,6 +834,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeSInt32List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof IntArrayList) { + writeSInt32ListInternal(fieldNumber, (IntArrayList) value, packed); + } else { + writeSInt32ListInternal(fieldNumber, value, packed); + } + } + + private void writeSInt32ListInternal(int fieldNumber, IntArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeSInt32SizeNoTag(value.getInt(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeSInt32NoTag(value.getInt(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeSInt32(fieldNumber, value.getInt(i)); + } + } + } + + public void writeSInt32ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); @@ -507,6 +890,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeSInt64List(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof LongArrayList) { + writeSInt64ListInternal(fieldNumber, (LongArrayList) value, packed); + } else { + writeSInt64ListInternal(fieldNumber, value, packed); + } + } + + private void writeSInt64ListInternal(int fieldNumber, LongArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeSInt64SizeNoTag(value.getLong(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeSInt64NoTag(value.getLong(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeSInt64(fieldNumber, value.getLong(i)); + } + } + } + + private void writeSInt64ListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); From 1a7aa2c4e8b896cedb649b001cba333d60b3ed00 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Apr 2024 16:21:52 -0700 Subject: [PATCH 014/179] Internal changes. PiperOrigin-RevId: 627188953 --- python/google/protobuf/runtime_version.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/google/protobuf/runtime_version.py b/python/google/protobuf/runtime_version.py index bc4061cfeb..7f294b36fb 100644 --- a/python/google/protobuf/runtime_version.py +++ b/python/google/protobuf/runtime_version.py @@ -42,6 +42,10 @@ class VersionError(Exception): """Exception class for version violation.""" +def _ReportVersionError(msg): + raise VersionError(msg) + + def ValidateProtobufRuntimeVersion( gen_domain, gen_major, gen_minor, gen_patch, gen_suffix, location ): @@ -76,28 +80,28 @@ def ValidateProtobufRuntimeVersion( ) if gen_domain != DOMAIN: - raise VersionError( + _ReportVersionError( 'Detected mismatched Protobuf Gencode/Runtime domains when loading' f' {location}: gencode {gen_domain.name} runtime {DOMAIN.name}.' ' Cross-domain usage of Protobuf is not supported.' ) if gen_major != MAJOR: - raise VersionError( + _ReportVersionError( 'Detected mismatched Protobuf Gencode/Runtime major versions when' f' loading {location}: gencode {gen_version} runtime {version}.' f' Same major version is required. {error_prompt}' ) if MINOR < gen_minor or (MINOR == gen_minor and PATCH < gen_patch): - raise VersionError( + _ReportVersionError( 'Detected incompatible Protobuf Gencode/Runtime versions when loading' f' {location}: gencode {gen_version} runtime {version}. Runtime version' f' cannot be older than the linked gencode version. {error_prompt}' ) if gen_suffix != SUFFIX: - raise VersionError( + _ReportVersionError( 'Detected mismatched Protobuf Gencode/Runtime version suffixes when' f' loading {location}: gencode {gen_version} runtime {version}.' f' Version suffixes must be the same. {error_prompt}' From a9e0480768ee011180412a391010823474667ded Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 22 Apr 2024 17:26:44 -0700 Subject: [PATCH 015/179] Use Emit for ByteSizeLong PiperOrigin-RevId: 627204923 --- .../golden/compare_cpp_codegen_failure.txt | 14 +- .../golden/compare_cpp_codegen_failure.xml | 2 +- .../cpp/field_generators/enum_field.cc | 16 +- .../cpp/field_generators/primitive_field.cc | 10 +- src/google/protobuf/compiler/cpp/message.cc | 510 ++++++---- .../compiler/java/java_features.pb.cc | 9 +- src/google/protobuf/compiler/plugin.pb.cc | 92 +- src/google/protobuf/cpp_features.pb.cc | 9 +- src/google/protobuf/descriptor.pb.cc | 936 +++++++++--------- 9 files changed, 843 insertions(+), 755 deletions(-) diff --git a/editions/golden/compare_cpp_codegen_failure.txt b/editions/golden/compare_cpp_codegen_failure.txt index 03be36fe32..27ef82e033 100644 --- a/editions/golden/compare_cpp_codegen_failure.txt +++ b/editions/golden/compare_cpp_codegen_failure.txt @@ -25,14 +25,14 @@ target = ::proto2::internal::WireFormatLite:: WriteInt32ToArrayWithField<1>( @@ @@ - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; -- // optional int32 int32_field = 1; -+ // int32 int32_field = 1; - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + { +- // optional int32 int32_field = 1; ++ // int32 int32_field = 1; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( [ FAILED ] third_party/protobuf/editions/golden/simple_proto3.pb.cc [ RUN ] third_party/protobuf/editions/golden/simple_proto3.pb.h @@ @@ diff --git a/editions/golden/compare_cpp_codegen_failure.xml b/editions/golden/compare_cpp_codegen_failure.xml index 5c96f33eea..33f961215f 100644 --- a/editions/golden/compare_cpp_codegen_failure.xml +++ b/editions/golden/compare_cpp_codegen_failure.xml @@ -2,7 +2,7 @@ - + diff --git a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc index 529014ea89..9e83ad2f50 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc @@ -544,17 +544,15 @@ void RepeatedEnum::GenerateByteSize(io::Printer* p) const { }}, }, R"cc( - { - std::size_t data_size = 0; - auto count = static_cast(this->_internal_$name$_size()); + std::size_t data_size = 0; + auto count = static_cast(this->_internal_$name$_size()); - for (std::size_t i = 0; i < count; ++i) { - data_size += ::_pbi::WireFormatLite::EnumSize( - this->_internal_$name$().Get(static_cast(i))); - } - total_size += data_size; - $add_to_size$; + for (std::size_t i = 0; i < count; ++i) { + data_size += ::_pbi::WireFormatLite::EnumSize( + this->_internal_$name$().Get(static_cast(i))); } + total_size += data_size; + $add_to_size$; )cc"); } } // namespace diff --git a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc index 607fe10aad..22279db2bf 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc @@ -635,12 +635,10 @@ void RepeatedPrimitive::GenerateByteSize(io::Printer* p) const { .WithSuffix(""), }, R"cc( - { - std::size_t data_size = $data_size$; - $maybe_cache_data_size$; - std::size_t tag_size = $tag_size$; - total_size += tag_size + data_size; - } + std::size_t data_size = $data_size$; + $maybe_cache_data_size$; + std::size_t tag_size = $tag_size$; + total_size += tag_size + data_size; )cc"); } } // namespace diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index b54749c245..9e43396b9d 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -183,50 +183,55 @@ RunMap FindRuns(const std::vector& fields, return runs; } -// Emits an if-statement with a condition that evaluates to true if |field| is -// considered non-default (will be sent over the wire), for message types -// without true field presence. Should only be called if -// !HasHasbit(field). -bool EmitFieldNonDefaultCondition(io::Printer* p, const std::string& prefix, - const FieldDescriptor* field) { +void EmitNonDefaultCheck(io::Printer* p, const std::string& prefix, + const FieldDescriptor* field) { ABSL_CHECK(!HasHasbit(field)); + ABSL_CHECK(!field->is_repeated()); + ABSL_CHECK(!field->containing_oneof() || field->real_containing_oneof()); + auto v = p->WithVars({{ {"prefix", prefix}, {"name", FieldName(field)}, }}); // Merge and serialize semantics: primitive fields are merged/serialized only // if non-zero (numeric) or non-empty (string). - if (!field->is_repeated() && !field->containing_oneof()) { + if (!field->containing_oneof()) { if (field->cpp_type() == FieldDescriptor::CPPTYPE_STRING) { - p->Emit(R"cc( - if (!$prefix$_internal_$name$().empty()) { - )cc"); + p->Emit("!$prefix$_internal_$name$().empty()"); } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { // Message fields still have has_$name$() methods. - p->Emit(R"cc( - if ($prefix$_internal_has_$name$()) { - )cc"); + p->Emit("$prefix$_internal_has_$name$()"); } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT) { - p->Emit(R"cc( - if (::absl::bit_cast<::uint32_t>($prefix$_internal_$name$()) != 0) { - )cc"); + p->Emit("::absl::bit_cast<::uint32_t>($prefix$_internal_$name$()) != 0"); } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_DOUBLE) { - p->Emit(R"cc( - if (::absl::bit_cast<::uint64_t>($prefix$_internal_$name$()) != 0) { - )cc"); + p->Emit("::absl::bit_cast<::uint64_t>($prefix$_internal_$name$()) != 0"); } else { - p->Emit(R"cc( - if ($prefix$_internal_$name$() != 0) { - )cc"); + p->Emit("$prefix$_internal_$name$() != 0"); } - return true; } else if (field->real_containing_oneof()) { - p->Emit(R"cc( - if ($has_field$) { - )cc"); - return true; + p->Emit("$has_field$"); } - return false; +} + +bool ShouldEmitNonDefaultCheck(const FieldDescriptor* field) { + return (!field->is_repeated() && !field->containing_oneof()) || + field->real_containing_oneof(); +} + +// Emits an if-statement with a condition that evaluates to true if |field| is +// considered non-default (will be sent over the wire), for message types +// without true field presence. Should only be called if +// !HasHasbit(field). +bool MayEmitIfNonDefaultCheck(io::Printer* p, const std::string& prefix, + const FieldDescriptor* field) { + ABSL_CHECK(!HasHasbit(field)); + if (!ShouldEmitNonDefaultCheck(field)) return false; + + p->Emit({{"condition", [&] { EmitNonDefaultCheck(p, prefix, field); }}}, + R"cc( + if ($condition$) { + )cc"); + return true; } bool HasInternalHasMethod(const FieldDescriptor* field) { @@ -3857,8 +3862,7 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) { } else if (field->is_optional() && !HasHasbit(field)) { // Merge semantics without true field presence: primitive fields are // merged only if non-zero (numeric) or non-empty (string). - bool have_enclosing_if = - EmitFieldNonDefaultCondition(p, "from.", field); + bool have_enclosing_if = MayEmitIfNonDefaultCheck(p, "from.", field); if (have_enclosing_if) format.Indent(); generator.GenerateMergingCode(p); if (have_enclosing_if) { @@ -4130,7 +4134,7 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* p, } )cc"); } else if (field->is_optional()) { - bool have_enclosing_if = EmitFieldNonDefaultCondition(p, "this->", field); + bool have_enclosing_if = MayEmitIfNonDefaultCheck(p, "this->", field); if (have_enclosing_if) p->Indent(); emit_body(); if (have_enclosing_if) { @@ -4551,23 +4555,6 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) { return; } - Formatter format(p); - format( - "::size_t $classname$::ByteSizeLong() const {\n" - "$WeakDescriptorSelfPin$" - "$annotate_bytesize$" - "// @@protoc_insertion_point(message_byte_size_start:$full_name$)\n"); - format.Indent(); - format( - "::size_t total_size = 0;\n" - "\n"); - - if (descriptor_->extension_range_count() > 0) { - format( - "total_size += $extensions$.ByteSize();\n" - "\n"); - } - std::vector chunks = CollectFields( optimized_order_, options_, [&](const FieldDescriptor* a, const FieldDescriptor* b) -> bool { @@ -4576,177 +4563,278 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) { ShouldSplit(a, options_) == ShouldSplit(b, options_); }); - auto it = chunks.begin(); - auto end = chunks.end(); - int cached_has_word_index = -1; - - format( - "$uint32$ cached_has_bits = 0;\n" - "// Prevent compiler warnings about cached_has_bits being unused\n" - "(void) cached_has_bits;\n\n"); - - // See comment in third_party/protobuf/port.h for details, - // on how much we are prefetching. Only insert prefetch once per - // function, since advancing is actually slower. We sometimes - // prefetch more than sizeof(message), because it helps with - // next message on arena. - bool generate_prefetch = false; - // Skip trivial messages with 0 or 1 fields, unless they are repeated, - // to reduce codesize. - switch (optimized_order_.size()) { - case 1: - generate_prefetch = optimized_order_[0]->is_repeated(); - break; - case 0: - break; - default: - generate_prefetch = true; - } - if (!IsPresentMessage(descriptor_, options_)) { - generate_prefetch = false; - } - if (generate_prefetch) { - p->Emit(R"cc( - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - )cc"); - } - - while (it != end) { - auto next = FindNextUnequalChunk(it, end, MayGroupChunksForHaswordsCheck); - bool has_haswords_check = MaybeEmitHaswordsCheck( - it, next, options_, has_bit_indices_, cached_has_word_index, "", p); - - while (it != next) { - const std::vector& fields = it->fields; - const bool check_has_byte = fields.size() > 1 && - HasWordIndex(fields[0]) != kNoHasbit && - !IsLikelyPresent(fields.back(), options_); - - if (check_has_byte) { - // Emit an if() that will let us skip the whole chunk if none are set. - uint32_t chunk_mask = GenChunkMask(fields, has_bit_indices_); - std::string chunk_mask_str = - absl::StrCat(absl::Hex(chunk_mask, absl::kZeroPad8)); - - // Check (up to) 8 has_bits at a time if we have more than one field in - // this chunk. Due to field layout ordering, we may check - // _has_bits_[last_chunk * 8 / 32] multiple times. - ABSL_DCHECK_LE(2, popcnt(chunk_mask)); - ABSL_DCHECK_GE(8, popcnt(chunk_mask)); - - if (cached_has_word_index != HasWordIndex(fields.front())) { - cached_has_word_index = HasWordIndex(fields.front()); - format("cached_has_bits = $has_bits$[$1$];\n", cached_has_word_index); - } - format("if (cached_has_bits & 0x$1$u) {\n", chunk_mask_str); - format.Indent(); - } - - // Go back and emit checks for each of the fields we processed. - for (const auto* field : fields) { - bool have_enclosing_if = false; - - PrintFieldComment(format, field, options_); - - if (field->is_repeated()) { - // No presence check is required. - } else if (HasHasbit(field)) { - PrintPresenceCheck(field, has_bit_indices_, p, - &cached_has_word_index); - have_enclosing_if = true; - } else { - // Without field presence: field is serialized only if it has a - // non-default value. - have_enclosing_if = EmitFieldNonDefaultCondition(p, "this->", field); - } + p->Emit( + {{"handle_extension_set", + [&] { + if (descriptor_->extension_range_count() == 0) return; + p->Emit(R"cc( + total_size += $extensions$.ByteSize(); + )cc"); + }}, + {"prefetch", + [&] { + // See comment in third_party/protobuf/port.h for details, + // on how much we are prefetching. Only insert prefetch once per + // function, since advancing is actually slower. We sometimes + // prefetch more than sizeof(message), because it helps with + // next message on arena. + bool generate_prefetch = false; + // Skip trivial messages with 0 or 1 fields, unless they are + // repeated, to reduce codesize. + switch (optimized_order_.size()) { + case 1: + generate_prefetch = optimized_order_[0]->is_repeated(); + break; + case 0: + break; + default: + generate_prefetch = true; + } + if (!generate_prefetch || !IsPresentMessage(descriptor_, options_)) { + return; + } + p->Emit(R"cc( + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + )cc"); + }}, + {"handle_fields", + [&] { + auto it = chunks.begin(); + auto end = chunks.end(); + int cached_has_word_index = -1; + + while (it != end) { + auto next = + FindNextUnequalChunk(it, end, MayGroupChunksForHaswordsCheck); + bool has_haswords_check = + MaybeEmitHaswordsCheck(it, next, options_, has_bit_indices_, + cached_has_word_index, "", p); + + while (it != next) { + const auto& fields = it->fields; + const bool check_has_byte = + fields.size() > 1 && HasWordIndex(fields[0]) != kNoHasbit && + !IsLikelyPresent(fields.back(), options_); + p->Emit( + {{"update_byte_size_for_chunk", + [&] { + // Go back and emit checks for each of the fields we + // processed. + for (const auto* field : fields) { + p->Emit( + {{"comment", + [&] { + PrintFieldComment(Formatter{p}, field, + options_); + }}, + {"update_byte_size_for_field", + [&] { + field_generators_.get(field).GenerateByteSize( + p); + }}, + {"update_cached_has_bits", + [&] { + if (!HasHasbit(field) || + field->options().weak()) + return; + int has_bit_index = + has_bit_indices_[field->index()]; + if (cached_has_word_index == + (has_bit_index / 32)) + return; + cached_has_word_index = (has_bit_index / 32); + p->Emit({{"index", cached_has_word_index}}, + R"cc( + cached_has_bits = $has_bits$[$index$]; + )cc"); + }}, + {"check_if_field_present", + [&] { + if (HasHasbit(field)) { + if (field->options().weak()) { + p->Emit("if (has_$name$())"); + return; + } + + int has_bit_index = + has_bit_indices_[field->index()]; + p->Emit({{"mask", + absl::StrFormat( + "0x%08xu", + 1u << (has_bit_index % 32))}}, + "if (cached_has_bits & $mask$)"); + } else if (ShouldEmitNonDefaultCheck(field)) { + // Without field presence: field is + // serialized only if it has a non-default + // value. + p->Emit({{"non_default_check", + [&] { + EmitNonDefaultCheck(p, "this->", + field); + }}}, + "if ($non_default_check$)"); + } + }}}, + R"cc( + $comment$; + $update_cached_has_bits$; + $check_if_field_present$ { + //~ Force newline. + $update_byte_size_for_field$; + } + )cc"); + } + }}, + {"may_update_cached_has_word_index", + [&] { + if (!check_has_byte) return; + if (cached_has_word_index == HasWordIndex(fields.front())) + return; + + cached_has_word_index = HasWordIndex(fields.front()); + p->Emit({{"index", cached_has_word_index}}, + R"cc( + cached_has_bits = $has_bits$[$index$]; + )cc"); + }}, + {"check_if_chunk_present", + [&] { + if (!check_has_byte) { + return; + } - if (have_enclosing_if) format.Indent(); + // Emit an if() that will let us skip the whole chunk + // if none are set. + uint32_t chunk_mask = + GenChunkMask(fields, has_bit_indices_); - field_generators_.get(field).GenerateByteSize(p); + // Check (up to) 8 has_bits at a time if we have more + // than one field in this chunk. Due to field layout + // ordering, we may check _has_bits_[last_chunk * 8 / + // 32] multiple times. + ABSL_DCHECK_LE(2, popcnt(chunk_mask)); + ABSL_DCHECK_GE(8, popcnt(chunk_mask)); - if (have_enclosing_if) { - format.Outdent(); - format( - "}\n" - "\n"); - } - } + p->Emit( + {{"mask", absl::StrFormat("0x%08xu", chunk_mask)}}, + "if (cached_has_bits & $mask$)"); + }}}, + R"cc( + $may_update_cached_has_word_index$; + $check_if_chunk_present$ { + //~ Force newline. + $update_byte_size_for_chunk$; + } + )cc"); - if (check_has_byte) { - format.Outdent(); - format("}\n"); - } + // To next chunk. + ++it; + } - // To next chunk. - ++it; - } + if (has_haswords_check) { + p->Emit(R"cc( + } + )cc"); - if (has_haswords_check) { - p->Outdent(); - p->Emit(R"cc( + // Reset here as it may have been updated in just closed if + // statement. + cached_has_word_index = -1; + } + } + }}, + {"handle_oneof_fields", + [&] { + // Fields inside a oneof don't use _has_bits_ so we count them in a + // separate pass. + for (auto oneof : OneOfRange(descriptor_)) { + p->Emit( + {{"oneof_name", oneof->name()}, + {"oneof_case_name", absl::AsciiStrToUpper(oneof->name())}, + {"case_per_field", + [&] { + for (auto field : FieldRange(oneof)) { + PrintFieldComment(Formatter{p}, field, options_); + p->Emit( + {{"field_name", + UnderscoresToCamelCase(field->name(), true)}, + {"field_byte_size", + [&] { + field_generators_.get(field).GenerateByteSize(p); + }}}, + R"cc( + case k$field_name$: { + $field_byte_size$; + break; + } + )cc"); + } + }}}, + R"cc( + switch ($oneof_name$_case()) { + $case_per_field$; + case $oneof_case_name$_NOT_SET: { + break; + } + } + )cc"); + } + }}, + {"handle_weak_fields", + [&] { + if (num_weak_fields_ == 0) return; + // TagSize + MessageSize + p->Emit(R"cc( + total_size += $weak_field_map$.ByteSizeLong(); + )cc"); + }}, + {"handle_unknown_fields", + [&] { + if (UseUnknownFieldSet(descriptor_->file(), options_)) { + // We go out of our way to put the computation of the uncommon + // path of unknown fields in tail position. This allows for + // better code generation of this function for simple protos. + p->Emit(R"cc( + return MaybeComputeUnknownFieldsSize(total_size, &$cached_size$); + )cc"); + } else { + // We update _cached_size_ even though this is a const method. + // Because const methods might be called concurrently this needs + // to be atomic operations or the program is undefined. In + // practice, since any concurrent writes will be writing the + // exact same value, normal writes will work on all common + // processors. We use a dedicated wrapper class to abstract away + // the underlying atomic. This makes it easier on platforms where + // even relaxed memory order might have perf impact to replace it + // with ordinary loads and stores. + p->Emit(R"cc( + if (PROTOBUF_PREDICT_FALSE($have_unknown_fields$)) { + total_size += $unknown_fields$.size(); + } + $cached_size$.Set(::_pbi::ToCachedSize(total_size)); + return total_size; + )cc"); + } + }}}, + R"cc( + ::size_t $classname$::ByteSizeLong() const { + $WeakDescriptorSelfPin$; + $annotate_bytesize$; + // @@protoc_insertion_point(message_byte_size_start:$full_name$) + ::size_t total_size = 0; + $handle_extension_set$; + + $uint32$ cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void)cached_has_bits; + + $prefetch$; + $handle_fields$; + $handle_oneof_fields$; + $handle_weak_fields$; + $handle_unknown_fields$; } )cc"); - - // Reset here as it may have been updated in just closed if statement. - cached_has_word_index = -1; - } - } - - // Fields inside a oneof don't use _has_bits_ so we count them in a separate - // pass. - for (auto oneof : OneOfRange(descriptor_)) { - format("switch ($1$_case()) {\n", oneof->name()); - format.Indent(); - for (auto field : FieldRange(oneof)) { - PrintFieldComment(format, field, options_); - format("case k$1$: {\n", UnderscoresToCamelCase(field->name(), true)); - format.Indent(); - field_generators_.get(field).GenerateByteSize(p); - format("break;\n"); - format.Outdent(); - format("}\n"); - } - format( - "case $1$_NOT_SET: {\n" - " break;\n" - "}\n", - absl::AsciiStrToUpper(oneof->name())); - format.Outdent(); - format("}\n"); - } - - if (num_weak_fields_) { - // TagSize + MessageSize - format("total_size += $weak_field_map$.ByteSizeLong();\n"); - } - - if (UseUnknownFieldSet(descriptor_->file(), options_)) { - // We go out of our way to put the computation of the uncommon path of - // unknown fields in tail position. This allows for better code generation - // of this function for simple protos. - format( - "return MaybeComputeUnknownFieldsSize(total_size, &$cached_size$);\n"); - } else { - format("if (PROTOBUF_PREDICT_FALSE($have_unknown_fields$)) {\n"); - format(" total_size += $unknown_fields$.size();\n"); - format("}\n"); - - // We update _cached_size_ even though this is a const method. Because - // const methods might be called concurrently this needs to be atomic - // operations or the program is undefined. In practice, since any - // concurrent writes will be writing the exact same value, normal writes - // will work on all common processors. We use a dedicated wrapper class to - // abstract away the underlying atomic. This makes it easier on platforms - // where even relaxed memory order might have perf impact to replace it with - // ordinary loads and stores. - p->Emit(R"cc( - $cached_size$.Set(::_pbi::ToCachedSize(total_size)); - return total_size; - )cc"); - } - - format.Outdent(); - format("}\n"); } bool MessageGenerator::NeedsIsInitialized() { diff --git a/src/google/protobuf/compiler/java/java_features.pb.cc b/src/google/protobuf/compiler/java/java_features.pb.cc index 799299210b..d6b304f1b2 100644 --- a/src/google/protobuf/compiler/java/java_features.pb.cc +++ b/src/google/protobuf/compiler/java/java_features.pb.cc @@ -292,27 +292,26 @@ PROTOBUF_NOINLINE void JavaFeatures::Clear() { } ::size_t JavaFeatures::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:pb.JavaFeatures) + // @@protoc_insertion_point(message_byte_size_start:pb.JavaFeatures) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000001u) { total_size += 2; } - // optional .pb.JavaFeatures.Utf8Validation utf8_validation = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_utf8_validation()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/compiler/plugin.pb.cc b/src/google/protobuf/compiler/plugin.pb.cc index 61d8c33097..b10c89f7f9 100644 --- a/src/google/protobuf/compiler/plugin.pb.cc +++ b/src/google/protobuf/compiler/plugin.pb.cc @@ -501,14 +501,15 @@ PROTOBUF_NOINLINE void Version::Clear() { } ::size_t Version::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.Version) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.Version) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000000fu) { // optional string suffix = 4; @@ -516,25 +517,21 @@ PROTOBUF_NOINLINE void Version::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_suffix()); } - // optional int32 major = 1; if (cached_has_bits & 0x00000002u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_major()); } - // optional int32 minor = 2; if (cached_has_bits & 0x00000004u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_minor()); } - // optional int32 patch = 3; if (cached_has_bits & 0x00000008u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_patch()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -838,29 +835,38 @@ PROTOBUF_NOINLINE void CodeGeneratorRequest::Clear() { } ::size_t CodeGeneratorRequest::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.CodeGeneratorRequest) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.CodeGeneratorRequest) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated string file_to_generate = 1; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_file_to_generate().size()); - for (int i = 0, n = _internal_file_to_generate().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_file_to_generate().Get(i)); - } - // repeated .google.protobuf.FileDescriptorProto proto_file = 15; - total_size += 1UL * this->_internal_proto_file_size(); - for (const auto& msg : this->_internal_proto_file()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.FileDescriptorProto source_file_descriptors = 17; - total_size += 2UL * this->_internal_source_file_descriptors_size(); - for (const auto& msg : this->_internal_source_file_descriptors()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated string file_to_generate = 1; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_file_to_generate().size()); + for (int i = 0, n = _internal_file_to_generate().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_file_to_generate().Get(i)); + } + } + // repeated .google.protobuf.FileDescriptorProto proto_file = 15; + { + total_size += 1UL * this->_internal_proto_file_size(); + for (const auto& msg : this->_internal_proto_file()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.FileDescriptorProto source_file_descriptors = 17; + { + total_size += 2UL * this->_internal_source_file_descriptors_size(); + for (const auto& msg : this->_internal_source_file_descriptors()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -869,13 +875,11 @@ PROTOBUF_NOINLINE void CodeGeneratorRequest::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_parameter()); } - // optional .google.protobuf.compiler.Version compiler_version = 3; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.compiler_version_); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1170,14 +1174,15 @@ PROTOBUF_NOINLINE void CodeGeneratorResponse_File::Clear() { } ::size_t CodeGeneratorResponse_File::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.CodeGeneratorResponse.File) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.CodeGeneratorResponse.File) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000000fu) { // optional string name = 1; @@ -1185,25 +1190,21 @@ PROTOBUF_NOINLINE void CodeGeneratorResponse_File::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional string insertion_point = 2; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_insertion_point()); } - // optional string content = 15; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_content()); } - // optional .google.protobuf.GeneratedCodeInfo generated_code_info = 16; if (cached_has_bits & 0x00000008u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.generated_code_info_); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1498,18 +1499,23 @@ PROTOBUF_NOINLINE void CodeGeneratorResponse::Clear() { } ::size_t CodeGeneratorResponse::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.CodeGeneratorResponse) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.compiler.CodeGeneratorResponse) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.compiler.CodeGeneratorResponse.File file = 15; - total_size += 1UL * this->_internal_file_size(); - for (const auto& msg : this->_internal_file()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.compiler.CodeGeneratorResponse.File file = 15; + { + total_size += 1UL * this->_internal_file_size(); + for (const auto& msg : this->_internal_file()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000000fu) { @@ -1518,25 +1524,21 @@ PROTOBUF_NOINLINE void CodeGeneratorResponse::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_error()); } - // optional uint64 supported_features = 2; if (cached_has_bits & 0x00000002u) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne( this->_internal_supported_features()); } - // optional int32 minimum_edition = 3; if (cached_has_bits & 0x00000004u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_minimum_edition()); } - // optional int32 maximum_edition = 4; if (cached_has_bits & 0x00000008u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_maximum_edition()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/cpp_features.pb.cc b/src/google/protobuf/cpp_features.pb.cc index 0e82110847..71f69b9793 100644 --- a/src/google/protobuf/cpp_features.pb.cc +++ b/src/google/protobuf/cpp_features.pb.cc @@ -287,27 +287,26 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { } ::size_t CppFeatures::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:pb.CppFeatures) + // @@protoc_insertion_point(message_byte_size_start:pb.CppFeatures) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000001u) { total_size += 2; } - // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_string_type()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index ef5e9900ba..ba06a6be83 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -2535,18 +2535,23 @@ PROTOBUF_NOINLINE void FileDescriptorSet::Clear() { } ::size_t FileDescriptorSet::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FileDescriptorSet) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FileDescriptorSet) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.FileDescriptorProto file = 1; - total_size += 1UL * this->_internal_file_size(); - for (const auto& msg : this->_internal_file()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.FileDescriptorProto file = 1; + { + total_size += 1UL * this->_internal_file_size(); + for (const auto& msg : this->_internal_file()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -2985,59 +2990,72 @@ PROTOBUF_NOINLINE void FileDescriptorProto::Clear() { } ::size_t FileDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FileDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FileDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated string dependency = 3; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_dependency().size()); - for (int i = 0, n = _internal_dependency().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_dependency().Get(i)); - } - // repeated .google.protobuf.DescriptorProto message_type = 4; - total_size += 1UL * this->_internal_message_type_size(); - for (const auto& msg : this->_internal_message_type()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.EnumDescriptorProto enum_type = 5; - total_size += 1UL * this->_internal_enum_type_size(); - for (const auto& msg : this->_internal_enum_type()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.ServiceDescriptorProto service = 6; - total_size += 1UL * this->_internal_service_size(); - for (const auto& msg : this->_internal_service()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.FieldDescriptorProto extension = 7; - total_size += 1UL * this->_internal_extension_size(); - for (const auto& msg : this->_internal_extension()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated int32 public_dependency = 10; - { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this->_internal_public_dependency()) - ; - std::size_t tag_size = std::size_t{1} * - ::_pbi::FromIntSize(this->_internal_public_dependency_size()); - ; - total_size += tag_size + data_size; - } - // repeated int32 weak_dependency = 11; - { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this->_internal_weak_dependency()) - ; - std::size_t tag_size = std::size_t{1} * - ::_pbi::FromIntSize(this->_internal_weak_dependency_size()); - ; - total_size += tag_size + data_size; + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated string dependency = 3; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_dependency().size()); + for (int i = 0, n = _internal_dependency().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_dependency().Get(i)); + } + } + // repeated .google.protobuf.DescriptorProto message_type = 4; + { + total_size += 1UL * this->_internal_message_type_size(); + for (const auto& msg : this->_internal_message_type()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.EnumDescriptorProto enum_type = 5; + { + total_size += 1UL * this->_internal_enum_type_size(); + for (const auto& msg : this->_internal_enum_type()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.ServiceDescriptorProto service = 6; + { + total_size += 1UL * this->_internal_service_size(); + for (const auto& msg : this->_internal_service()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.FieldDescriptorProto extension = 7; + { + total_size += 1UL * this->_internal_extension_size(); + for (const auto& msg : this->_internal_extension()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated int32 public_dependency = 10; + { + std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( + this->_internal_public_dependency()) + ; + std::size_t tag_size = std::size_t{1} * + ::_pbi::FromIntSize(this->_internal_public_dependency_size()); + ; + total_size += tag_size + data_size; + } + // repeated int32 weak_dependency = 11; + { + std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( + this->_internal_weak_dependency()) + ; + std::size_t tag_size = std::size_t{1} * + ::_pbi::FromIntSize(this->_internal_weak_dependency_size()); + ; + total_size += tag_size + data_size; + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000003fu) { @@ -3046,37 +3064,31 @@ PROTOBUF_NOINLINE void FileDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional string package = 2; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_package()); } - // optional string syntax = 12; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_syntax()); } - // optional .google.protobuf.FileOptions options = 8; if (cached_has_bits & 0x00000008u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - // optional .google.protobuf.SourceCodeInfo source_code_info = 9; if (cached_has_bits & 0x00000010u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_code_info_); } - // optional .google.protobuf.Edition edition = 14; if (cached_has_bits & 0x00000020u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_edition()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -3384,14 +3396,15 @@ PROTOBUF_NOINLINE void DescriptorProto_ExtensionRange::Clear() { } ::size_t DescriptorProto_ExtensionRange::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.DescriptorProto.ExtensionRange) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.DescriptorProto.ExtensionRange) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000007u) { // optional .google.protobuf.ExtensionRangeOptions options = 3; @@ -3399,19 +3412,16 @@ PROTOBUF_NOINLINE void DescriptorProto_ExtensionRange::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - // optional int32 start = 1; if (cached_has_bits & 0x00000002u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_start()); } - // optional int32 end = 2; if (cached_has_bits & 0x00000004u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_end()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -3631,14 +3641,15 @@ PROTOBUF_NOINLINE void DescriptorProto_ReservedRange::Clear() { } ::size_t DescriptorProto_ReservedRange::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.DescriptorProto.ReservedRange) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.DescriptorProto.ReservedRange) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // optional int32 start = 1; @@ -3646,13 +3657,11 @@ PROTOBUF_NOINLINE void DescriptorProto_ReservedRange::Clear() { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_start()); } - // optional int32 end = 2; if (cached_has_bits & 0x00000002u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_end()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -4048,54 +4057,73 @@ PROTOBUF_NOINLINE void DescriptorProto::Clear() { } ::size_t DescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.DescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.DescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.FieldDescriptorProto field = 2; - total_size += 1UL * this->_internal_field_size(); - for (const auto& msg : this->_internal_field()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.DescriptorProto nested_type = 3; - total_size += 1UL * this->_internal_nested_type_size(); - for (const auto& msg : this->_internal_nested_type()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.EnumDescriptorProto enum_type = 4; - total_size += 1UL * this->_internal_enum_type_size(); - for (const auto& msg : this->_internal_enum_type()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.DescriptorProto.ExtensionRange extension_range = 5; - total_size += 1UL * this->_internal_extension_range_size(); - for (const auto& msg : this->_internal_extension_range()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.FieldDescriptorProto extension = 6; - total_size += 1UL * this->_internal_extension_size(); - for (const auto& msg : this->_internal_extension()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.OneofDescriptorProto oneof_decl = 8; - total_size += 1UL * this->_internal_oneof_decl_size(); - for (const auto& msg : this->_internal_oneof_decl()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.DescriptorProto.ReservedRange reserved_range = 9; - total_size += 1UL * this->_internal_reserved_range_size(); - for (const auto& msg : this->_internal_reserved_range()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated string reserved_name = 10; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_reserved_name().size()); - for (int i = 0, n = _internal_reserved_name().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_reserved_name().Get(i)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.FieldDescriptorProto field = 2; + { + total_size += 1UL * this->_internal_field_size(); + for (const auto& msg : this->_internal_field()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.DescriptorProto nested_type = 3; + { + total_size += 1UL * this->_internal_nested_type_size(); + for (const auto& msg : this->_internal_nested_type()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.EnumDescriptorProto enum_type = 4; + { + total_size += 1UL * this->_internal_enum_type_size(); + for (const auto& msg : this->_internal_enum_type()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.DescriptorProto.ExtensionRange extension_range = 5; + { + total_size += 1UL * this->_internal_extension_range_size(); + for (const auto& msg : this->_internal_extension_range()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.FieldDescriptorProto extension = 6; + { + total_size += 1UL * this->_internal_extension_size(); + for (const auto& msg : this->_internal_extension()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.OneofDescriptorProto oneof_decl = 8; + { + total_size += 1UL * this->_internal_oneof_decl_size(); + for (const auto& msg : this->_internal_oneof_decl()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.DescriptorProto.ReservedRange reserved_range = 9; + { + total_size += 1UL * this->_internal_reserved_range_size(); + for (const auto& msg : this->_internal_reserved_range()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated string reserved_name = 10; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_reserved_name().size()); + for (int i = 0, n = _internal_reserved_name().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_reserved_name().Get(i)); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -4104,13 +4132,11 @@ PROTOBUF_NOINLINE void DescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional .google.protobuf.MessageOptions options = 7; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -4442,14 +4468,15 @@ PROTOBUF_NOINLINE void ExtensionRangeOptions_Declaration::Clear() { } ::size_t ExtensionRangeOptions_Declaration::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.ExtensionRangeOptions.Declaration) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.ExtensionRangeOptions.Declaration) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000001fu) { // optional string full_name = 2; @@ -4457,29 +4484,24 @@ PROTOBUF_NOINLINE void ExtensionRangeOptions_Declaration::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_full_name()); } - // optional string type = 3; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_type()); } - // optional int32 number = 1; if (cached_has_bits & 0x00000004u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_number()); } - // optional bool reserved = 5; if (cached_has_bits & 0x00000008u) { total_size += 2; } - // optional bool repeated = 6; if (cached_has_bits & 0x00000010u) { total_size += 2; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -4765,25 +4787,31 @@ PROTOBUF_NOINLINE void ExtensionRangeOptions::Clear() { } ::size_t ExtensionRangeOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.ExtensionRangeOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.ExtensionRangeOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.ExtensionRangeOptions.Declaration declaration = 2 [retention = RETENTION_SOURCE]; - total_size += 1UL * this->_internal_declaration_size(); - for (const auto& msg : this->_internal_declaration()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.ExtensionRangeOptions.Declaration declaration = 2 [retention = RETENTION_SOURCE]; + { + total_size += 1UL * this->_internal_declaration_size(); + for (const auto& msg : this->_internal_declaration()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -4792,13 +4820,11 @@ PROTOBUF_NOINLINE void ExtensionRangeOptions::Clear() { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional .google.protobuf.ExtensionRangeOptions.VerificationState verification = 3 [default = UNVERIFIED, retention = RETENTION_SOURCE]; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_verification()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -5230,14 +5256,15 @@ PROTOBUF_NOINLINE void FieldDescriptorProto::Clear() { } ::size_t FieldDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x000000ffu) { // optional string name = 1; @@ -5245,68 +5272,57 @@ PROTOBUF_NOINLINE void FieldDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional string extendee = 2; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_extendee()); } - // optional string type_name = 6; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_type_name()); } - // optional string default_value = 7; if (cached_has_bits & 0x00000008u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_default_value()); } - // optional string json_name = 10; if (cached_has_bits & 0x00000010u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_json_name()); } - // optional .google.protobuf.FieldOptions options = 8; if (cached_has_bits & 0x00000020u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - // optional int32 number = 3; if (cached_has_bits & 0x00000040u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_number()); } - // optional int32 oneof_index = 9; if (cached_has_bits & 0x00000080u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_oneof_index()); } - } if (cached_has_bits & 0x00000700u) { // optional bool proto3_optional = 17; if (cached_has_bits & 0x00000100u) { total_size += 3; } - // optional .google.protobuf.FieldDescriptorProto.Label label = 4; if (cached_has_bits & 0x00000200u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_label()); } - // optional .google.protobuf.FieldDescriptorProto.Type type = 5; if (cached_has_bits & 0x00000400u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_type()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -5582,14 +5598,15 @@ PROTOBUF_NOINLINE void OneofDescriptorProto::Clear() { } ::size_t OneofDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.OneofDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.OneofDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // optional string name = 1; @@ -5597,13 +5614,11 @@ PROTOBUF_NOINLINE void OneofDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional .google.protobuf.OneofOptions options = 2; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -5818,14 +5833,15 @@ PROTOBUF_NOINLINE void EnumDescriptorProto_EnumReservedRange::Clear() { } ::size_t EnumDescriptorProto_EnumReservedRange::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumDescriptorProto.EnumReservedRange) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumDescriptorProto.EnumReservedRange) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // optional int32 start = 1; @@ -5833,13 +5849,11 @@ PROTOBUF_NOINLINE void EnumDescriptorProto_EnumReservedRange::Clear() { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_start()); } - // optional int32 end = 2; if (cached_has_bits & 0x00000002u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_end()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -6127,29 +6141,38 @@ PROTOBUF_NOINLINE void EnumDescriptorProto::Clear() { } ::size_t EnumDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.EnumValueDescriptorProto value = 2; - total_size += 1UL * this->_internal_value_size(); - for (const auto& msg : this->_internal_value()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.EnumDescriptorProto.EnumReservedRange reserved_range = 4; - total_size += 1UL * this->_internal_reserved_range_size(); - for (const auto& msg : this->_internal_reserved_range()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated string reserved_name = 5; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_reserved_name().size()); - for (int i = 0, n = _internal_reserved_name().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_reserved_name().Get(i)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.EnumValueDescriptorProto value = 2; + { + total_size += 1UL * this->_internal_value_size(); + for (const auto& msg : this->_internal_value()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.EnumDescriptorProto.EnumReservedRange reserved_range = 4; + { + total_size += 1UL * this->_internal_reserved_range_size(); + for (const auto& msg : this->_internal_reserved_range()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated string reserved_name = 5; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_reserved_name().size()); + for (int i = 0, n = _internal_reserved_name().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_reserved_name().Get(i)); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -6158,13 +6181,11 @@ PROTOBUF_NOINLINE void EnumDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional .google.protobuf.EnumOptions options = 3; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -6433,14 +6454,15 @@ PROTOBUF_NOINLINE void EnumValueDescriptorProto::Clear() { } ::size_t EnumValueDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumValueDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumValueDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000007u) { // optional string name = 1; @@ -6448,19 +6470,16 @@ PROTOBUF_NOINLINE void EnumValueDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional .google.protobuf.EnumValueOptions options = 3; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - // optional int32 number = 2; if (cached_has_bits & 0x00000004u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_number()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -6728,18 +6747,23 @@ PROTOBUF_NOINLINE void ServiceDescriptorProto::Clear() { } ::size_t ServiceDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.ServiceDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.ServiceDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.MethodDescriptorProto method = 2; - total_size += 1UL * this->_internal_method_size(); - for (const auto& msg : this->_internal_method()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.MethodDescriptorProto method = 2; + { + total_size += 1UL * this->_internal_method_size(); + for (const auto& msg : this->_internal_method()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -6748,13 +6772,11 @@ PROTOBUF_NOINLINE void ServiceDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional .google.protobuf.ServiceOptions options = 3; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -7082,14 +7104,15 @@ PROTOBUF_NOINLINE void MethodDescriptorProto::Clear() { } ::size_t MethodDescriptorProto::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.MethodDescriptorProto) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.MethodDescriptorProto) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000003fu) { // optional string name = 1; @@ -7097,35 +7120,29 @@ PROTOBUF_NOINLINE void MethodDescriptorProto::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name()); } - // optional string input_type = 2; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_input_type()); } - // optional string output_type = 3; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_output_type()); } - // optional .google.protobuf.MethodOptions options = 4; if (cached_has_bits & 0x00000008u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.options_); } - // optional bool client_streaming = 5 [default = false]; if (cached_has_bits & 0x00000010u) { total_size += 2; } - // optional bool server_streaming = 6 [default = false]; if (cached_has_bits & 0x00000020u) { total_size += 2; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -7751,20 +7768,24 @@ PROTOBUF_NOINLINE void FileOptions::Clear() { } ::size_t FileOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FileOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FileOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x000000ffu) { @@ -7773,49 +7794,41 @@ PROTOBUF_NOINLINE void FileOptions::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_java_package()); } - // optional string java_outer_classname = 8; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_java_outer_classname()); } - // optional string go_package = 11; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_go_package()); } - // optional string objc_class_prefix = 36; if (cached_has_bits & 0x00000008u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_objc_class_prefix()); } - // optional string csharp_namespace = 37; if (cached_has_bits & 0x00000010u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_csharp_namespace()); } - // optional string swift_prefix = 39; if (cached_has_bits & 0x00000020u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_swift_prefix()); } - // optional string php_class_prefix = 40; if (cached_has_bits & 0x00000040u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_php_class_prefix()); } - // optional string php_namespace = 41; if (cached_has_bits & 0x00000080u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_php_namespace()); } - } if (cached_has_bits & 0x0000ff00u) { // optional string php_metadata_namespace = 44; @@ -7823,67 +7836,55 @@ PROTOBUF_NOINLINE void FileOptions::Clear() { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_php_metadata_namespace()); } - // optional string ruby_package = 45; if (cached_has_bits & 0x00000200u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_ruby_package()); } - // optional .google.protobuf.FeatureSet features = 50; if (cached_has_bits & 0x00000400u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional bool java_multiple_files = 10 [default = false]; if (cached_has_bits & 0x00000800u) { total_size += 2; } - // optional bool java_generate_equals_and_hash = 20 [deprecated = true]; if (cached_has_bits & 0x00001000u) { total_size += 3; } - // optional bool java_string_check_utf8 = 27 [default = false]; if (cached_has_bits & 0x00002000u) { total_size += 3; } - // optional bool cc_generic_services = 16 [default = false]; if (cached_has_bits & 0x00004000u) { total_size += 3; } - // optional bool java_generic_services = 17 [default = false]; if (cached_has_bits & 0x00008000u) { total_size += 3; } - } if (cached_has_bits & 0x000f0000u) { // optional bool py_generic_services = 18 [default = false]; if (cached_has_bits & 0x00010000u) { total_size += 3; } - // optional bool deprecated = 23 [default = false]; if (cached_has_bits & 0x00020000u) { total_size += 3; } - // optional .google.protobuf.FileOptions.OptimizeMode optimize_for = 9 [default = SPEED]; if (cached_has_bits & 0x00040000u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_optimize_for()); } - // optional bool cc_enable_arenas = 31 [default = true]; if (cached_has_bits & 0x00080000u) { total_size += 3; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -8288,20 +8289,24 @@ PROTOBUF_NOINLINE void MessageOptions::Clear() { } ::size_t MessageOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.MessageOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.MessageOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000003fu) { @@ -8310,32 +8315,26 @@ PROTOBUF_NOINLINE void MessageOptions::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional bool message_set_wire_format = 1 [default = false]; if (cached_has_bits & 0x00000002u) { total_size += 2; } - // optional bool no_standard_descriptor_accessor = 2 [default = false]; if (cached_has_bits & 0x00000004u) { total_size += 2; } - // optional bool deprecated = 3 [default = false]; if (cached_has_bits & 0x00000008u) { total_size += 2; } - // optional bool map_entry = 7; if (cached_has_bits & 0x00000010u) { total_size += 2; } - // optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; if (cached_has_bits & 0x00000020u) { total_size += 2; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -8590,14 +8589,15 @@ PROTOBUF_NOINLINE void FieldOptions_EditionDefault::Clear() { } ::size_t FieldOptions_EditionDefault::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldOptions.EditionDefault) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldOptions.EditionDefault) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // optional string value = 2; @@ -8605,13 +8605,11 @@ PROTOBUF_NOINLINE void FieldOptions_EditionDefault::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_value()); } - // optional .google.protobuf.Edition edition = 3; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_edition()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -8869,14 +8867,15 @@ PROTOBUF_NOINLINE void FieldOptions_FeatureSupport::Clear() { } ::size_t FieldOptions_FeatureSupport::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldOptions.FeatureSupport) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldOptions.FeatureSupport) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000000fu) { // optional string deprecation_warning = 3; @@ -8884,25 +8883,21 @@ PROTOBUF_NOINLINE void FieldOptions_FeatureSupport::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_deprecation_warning()); } - // optional .google.protobuf.Edition edition_introduced = 1; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_edition_introduced()); } - // optional .google.protobuf.Edition edition_deprecated = 2; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_edition_deprecated()); } - // optional .google.protobuf.Edition edition_removed = 4; if (cached_has_bits & 0x00000008u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_edition_removed()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -9339,37 +9334,43 @@ PROTOBUF_NOINLINE void FieldOptions::Clear() { } ::size_t FieldOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.FieldOptions.OptionTargetType targets = 19; - { - std::size_t data_size = 0; - auto count = static_cast(this->_internal_targets_size()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.FieldOptions.OptionTargetType targets = 19; + { + std::size_t data_size = 0; + auto count = static_cast(this->_internal_targets_size()); - for (std::size_t i = 0; i < count; ++i) { - data_size += ::_pbi::WireFormatLite::EnumSize( - this->_internal_targets().Get(static_cast(i))); + for (std::size_t i = 0; i < count; ++i) { + data_size += ::_pbi::WireFormatLite::EnumSize( + this->_internal_targets().Get(static_cast(i))); + } + total_size += data_size; + total_size += std::size_t{2} * count; + } + // repeated .google.protobuf.FieldOptions.EditionDefault edition_defaults = 20; + { + total_size += 2UL * this->_internal_edition_defaults_size(); + for (const auto& msg : this->_internal_edition_defaults()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } } - total_size += data_size; - total_size += std::size_t{2} * count; - } - // repeated .google.protobuf.FieldOptions.EditionDefault edition_defaults = 20; - total_size += 2UL * this->_internal_edition_defaults_size(); - for (const auto& msg : this->_internal_edition_defaults()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x000000ffu) { @@ -9378,63 +9379,52 @@ PROTOBUF_NOINLINE void FieldOptions::Clear() { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional .google.protobuf.FieldOptions.FeatureSupport feature_support = 22; if (cached_has_bits & 0x00000002u) { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.feature_support_); } - // optional .google.protobuf.FieldOptions.CType ctype = 1 [default = STRING]; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_ctype()); } - // optional .google.protobuf.FieldOptions.JSType jstype = 6 [default = JS_NORMAL]; if (cached_has_bits & 0x00000008u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_jstype()); } - // optional bool packed = 2; if (cached_has_bits & 0x00000010u) { total_size += 2; } - // optional bool lazy = 5 [default = false]; if (cached_has_bits & 0x00000020u) { total_size += 2; } - // optional bool unverified_lazy = 15 [default = false]; if (cached_has_bits & 0x00000040u) { total_size += 2; } - // optional bool deprecated = 3 [default = false]; if (cached_has_bits & 0x00000080u) { total_size += 2; } - } if (cached_has_bits & 0x00000700u) { // optional bool weak = 10 [default = false]; if (cached_has_bits & 0x00000100u) { total_size += 2; } - // optional bool debug_redact = 16 [default = false]; if (cached_has_bits & 0x00000200u) { total_size += 3; } - // optional .google.protobuf.FieldOptions.OptionRetention retention = 17; if (cached_has_bits & 0x00000400u) { total_size += 2 + ::_pbi::WireFormatLite::EnumSize(this->_internal_retention()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -9732,28 +9722,33 @@ PROTOBUF_NOINLINE void OneofOptions::Clear() { } ::size_t OneofOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.OneofOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.OneofOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - // optional .google.protobuf.FeatureSet features = 1; - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += - 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); + { + // optional .google.protobuf.FeatureSet features = 1; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + total_size += + 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -10053,20 +10048,24 @@ PROTOBUF_NOINLINE void EnumOptions::Clear() { } ::size_t EnumOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000000fu) { @@ -10075,22 +10074,18 @@ PROTOBUF_NOINLINE void EnumOptions::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional bool allow_alias = 2; if (cached_has_bits & 0x00000002u) { total_size += 2; } - // optional bool deprecated = 3 [default = false]; if (cached_has_bits & 0x00000004u) { total_size += 2; } - // optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; if (cached_has_bits & 0x00000008u) { total_size += 2; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -10397,20 +10392,24 @@ PROTOBUF_NOINLINE void EnumValueOptions::Clear() { } ::size_t EnumValueOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumValueOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumValueOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000007u) { @@ -10419,17 +10418,14 @@ PROTOBUF_NOINLINE void EnumValueOptions::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional bool deprecated = 1 [default = false]; if (cached_has_bits & 0x00000002u) { total_size += 2; } - // optional bool debug_redact = 3 [default = false]; if (cached_has_bits & 0x00000004u) { total_size += 2; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -10710,20 +10706,24 @@ PROTOBUF_NOINLINE void ServiceOptions::Clear() { } ::size_t ServiceOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.ServiceOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.ServiceOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -10732,12 +10732,10 @@ PROTOBUF_NOINLINE void ServiceOptions::Clear() { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional bool deprecated = 33 [default = false]; if (cached_has_bits & 0x00000002u) { total_size += 3; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -11042,20 +11040,24 @@ PROTOBUF_NOINLINE void MethodOptions::Clear() { } ::size_t MethodOptions::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.MethodOptions) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.MethodOptions) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; - total_size += 2UL * this->_internal_uninterpreted_option_size(); - for (const auto& msg : this->_internal_uninterpreted_option()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption uninterpreted_option = 999; + { + total_size += 2UL * this->_internal_uninterpreted_option_size(); + for (const auto& msg : this->_internal_uninterpreted_option()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000007u) { @@ -11064,18 +11066,15 @@ PROTOBUF_NOINLINE void MethodOptions::Clear() { total_size += 2 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.features_); } - // optional bool deprecated = 33 [default = false]; if (cached_has_bits & 0x00000002u) { total_size += 3; } - // optional .google.protobuf.MethodOptions.IdempotencyLevel idempotency_level = 34 [default = IDEMPOTENCY_UNKNOWN]; if (cached_has_bits & 0x00000004u) { total_size += 2 + ::_pbi::WireFormatLite::EnumSize(this->_internal_idempotency_level()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -11324,14 +11323,15 @@ PROTOBUF_NOINLINE void UninterpretedOption_NamePart::Clear() { } ::size_t UninterpretedOption_NamePart::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.UninterpretedOption.NamePart) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.UninterpretedOption.NamePart) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { // required string name_part = 1; @@ -11339,12 +11339,10 @@ PROTOBUF_NOINLINE void UninterpretedOption_NamePart::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_name_part()); } - // required bool is_extension = 2; if (cached_has_bits & 0x00000002u) { total_size += 2; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -11670,18 +11668,23 @@ PROTOBUF_NOINLINE void UninterpretedOption::Clear() { } ::size_t UninterpretedOption::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.UninterpretedOption) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.UninterpretedOption) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.UninterpretedOption.NamePart name = 2; - total_size += 1UL * this->_internal_name_size(); - for (const auto& msg : this->_internal_name()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.UninterpretedOption.NamePart name = 2; + { + total_size += 1UL * this->_internal_name_size(); + for (const auto& msg : this->_internal_name()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000003fu) { @@ -11690,36 +11693,30 @@ PROTOBUF_NOINLINE void UninterpretedOption::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_identifier_value()); } - // optional bytes string_value = 7; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::BytesSize( this->_internal_string_value()); } - // optional string aggregate_value = 8; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_aggregate_value()); } - // optional uint64 positive_int_value = 4; if (cached_has_bits & 0x00000008u) { total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne( this->_internal_positive_int_value()); } - // optional int64 negative_int_value = 5; if (cached_has_bits & 0x00000010u) { total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( this->_internal_negative_int_value()); } - // optional double double_value = 6; if (cached_has_bits & 0x00000020u) { total_size += 9; } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -12034,16 +12031,16 @@ PROTOBUF_NOINLINE void FeatureSet::Clear() { } ::size_t FeatureSet::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FeatureSet) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FeatureSet) ::size_t total_size = 0; - total_size += _impl_._extensions_.ByteSize(); ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000003fu) { // optional .google.protobuf.FeatureSet.FieldPresence field_presence = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { @@ -12051,37 +12048,31 @@ PROTOBUF_NOINLINE void FeatureSet::Clear() { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_field_presence()); } - // optional .google.protobuf.FeatureSet.EnumType enum_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_enum_type()); } - // optional .google.protobuf.FeatureSet.RepeatedFieldEncoding repeated_field_encoding = 3 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000004u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_repeated_field_encoding()); } - // optional .google.protobuf.FeatureSet.Utf8Validation utf8_validation = 4 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000008u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_utf8_validation()); } - // optional .google.protobuf.FeatureSet.MessageEncoding message_encoding = 5 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000010u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_message_encoding()); } - // optional .google.protobuf.FeatureSet.JsonFormat json_format = 6 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_MESSAGE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { if (cached_has_bits & 0x00000020u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_json_format()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -12349,14 +12340,15 @@ PROTOBUF_NOINLINE void FeatureSetDefaults_FeatureSetEditionDefault::Clear() { } ::size_t FeatureSetDefaults_FeatureSetEditionDefault::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000007u) { // optional .google.protobuf.FeatureSet overridable_features = 4; @@ -12364,19 +12356,16 @@ PROTOBUF_NOINLINE void FeatureSetDefaults_FeatureSetEditionDefault::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.overridable_features_); } - // optional .google.protobuf.FeatureSet fixed_features = 5; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.fixed_features_); } - // optional .google.protobuf.Edition edition = 3; if (cached_has_bits & 0x00000004u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_edition()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -12644,18 +12633,23 @@ PROTOBUF_NOINLINE void FeatureSetDefaults::Clear() { } ::size_t FeatureSetDefaults::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FeatureSetDefaults) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FeatureSetDefaults) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault defaults = 1; - total_size += 1UL * this->_internal_defaults_size(); - for (const auto& msg : this->_internal_defaults()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault defaults = 1; + { + total_size += 1UL * this->_internal_defaults_size(); + for (const auto& msg : this->_internal_defaults()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -12664,13 +12658,11 @@ PROTOBUF_NOINLINE void FeatureSetDefaults::Clear() { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_minimum_edition()); } - // optional .google.protobuf.Edition maximum_edition = 5; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_maximum_edition()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -12965,45 +12957,50 @@ PROTOBUF_NOINLINE void SourceCodeInfo_Location::Clear() { } ::size_t SourceCodeInfo_Location::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.SourceCodeInfo.Location) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.SourceCodeInfo.Location) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated int32 path = 1 [packed = true]; - { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this->_internal_path()) - ; - _impl_._path_cached_byte_size_.Set(::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; - } - // repeated int32 span = 2 [packed = true]; - { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this->_internal_span()) - ; - _impl_._span_cached_byte_size_.Set(::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; - } - // repeated string leading_detached_comments = 6; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_leading_detached_comments().size()); - for (int i = 0, n = _internal_leading_detached_comments().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_leading_detached_comments().Get(i)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated int32 path = 1 [packed = true]; + { + std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( + this->_internal_path()) + ; + _impl_._path_cached_byte_size_.Set(::_pbi::ToCachedSize(data_size)); + std::size_t tag_size = data_size == 0 + ? 0 + : 1 + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)) + ; + total_size += tag_size + data_size; + } + // repeated int32 span = 2 [packed = true]; + { + std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( + this->_internal_span()) + ; + _impl_._span_cached_byte_size_.Set(::_pbi::ToCachedSize(data_size)); + std::size_t tag_size = data_size == 0 + ? 0 + : 1 + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)) + ; + total_size += tag_size + data_size; + } + // repeated string leading_detached_comments = 6; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_leading_detached_comments().size()); + for (int i = 0, n = _internal_leading_detached_comments().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_leading_detached_comments().Get(i)); + } + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x00000003u) { @@ -13012,13 +13009,11 @@ PROTOBUF_NOINLINE void SourceCodeInfo_Location::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_leading_comments()); } - // optional string trailing_comments = 4; if (cached_has_bits & 0x00000002u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_trailing_comments()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -13212,18 +13207,23 @@ PROTOBUF_NOINLINE void SourceCodeInfo::Clear() { } ::size_t SourceCodeInfo::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.SourceCodeInfo) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.SourceCodeInfo) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.SourceCodeInfo.Location location = 1; - total_size += 1UL * this->_internal_location_size(); - for (const auto& msg : this->_internal_location()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.SourceCodeInfo.Location location = 1; + { + total_size += 1UL * this->_internal_location_size(); + for (const auto& msg : this->_internal_location()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -13490,26 +13490,29 @@ PROTOBUF_NOINLINE void GeneratedCodeInfo_Annotation::Clear() { } ::size_t GeneratedCodeInfo_Annotation::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.GeneratedCodeInfo.Annotation) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.GeneratedCodeInfo.Annotation) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated int32 path = 1 [packed = true]; - { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this->_internal_path()) - ; - _impl_._path_cached_byte_size_.Set(::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated int32 path = 1 [packed = true]; + { + std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( + this->_internal_path()) + ; + _impl_._path_cached_byte_size_.Set(::_pbi::ToCachedSize(data_size)); + std::size_t tag_size = data_size == 0 + ? 0 + : 1 + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)) + ; + total_size += tag_size + data_size; + } } cached_has_bits = _impl_._has_bits_[0]; if (cached_has_bits & 0x0000000fu) { @@ -13518,25 +13521,21 @@ PROTOBUF_NOINLINE void GeneratedCodeInfo_Annotation::Clear() { total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( this->_internal_source_file()); } - // optional int32 begin = 3; if (cached_has_bits & 0x00000002u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_begin()); } - // optional int32 end = 4; if (cached_has_bits & 0x00000004u) { total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( this->_internal_end()); } - // optional .google.protobuf.GeneratedCodeInfo.Annotation.Semantic semantic = 5; if (cached_has_bits & 0x00000008u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_semantic()); } - } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -13737,18 +13736,23 @@ PROTOBUF_NOINLINE void GeneratedCodeInfo::Clear() { } ::size_t GeneratedCodeInfo::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.GeneratedCodeInfo) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.GeneratedCodeInfo) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.GeneratedCodeInfo.Annotation annotation = 1; - total_size += 1UL * this->_internal_annotation_size(); - for (const auto& msg : this->_internal_annotation()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.GeneratedCodeInfo.Annotation annotation = 1; + { + total_size += 1UL * this->_internal_annotation_size(); + for (const auto& msg : this->_internal_annotation()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } From b618f6750aed641a23d5f26fbbaf654668846d24 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 00:43:40 +0000 Subject: [PATCH 016/179] Auto-generate files after cl/627204923 --- src/google/protobuf/any.pb.cc | 29 +- src/google/protobuf/api.pb.cc | 199 +++++++------ src/google/protobuf/duration.pb.cc | 29 +- src/google/protobuf/field_mask.pb.cc | 21 +- src/google/protobuf/source_context.pb.cc | 15 +- src/google/protobuf/struct.pb.cc | 46 +-- src/google/protobuf/timestamp.pb.cc | 29 +- src/google/protobuf/type.pb.cc | 356 ++++++++++++----------- src/google/protobuf/wrappers.pb.cc | 129 ++++---- 9 files changed, 460 insertions(+), 393 deletions(-) diff --git a/src/google/protobuf/any.pb.cc b/src/google/protobuf/any.pb.cc index c9b4ba0b23..1d70e242ad 100644 --- a/src/google/protobuf/any.pb.cc +++ b/src/google/protobuf/any.pb.cc @@ -280,26 +280,27 @@ PROTOBUF_NOINLINE void Any::Clear() { } ::size_t Any::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Any) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Any) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // string type_url = 1; - if (!this->_internal_type_url().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_type_url()); - } + (void)cached_has_bits; - // bytes value = 2; - if (!this->_internal_value().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::BytesSize( - this->_internal_value()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // string type_url = 1; + if (!this->_internal_type_url().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_type_url()); + } + // bytes value = 2; + if (!this->_internal_value().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index d24dd96095..1d572b11d1 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -508,54 +508,65 @@ PROTOBUF_NOINLINE void Api::Clear() { } ::size_t Api::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Api) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Api) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.Method methods = 2; - total_size += 1UL * this->_internal_methods_size(); - for (const auto& msg : this->_internal_methods()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.Option options = 3; - total_size += 1UL * this->_internal_options_size(); - for (const auto& msg : this->_internal_options()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.Mixin mixins = 6; - total_size += 1UL * this->_internal_mixins_size(); - for (const auto& msg : this->_internal_mixins()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.Method methods = 2; + { + total_size += 1UL * this->_internal_methods_size(); + for (const auto& msg : this->_internal_methods()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.Option options = 3; + { + total_size += 1UL * this->_internal_options_size(); + for (const auto& msg : this->_internal_options()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.Mixin mixins = 6; + { + total_size += 1UL * this->_internal_mixins_size(); + for (const auto& msg : this->_internal_mixins()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - - // string version = 4; - if (!this->_internal_version().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_version()); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // string version = 4; + if (!this->_internal_version().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_version()); + } } - - // .google.protobuf.SourceContext source_context = 5; - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += - 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_context_); + { + // .google.protobuf.SourceContext source_context = 5; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + total_size += + 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_context_); + } } - - // .google.protobuf.Syntax syntax = 7; - if (this->_internal_syntax() != 0) { - total_size += 1 + - ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + { + // .google.protobuf.Syntax syntax = 7; + if (this->_internal_syntax() != 0) { + total_size += 1 + + ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -888,53 +899,54 @@ PROTOBUF_NOINLINE void Method::Clear() { } ::size_t Method::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Method) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Method) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.Option options = 6; - total_size += 1UL * this->_internal_options_size(); - for (const auto& msg : this->_internal_options()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); - } - - // string request_type_url = 2; - if (!this->_internal_request_type_url().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_request_type_url()); - } - - // string response_type_url = 4; - if (!this->_internal_response_type_url().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_response_type_url()); - } - - // bool request_streaming = 3; - if (this->_internal_request_streaming() != 0) { - total_size += 2; - } + (void)cached_has_bits; - // bool response_streaming = 5; - if (this->_internal_response_streaming() != 0) { - total_size += 2; + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.Option options = 6; + { + total_size += 1UL * this->_internal_options_size(); + for (const auto& msg : this->_internal_options()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - - // .google.protobuf.Syntax syntax = 7; - if (this->_internal_syntax() != 0) { - total_size += 1 + - ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // string request_type_url = 2; + if (!this->_internal_request_type_url().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_request_type_url()); + } + // string response_type_url = 4; + if (!this->_internal_response_type_url().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_response_type_url()); + } + // bool request_streaming = 3; + if (this->_internal_request_streaming() != 0) { + total_size += 2; + } + // bool response_streaming = 5; + if (this->_internal_response_streaming() != 0) { + total_size += 2; + } + // .google.protobuf.Syntax syntax = 7; + if (this->_internal_syntax() != 0) { + total_size += 1 + + ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1159,26 +1171,27 @@ PROTOBUF_NOINLINE void Mixin::Clear() { } ::size_t Mixin::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Mixin) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Mixin) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); - } + (void)cached_has_bits; - // string root = 2; - if (!this->_internal_root().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_root()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // string root = 2; + if (!this->_internal_root().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_root()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/duration.pb.cc b/src/google/protobuf/duration.pb.cc index d15b7b08f0..76073738be 100644 --- a/src/google/protobuf/duration.pb.cc +++ b/src/google/protobuf/duration.pb.cc @@ -245,26 +245,27 @@ PROTOBUF_NOINLINE void Duration::Clear() { } ::size_t Duration::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Duration) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Duration) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // int64 seconds = 1; - if (this->_internal_seconds() != 0) { - total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( - this->_internal_seconds()); - } + (void)cached_has_bits; - // int32 nanos = 2; - if (this->_internal_nanos() != 0) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( - this->_internal_nanos()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // int64 seconds = 1; + if (this->_internal_seconds() != 0) { + total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( + this->_internal_seconds()); + } + // int32 nanos = 2; + if (this->_internal_nanos() != 0) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + this->_internal_nanos()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/field_mask.pb.cc b/src/google/protobuf/field_mask.pb.cc index 85e3639111..52e4183374 100644 --- a/src/google/protobuf/field_mask.pb.cc +++ b/src/google/protobuf/field_mask.pb.cc @@ -240,19 +240,24 @@ PROTOBUF_NOINLINE void FieldMask::Clear() { } ::size_t FieldMask::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldMask) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FieldMask) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated string paths = 1; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_paths().size()); - for (int i = 0, n = _internal_paths().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_paths().Get(i)); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated string paths = 1; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_paths().size()); + for (int i = 0, n = _internal_paths().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_paths().Get(i)); + } + } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/source_context.pb.cc b/src/google/protobuf/source_context.pb.cc index 3820c034d3..3d129e5b1a 100644 --- a/src/google/protobuf/source_context.pb.cc +++ b/src/google/protobuf/source_context.pb.cc @@ -243,19 +243,20 @@ PROTOBUF_NOINLINE void SourceContext::Clear() { } ::size_t SourceContext::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.SourceContext) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.SourceContext) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // string file_name = 1; - if (!this->_internal_file_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_file_name()); + { + // string file_name = 1; + if (!this->_internal_file_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_file_name()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index cefa306e1c..f03af5772c 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -396,20 +396,25 @@ PROTOBUF_NOINLINE void Struct::Clear() { } ::size_t Struct::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Struct) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Struct) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // map fields = 1; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_fields_size()); - for (const auto& entry : _internal_fields()) { - total_size += _pbi::MapEntryFuncs::ByteSizeLong(entry.first, entry.second); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // map fields = 1; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_fields_size()); + for (const auto& entry : _internal_fields()) { + total_size += _pbi::MapEntryFuncs::ByteSizeLong(entry.first, entry.second); + } + } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -729,12 +734,12 @@ PROTOBUF_NOINLINE void Value::Clear() { } ::size_t Value::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Value) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Value) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; switch (kind_case()) { // .google.protobuf.NullValue null_value = 1; @@ -1002,18 +1007,23 @@ PROTOBUF_NOINLINE void ListValue::Clear() { } ::size_t ListValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.ListValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.ListValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.Value values = 1; - total_size += 1UL * this->_internal_values_size(); - for (const auto& msg : this->_internal_values()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.Value values = 1; + { + total_size += 1UL * this->_internal_values_size(); + for (const auto& msg : this->_internal_values()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/timestamp.pb.cc b/src/google/protobuf/timestamp.pb.cc index 188c3e9c73..ab1e8af510 100644 --- a/src/google/protobuf/timestamp.pb.cc +++ b/src/google/protobuf/timestamp.pb.cc @@ -245,26 +245,27 @@ PROTOBUF_NOINLINE void Timestamp::Clear() { } ::size_t Timestamp::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Timestamp) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Timestamp) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // int64 seconds = 1; - if (this->_internal_seconds() != 0) { - total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( - this->_internal_seconds()); - } + (void)cached_has_bits; - // int32 nanos = 2; - if (this->_internal_nanos() != 0) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( - this->_internal_nanos()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // int64 seconds = 1; + if (this->_internal_seconds() != 0) { + total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( + this->_internal_seconds()); + } + // int32 nanos = 2; + if (this->_internal_nanos() != 0) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + this->_internal_nanos()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index eaf4384bc2..a6a22cf368 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -685,55 +685,66 @@ PROTOBUF_NOINLINE void Type::Clear() { } ::size_t Type::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Type) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Type) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.Field fields = 2; - total_size += 1UL * this->_internal_fields_size(); - for (const auto& msg : this->_internal_fields()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated string oneofs = 3; - total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_oneofs().size()); - for (int i = 0, n = _internal_oneofs().size(); i < n; ++i) { - total_size += ::google::protobuf::internal::WireFormatLite::StringSize( - _internal_oneofs().Get(i)); - } - // repeated .google.protobuf.Option options = 4; - total_size += 1UL * this->_internal_options_size(); - for (const auto& msg : this->_internal_options()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.Field fields = 2; + { + total_size += 1UL * this->_internal_fields_size(); + for (const auto& msg : this->_internal_fields()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated string oneofs = 3; + { + total_size += 1 * ::google::protobuf::internal::FromIntSize(_internal_oneofs().size()); + for (int i = 0, n = _internal_oneofs().size(); i < n; ++i) { + total_size += ::google::protobuf::internal::WireFormatLite::StringSize( + _internal_oneofs().Get(i)); + } + } + // repeated .google.protobuf.Option options = 4; + { + total_size += 1UL * this->_internal_options_size(); + for (const auto& msg : this->_internal_options()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - - // string edition = 7; - if (!this->_internal_edition().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_edition()); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // string edition = 7; + if (!this->_internal_edition().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_edition()); + } } - - // .google.protobuf.SourceContext source_context = 5; - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += - 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_context_); + { + // .google.protobuf.SourceContext source_context = 5; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + total_size += + 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_context_); + } } - - // .google.protobuf.Syntax syntax = 6; - if (this->_internal_syntax() != 0) { - total_size += 1 + - ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + { + // .google.protobuf.Syntax syntax = 6; + if (this->_internal_syntax() != 0) { + total_size += 1 + + ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1111,72 +1122,70 @@ PROTOBUF_NOINLINE void Field::Clear() { } ::size_t Field::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Field) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Field) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.Option options = 9; - total_size += 1UL * this->_internal_options_size(); - for (const auto& msg : this->_internal_options()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // string name = 4; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); - } - - // string type_url = 6; - if (!this->_internal_type_url().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_type_url()); - } - - // string json_name = 10; - if (!this->_internal_json_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_json_name()); - } - - // string default_value = 11; - if (!this->_internal_default_value().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_default_value()); - } - - // .google.protobuf.Field.Kind kind = 1; - if (this->_internal_kind() != 0) { - total_size += 1 + - ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); - } - - // .google.protobuf.Field.Cardinality cardinality = 2; - if (this->_internal_cardinality() != 0) { - total_size += 1 + - ::_pbi::WireFormatLite::EnumSize(this->_internal_cardinality()); - } - - // int32 number = 3; - if (this->_internal_number() != 0) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( - this->_internal_number()); - } + (void)cached_has_bits; - // int32 oneof_index = 7; - if (this->_internal_oneof_index() != 0) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( - this->_internal_oneof_index()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.Option options = 9; + { + total_size += 1UL * this->_internal_options_size(); + for (const auto& msg : this->_internal_options()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - - // bool packed = 8; - if (this->_internal_packed() != 0) { - total_size += 2; + { + // string name = 4; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // string type_url = 6; + if (!this->_internal_type_url().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_type_url()); + } + // string json_name = 10; + if (!this->_internal_json_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_json_name()); + } + // string default_value = 11; + if (!this->_internal_default_value().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_default_value()); + } + // .google.protobuf.Field.Kind kind = 1; + if (this->_internal_kind() != 0) { + total_size += 1 + + ::_pbi::WireFormatLite::EnumSize(this->_internal_kind()); + } + // .google.protobuf.Field.Cardinality cardinality = 2; + if (this->_internal_cardinality() != 0) { + total_size += 1 + + ::_pbi::WireFormatLite::EnumSize(this->_internal_cardinality()); + } + // int32 number = 3; + if (this->_internal_number() != 0) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + this->_internal_number()); + } + // int32 oneof_index = 7; + if (this->_internal_oneof_index() != 0) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + this->_internal_oneof_index()); + } + // bool packed = 8; + if (this->_internal_packed() != 0) { + total_size += 2; + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1510,49 +1519,58 @@ PROTOBUF_NOINLINE void Enum::Clear() { } ::size_t Enum::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Enum) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Enum) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.EnumValue enumvalue = 2; - total_size += 1UL * this->_internal_enumvalue_size(); - for (const auto& msg : this->_internal_enumvalue()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // repeated .google.protobuf.Option options = 3; - total_size += 1UL * this->_internal_options_size(); - for (const auto& msg : this->_internal_options()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.EnumValue enumvalue = 2; + { + total_size += 1UL * this->_internal_enumvalue_size(); + for (const auto& msg : this->_internal_enumvalue()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } + // repeated .google.protobuf.Option options = 3; + { + total_size += 1UL * this->_internal_options_size(); + for (const auto& msg : this->_internal_options()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - - // string edition = 6; - if (!this->_internal_edition().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_edition()); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // string edition = 6; + if (!this->_internal_edition().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_edition()); + } } - - // .google.protobuf.SourceContext source_context = 4; - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += - 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_context_); + { + // .google.protobuf.SourceContext source_context = 4; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + total_size += + 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.source_context_); + } } - - // .google.protobuf.Syntax syntax = 5; - if (this->_internal_syntax() != 0) { - total_size += 1 + - ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + { + // .google.protobuf.Syntax syntax = 5; + if (this->_internal_syntax() != 0) { + total_size += 1 + + ::_pbi::WireFormatLite::EnumSize(this->_internal_syntax()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1801,31 +1819,36 @@ PROTOBUF_NOINLINE void EnumValue::Clear() { } ::size_t EnumValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.EnumValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // repeated .google.protobuf.Option options = 3; - total_size += 1UL * this->_internal_options_size(); - for (const auto& msg : this->_internal_options()) { - total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); - } - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // repeated .google.protobuf.Option options = 3; + { + total_size += 1UL * this->_internal_options_size(); + for (const auto& msg : this->_internal_options()) { + total_size += ::google::protobuf::internal::WireFormatLite::MessageSize(msg); + } + } } - - // int32 number = 2; - if (this->_internal_number() != 0) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( - this->_internal_number()); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } + // int32 number = 2; + if (this->_internal_number() != 0) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + this->_internal_number()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -2047,27 +2070,30 @@ PROTOBUF_NOINLINE void Option::Clear() { } ::size_t Option::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Option) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Option) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - ::_pbi::Prefetch5LinesFrom7Lines(reinterpret_cast(this)); - // string name = 1; - if (!this->_internal_name().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_name()); + ::_pbi::Prefetch5LinesFrom7Lines( + reinterpret_cast(this)); + { + // string name = 1; + if (!this->_internal_name().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_name()); + } } - - // .google.protobuf.Any value = 2; - cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000001u) { - total_size += - 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.value_); + { + // .google.protobuf.Any value = 2; + cached_has_bits = _impl_._has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + total_size += + 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*_impl_.value_); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } diff --git a/src/google/protobuf/wrappers.pb.cc b/src/google/protobuf/wrappers.pb.cc index a47a849591..bae35ba17f 100644 --- a/src/google/protobuf/wrappers.pb.cc +++ b/src/google/protobuf/wrappers.pb.cc @@ -473,18 +473,19 @@ PROTOBUF_NOINLINE void DoubleValue::Clear() { } ::size_t DoubleValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.DoubleValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.DoubleValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // double value = 1; - if (::absl::bit_cast<::uint64_t>(this->_internal_value()) != 0) { - total_size += 9; + { + // double value = 1; + if (::absl::bit_cast<::uint64_t>(this->_internal_value()) != 0) { + total_size += 9; + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -644,18 +645,19 @@ PROTOBUF_NOINLINE void FloatValue::Clear() { } ::size_t FloatValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.FloatValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.FloatValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // float value = 1; - if (::absl::bit_cast<::uint32_t>(this->_internal_value()) != 0) { - total_size += 5; + { + // float value = 1; + if (::absl::bit_cast<::uint32_t>(this->_internal_value()) != 0) { + total_size += 5; + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -815,19 +817,20 @@ PROTOBUF_NOINLINE void Int64Value::Clear() { } ::size_t Int64Value::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Int64Value) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Int64Value) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // int64 value = 1; - if (this->_internal_value() != 0) { - total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( - this->_internal_value()); + { + // int64 value = 1; + if (this->_internal_value() != 0) { + total_size += ::_pbi::WireFormatLite::Int64SizePlusOne( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -987,19 +990,20 @@ PROTOBUF_NOINLINE void UInt64Value::Clear() { } ::size_t UInt64Value::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.UInt64Value) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.UInt64Value) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // uint64 value = 1; - if (this->_internal_value() != 0) { - total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne( - this->_internal_value()); + { + // uint64 value = 1; + if (this->_internal_value() != 0) { + total_size += ::_pbi::WireFormatLite::UInt64SizePlusOne( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1159,19 +1163,20 @@ PROTOBUF_NOINLINE void Int32Value::Clear() { } ::size_t Int32Value::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Int32Value) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.Int32Value) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // int32 value = 1; - if (this->_internal_value() != 0) { - total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( - this->_internal_value()); + { + // int32 value = 1; + if (this->_internal_value() != 0) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1331,19 +1336,20 @@ PROTOBUF_NOINLINE void UInt32Value::Clear() { } ::size_t UInt32Value::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.UInt32Value) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.UInt32Value) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // uint32 value = 1; - if (this->_internal_value() != 0) { - total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne( - this->_internal_value()); + { + // uint32 value = 1; + if (this->_internal_value() != 0) { + total_size += ::_pbi::WireFormatLite::UInt32SizePlusOne( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1503,18 +1509,19 @@ PROTOBUF_NOINLINE void BoolValue::Clear() { } ::size_t BoolValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.BoolValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.BoolValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // bool value = 1; - if (this->_internal_value() != 0) { - total_size += 2; + { + // bool value = 1; + if (this->_internal_value() != 0) { + total_size += 2; + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1692,19 +1699,20 @@ PROTOBUF_NOINLINE void StringValue::Clear() { } ::size_t StringValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.StringValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.StringValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // string value = 1; - if (!this->_internal_value().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( - this->_internal_value()); + { + // string value = 1; + if (!this->_internal_value().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::StringSize( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } @@ -1879,19 +1887,20 @@ PROTOBUF_NOINLINE void BytesValue::Clear() { } ::size_t BytesValue::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.BytesValue) + // @@protoc_insertion_point(message_byte_size_start:google.protobuf.BytesValue) ::size_t total_size = 0; ::uint32_t cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; + (void)cached_has_bits; - // bytes value = 1; - if (!this->_internal_value().empty()) { - total_size += 1 + ::google::protobuf::internal::WireFormatLite::BytesSize( - this->_internal_value()); + { + // bytes value = 1; + if (!this->_internal_value().empty()) { + total_size += 1 + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->_internal_value()); + } } - return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); } From b50ff60967b595765244d22945aed184cafee68a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 07:41:15 -0700 Subject: [PATCH 017/179] C#: Fix discovery of oneof declarations in descriptor protos. PiperOrigin-RevId: 627384633 --- .../Reflection/DescriptorDeclarationTest.cs | 10 ++++++++++ .../Google.Protobuf/Reflection/MessageDescriptor.cs | 1 + 2 files changed, 11 insertions(+) diff --git a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorDeclarationTest.cs b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorDeclarationTest.cs index 65a573ae70..c36acf726a 100644 --- a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorDeclarationTest.cs +++ b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorDeclarationTest.cs @@ -137,6 +137,16 @@ namespace Google.Protobuf.Test.Reflection Assert.AreEqual(" Zero value comment\n", value.Declaration.LeadingComments); } + [Test] + public void OneofComments() + { + // CommentMessage doesn't have an enum, but we can use TestAllTypes. + var message = unitTestProto3Descriptor.FindTypeByName("TestAllTypes"); + var oneof = message.Oneofs.Single(o => o.Name == "oneof_field"); + Assert.NotNull(oneof.Declaration); + Assert.AreEqual(" For oneof test\n", oneof.Declaration.LeadingComments); + } + private static FileDescriptor LoadProtos() { var type = typeof(DescriptorDeclarationTest); diff --git a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs index 911bef56ca..ead4676a5c 100644 --- a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs @@ -119,6 +119,7 @@ namespace Google.Protobuf.Reflection DescriptorProto.FieldFieldNumber => (IReadOnlyList)fieldsInDeclarationOrder, DescriptorProto.NestedTypeFieldNumber => (IReadOnlyList)NestedTypes, DescriptorProto.EnumTypeFieldNumber => (IReadOnlyList)EnumTypes, + DescriptorProto.OneofDeclFieldNumber => (IReadOnlyList)Oneofs, _ => null, }; From 335edb909f2f0922843b112fec229d3684e0c435 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Apr 2024 08:17:38 -0700 Subject: [PATCH 018/179] update decode_fast.c to assert that non-const messages are not frozen PiperOrigin-RevId: 627393634 --- upb/wire/internal/decode_fast.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/upb/wire/internal/decode_fast.c b/upb/wire/internal/decode_fast.c index 67be4135c4..726f65ad24 100644 --- a/upb/wire/internal/decode_fast.c +++ b/upb/wire/internal/decode_fast.c @@ -231,6 +231,7 @@ UPB_FORCEINLINE void* fastdecode_getfield(upb_Decoder* d, const char* ptr, upb_Message* msg, uint64_t* data, uint64_t* hasbits, fastdecode_arr* farr, int valbytes, upb_card card) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (card) { case CARD_s: { uint8_t hasbit_index = *data >> 24; @@ -606,6 +607,7 @@ UPB_NOINLINE static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView* dst = (upb_StringView*)data; if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); @@ -651,6 +653,7 @@ UPB_NOINLINE static const char* fastdecode_longstring_noutf8( struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView* dst = (upb_StringView*)data; FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); } From 8f2da7817cb115de05da7fa108375fde2970de49 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 15:32:31 +0000 Subject: [PATCH 019/179] Auto-generate files after cl/627393634 --- php/ext/google/protobuf/php-upb.c | 3 +++ ruby/ext/google/protobuf_c/ruby-upb.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index d32863b2cc..880a18876e 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -9851,6 +9851,7 @@ UPB_FORCEINLINE void* fastdecode_getfield(upb_Decoder* d, const char* ptr, upb_Message* msg, uint64_t* data, uint64_t* hasbits, fastdecode_arr* farr, int valbytes, upb_card card) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (card) { case CARD_s: { uint8_t hasbit_index = *data >> 24; @@ -10226,6 +10227,7 @@ UPB_NOINLINE static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView* dst = (upb_StringView*)data; if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); @@ -10271,6 +10273,7 @@ UPB_NOINLINE static const char* fastdecode_longstring_noutf8( struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView* dst = (upb_StringView*)data; FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 9dfbd6510e..2ad33d5618 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -9344,6 +9344,7 @@ UPB_FORCEINLINE void* fastdecode_getfield(upb_Decoder* d, const char* ptr, upb_Message* msg, uint64_t* data, uint64_t* hasbits, fastdecode_arr* farr, int valbytes, upb_card card) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (card) { case CARD_s: { uint8_t hasbit_index = *data >> 24; @@ -9719,6 +9720,7 @@ UPB_NOINLINE static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView* dst = (upb_StringView*)data; if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); @@ -9764,6 +9766,7 @@ UPB_NOINLINE static const char* fastdecode_longstring_noutf8( struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView* dst = (upb_StringView*)data; FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); } From 3c03e9351c57081d0dffae120ed37497017f105c Mon Sep 17 00:00:00 2001 From: Joshua Humphries <2035234+jhump@users.noreply.github.com> Date: Tue, 23 Apr 2024 08:38:54 -0700 Subject: [PATCH 020/179] protoc: support inf, -inf, nan, and -nan in option values (#15017) The parser already supports these values for the special "default" field option. But, inconsistently, does not support them for other options whose type is float or double. This addresses that inconsistency. Fixes #15010. Closes #15017 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/15017 from jhump:jh/inf-and-nan-in-option-values 1f8217c4bd0706b145a716c3db4a026b04f5a7ba PiperOrigin-RevId: 627399259 --- src/google/protobuf/compiler/parser.cc | 17 ++- .../protobuf/compiler/parser_unittest.cc | 105 ++++++++++++++++++ src/google/protobuf/descriptor.cc | 8 ++ .../protobuf/unittest_custom_options.proto | 20 ++++ 4 files changed, 146 insertions(+), 4 deletions(-) diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index f6f7d1d509..129ea2f3f2 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -1623,12 +1623,21 @@ bool Parser::ParseOption(Message* options, case io::Tokenizer::TYPE_IDENTIFIER: { value_location.AddPath( UninterpretedOption::kIdentifierValueFieldNumber); - if (is_negative) { - RecordError("Invalid '-' symbol before identifier."); - return false; - } std::string value; DO(ConsumeIdentifier(&value, "Expected identifier.")); + if (is_negative) { + if (value == "inf") { + uninterpreted_option->set_double_value( + -std::numeric_limits::infinity()); + } else if (value == "nan") { + uninterpreted_option->set_double_value( + std::numeric_limits::quiet_NaN()); + } else { + RecordError("Identifier after '-' symbol must be inf or nan."); + return false; + } + break; + } uninterpreted_option->set_identifier_value(value); break; } diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 46c1e2e5ff..f9b8f32317 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -12,6 +12,7 @@ #include "google/protobuf/compiler/parser.h" #include +#include #include #include #include @@ -460,6 +461,7 @@ TEST_F(ParseMessageTest, FieldDefaults) { " required double foo = 1 [default= inf ];\n" " required double foo = 1 [default=-inf ];\n" " required double foo = 1 [default= nan ];\n" + " required double foo = 1 [default= -nan ];\n" " required string foo = 1 [default='13\\001'];\n" " required string foo = 1 [default='a' \"b\" \n \"c\"];\n" " required bytes foo = 1 [default='14\\002'];\n" @@ -509,6 +511,8 @@ TEST_F(ParseMessageTest, FieldDefaults) { " }" " field { type:TYPE_DOUBLE default_value:\"nan\" " ETC " }" + " field { type:TYPE_DOUBLE default_value:\"-nan\" " ETC + " }" " field { type:TYPE_STRING default_value:\"13\\001\" " ETC " }" " field { type:TYPE_STRING default_value:\"abc\" " ETC @@ -658,6 +662,65 @@ TEST_F(ParseMessageTest, FieldOptionsSupportLargeDecimalLiteral) { "}"); } +TEST_F(ParseMessageTest, FieldOptionsSupportInfAndNan) { + ExpectParsesTo( + "import \"google/protobuf/descriptor.proto\";\n" + "extend google.protobuf.FieldOptions {\n" + " optional double f = 10101;\n" + "}\n" + "message TestMessage {\n" + " optional double a = 1 [(f) = inf];\n" + " optional double b = 2 [(f) = -inf];\n" + " optional double c = 3 [(f) = nan];\n" + " optional double d = 4 [(f) = -nan];\n" + "}\n", + + "dependency: \"google/protobuf/descriptor.proto\"" + "extension {" + " name: \"f\" label: LABEL_OPTIONAL type: TYPE_DOUBLE number: 10101" + " extendee: \"google.protobuf.FieldOptions\"" + "}" + "message_type {" + " name: \"TestMessage\"" + " field {" + " name: \"a\" label: LABEL_OPTIONAL type: TYPE_DOUBLE number: 1" + " options{" + " uninterpreted_option{" + " name{ name_part: \"f\" is_extension: true }" + " identifier_value: \"inf\"" + " }" + " }" + " }" + " field {" + " name: \"b\" label: LABEL_OPTIONAL type: TYPE_DOUBLE number: 2" + " options{" + " uninterpreted_option{" + " name{ name_part: \"f\" is_extension: true }" + " double_value: -infinity" + " }" + " }" + " }" + " field {" + " name: \"c\" label: LABEL_OPTIONAL type: TYPE_DOUBLE number: 3" + " options{" + " uninterpreted_option{" + " name{ name_part: \"f\" is_extension: true }" + " identifier_value: \"nan\"" + " }" + " }" + " }" + " field {" + " name: \"d\" label: LABEL_OPTIONAL type: TYPE_DOUBLE number: 4" + " options{" + " uninterpreted_option{" + " name{ name_part: \"f\" is_extension: true }" + " double_value: nan" + " }" + " }" + " }" + "}"); +} + TEST_F(ParseMessageTest, Oneof) { ExpectParsesTo( "message TestMessage {\n" @@ -1396,6 +1459,48 @@ TEST_F(ParseMiscTest, ParseFileOptions) { "}"); } +TEST_F(ParseMiscTest, InterpretedOptions) { + // Since we're importing the generated code from parsing/compiling + // unittest_custom_options.proto, we can just look at the option + // values from that file's descriptor in the generated code. + { + const MessageOptions& options = + protobuf_unittest::SettingRealsFromInf ::descriptor()->options(); + float float_val = options.GetExtension(protobuf_unittest::float_opt); + ASSERT_TRUE(std::isinf(float_val)); + ASSERT_GT(float_val, 0); + double double_val = options.GetExtension(protobuf_unittest::double_opt); + ASSERT_TRUE(std::isinf(double_val)); + ASSERT_GT(double_val, 0); + } + { + const MessageOptions& options = + protobuf_unittest::SettingRealsFromNegativeInf ::descriptor()->options(); + float float_val = options.GetExtension(protobuf_unittest::float_opt); + ASSERT_TRUE(std::isinf(float_val)); + ASSERT_LT(float_val, 0); + double double_val = options.GetExtension(protobuf_unittest::double_opt); + ASSERT_TRUE(std::isinf(double_val)); + ASSERT_LT(double_val, 0); + } + { + const MessageOptions& options = + protobuf_unittest::SettingRealsFromNan ::descriptor()->options(); + float float_val = options.GetExtension(protobuf_unittest::float_opt); + ASSERT_TRUE(std::isnan(float_val)); + double double_val = options.GetExtension(protobuf_unittest::double_opt); + ASSERT_TRUE(std::isnan(double_val)); + } + { + const MessageOptions& options = + protobuf_unittest::SettingRealsFromNegativeNan ::descriptor()->options(); + float float_val = options.GetExtension(protobuf_unittest::float_opt); + ASSERT_TRUE(std::isnan(float_val)); + double double_val = options.GetExtension(protobuf_unittest::double_opt); + ASSERT_TRUE(std::isnan(double_val)); + } +} + // =================================================================== // Error tests // diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index db45ca90ca..50d5a0a775 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -9149,6 +9149,10 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( value = uninterpreted_option_->positive_int_value(); } else if (uninterpreted_option_->has_negative_int_value()) { value = uninterpreted_option_->negative_int_value(); + } else if (uninterpreted_option_->identifier_value() == "inf") { + value = std::numeric_limits::infinity(); + } else if (uninterpreted_option_->identifier_value() == "nan") { + value = std::numeric_limits::quiet_NaN(); } else { return AddValueError([&] { return absl::StrCat("Value must be number for float option \"", @@ -9168,6 +9172,10 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( value = uninterpreted_option_->positive_int_value(); } else if (uninterpreted_option_->has_negative_int_value()) { value = uninterpreted_option_->negative_int_value(); + } else if (uninterpreted_option_->identifier_value() == "inf") { + value = std::numeric_limits::infinity(); + } else if (uninterpreted_option_->identifier_value() == "nan") { + value = std::numeric_limits::quiet_NaN(); } else { return AddValueError([&] { return absl::StrCat("Value must be number for double option \"", diff --git a/src/google/protobuf/unittest_custom_options.proto b/src/google/protobuf/unittest_custom_options.proto index ca6bec7a60..7075012778 100644 --- a/src/google/protobuf/unittest_custom_options.proto +++ b/src/google/protobuf/unittest_custom_options.proto @@ -191,6 +191,26 @@ message SettingRealsFromNegativeInts { option (double_opt) = -154; } +message SettingRealsFromInf { + option (float_opt) = inf; + option (double_opt) = inf; +} + +message SettingRealsFromNegativeInf { + option (float_opt) = -inf; + option (double_opt) = -inf; +} + +message SettingRealsFromNan { + option (float_opt) = nan; + option (double_opt) = nan; +} + +message SettingRealsFromNegativeNan { + option (float_opt) = -nan; + option (double_opt) = -nan; +} + // Options of complex message types, themselves combined and extended in // various ways. From 0e3650c29c562864243b616535a24d57b7a1ddb7 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Apr 2024 09:53:45 -0700 Subject: [PATCH 021/179] Add IsFrozen checks to conformance_upb and upb_js PiperOrigin-RevId: 627419894 --- upb/conformance/BUILD | 2 ++ upb/conformance/conformance_upb.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/upb/conformance/BUILD b/upb/conformance/BUILD index 9630c016bc..e645cc202b 100644 --- a/upb/conformance/BUILD +++ b/upb/conformance/BUILD @@ -73,6 +73,7 @@ cc_binary( "//editions:test_messages_proto3_editions_upbdefs", "//upb:base", "//upb:json", + "//upb:message", "//upb:port", "//upb:reflection", "//upb:text", @@ -127,6 +128,7 @@ cc_binary( "//editions:test_messages_proto3_editions_upbdefs", "//upb:base", "//upb:json", + "//upb:message", "//upb:port", "//upb:reflection", "//upb:text", diff --git a/upb/conformance/conformance_upb.c b/upb/conformance/conformance_upb.c index 349d9ec83f..28c713e8c5 100644 --- a/upb/conformance/conformance_upb.c +++ b/upb/conformance/conformance_upb.c @@ -23,6 +23,7 @@ #include "upb/base/upcast.h" #include "upb/json/decode.h" #include "upb/json/encode.h" +#include "upb/message/message.h" #include "upb/reflection/message.h" #include "upb/text/encode.h" #include "upb/wire/decode.h" @@ -68,6 +69,7 @@ typedef struct { } ctx; bool parse_proto(upb_Message* msg, const upb_MessageDef* m, const ctx* c) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView proto = conformance_ConformanceRequest_protobuf_payload(c->request); if (upb_Decode(proto.data, proto.size, msg, upb_MessageDef_MiniTable(m), NULL, @@ -117,6 +119,7 @@ void serialize_text(const upb_Message* msg, const upb_MessageDef* m, } bool parse_json(upb_Message* msg, const upb_MessageDef* m, const ctx* c) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView json = conformance_ConformanceRequest_json_payload(c->request); upb_Status status; int opts = 0; @@ -172,6 +175,7 @@ void serialize_json(const upb_Message* msg, const upb_MessageDef* m, } bool parse_input(upb_Message* msg, const upb_MessageDef* m, const ctx* c) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (conformance_ConformanceRequest_payload_case(c->request)) { case conformance_ConformanceRequest_payload_protobuf_payload: return parse_proto(msg, m, c); From bceb7f597ffd45a7c8c7c19205d24c58b6690930 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Apr 2024 12:23:24 -0700 Subject: [PATCH 022/179] Check IsFrozen for message/copy.c, experimental_jspb/decode.c, and json/decode.c PiperOrigin-RevId: 627468915 --- upb/json/decode.c | 17 +++++++++++++++++ upb/message/copy.c | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/upb/json/decode.c b/upb/json/decode.c index 46a122efe3..c578451836 100644 --- a/upb/json/decode.c +++ b/upb/json/decode.c @@ -861,6 +861,7 @@ static upb_MessageValue jsondec_bool(jsondec* d, const upb_FieldDef* f) { /* Composite types (array/message/map) ****************************************/ static void jsondec_array(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Array* arr = upb_Message_Mutable(msg, f, d->arena).array; jsondec_arrstart(d); @@ -874,6 +875,7 @@ static void jsondec_array(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { } static void jsondec_map(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Map* map = upb_Message_Mutable(msg, f, d->arena).map; const upb_MessageDef* entry = upb_FieldDef_MessageSubDef(f); const upb_FieldDef* key_f = upb_MessageDef_FindFieldByNumber(entry, 1); @@ -895,6 +897,7 @@ static void jsondec_map(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { static void jsondec_tomsg(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (upb_MessageDef_WellKnownType(m) == kUpb_WellKnown_Unspecified) { jsondec_object(d, msg, m); } else { @@ -915,6 +918,7 @@ static upb_MessageValue jsondec_msg(jsondec* d, const upb_FieldDef* f) { static void jsondec_field(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView name; const upb_FieldDef* f; const upb_FieldDef* preserved; @@ -980,6 +984,7 @@ static void jsondec_field(jsondec* d, upb_Message* msg, static void jsondec_object(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); jsondec_objstart(d); while (jsondec_objnext(d)) { jsondec_field(d, msg, m); @@ -1080,6 +1085,7 @@ static int64_t jsondec_unixtime(int y, int m, int d, int h, int min, int s) { static void jsondec_timestamp(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue seconds; upb_MessageValue nanos; upb_StringView str = jsondec_string(d); @@ -1145,6 +1151,7 @@ malformed: static void jsondec_duration(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue seconds; upb_MessageValue nanos; upb_StringView str = jsondec_string(d); @@ -1177,6 +1184,7 @@ static void jsondec_duration(jsondec* d, upb_Message* msg, static void jsondec_listvalue(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* values_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* value_m = upb_FieldDef_MessageSubDef(values_f); const upb_MiniTable* value_layout = upb_MessageDef_MiniTable(value_m); @@ -1195,6 +1203,7 @@ static void jsondec_listvalue(jsondec* d, upb_Message* msg, static void jsondec_struct(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* fields_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(fields_f); const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(entry_m, 2); @@ -1217,6 +1226,7 @@ static void jsondec_struct(jsondec* d, upb_Message* msg, static void jsondec_wellknownvalue(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue val; const upb_FieldDef* f; upb_Message* submsg; @@ -1305,6 +1315,7 @@ static upb_StringView jsondec_mask(jsondec* d, const char* buf, static void jsondec_fieldmask(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); /* repeated string paths = 1; */ const upb_FieldDef* paths_f = upb_MessageDef_FindFieldByNumber(m, 1); upb_Array* arr = upb_Message_Mutable(msg, paths_f, d->arena).array; @@ -1328,6 +1339,7 @@ static void jsondec_fieldmask(jsondec* d, upb_Message* msg, static void jsondec_anyfield(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (upb_MessageDef_WellKnownType(m) == kUpb_WellKnown_Unspecified) { /* For regular types: {"@type": "[user type]", "f1": , "f2": } * where f1, f2, etc. are the normal fields of this type. */ @@ -1346,6 +1358,7 @@ static void jsondec_anyfield(jsondec* d, upb_Message* msg, static const upb_MessageDef* jsondec_typeurl(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* type_url_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* type_m; upb_StringView type_url = jsondec_string(d); @@ -1375,6 +1388,7 @@ static const upb_MessageDef* jsondec_typeurl(jsondec* d, upb_Message* msg, } static void jsondec_any(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); /* string type_url = 1; * bytes value = 2; */ const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(m, 2); @@ -1443,6 +1457,7 @@ static void jsondec_any(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { static void jsondec_wrapper(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(m, 1); upb_JsonMessageValue val = jsondec_value(d, value_f); UPB_ASSUME(val.ignore == false); // Wrapper cannot be an enum. @@ -1451,6 +1466,7 @@ static void jsondec_wrapper(jsondec* d, upb_Message* msg, static void jsondec_wellknown(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (upb_MessageDef_WellKnownType(m)) { case kUpb_WellKnown_Any: jsondec_any(d, msg, m); @@ -1491,6 +1507,7 @@ static void jsondec_wellknown(jsondec* d, upb_Message* msg, static bool upb_JsonDecoder_Decode(jsondec* const d, upb_Message* const msg, const upb_MessageDef* const m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (UPB_SETJMP(d->err)) return false; jsondec_tomsg(d, msg, m); diff --git a/upb/message/copy.c b/upb/message/copy.c index 38cb8494bc..b3d6ef8624 100644 --- a/upb/message/copy.c +++ b/upb/message/copy.c @@ -117,6 +117,7 @@ static upb_Map* upb_Message_Map_DeepClone(const upb_Map* map, const upb_MiniTableField* f, upb_Message* clone, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(clone)); const upb_MiniTable* map_entry_table = upb_MiniTable_MapEntrySubMessage(mini_table, f); UPB_ASSERT(map_entry_table); @@ -160,6 +161,7 @@ static bool upb_Message_Array_DeepClone(const upb_Array* array, const upb_MiniTable* mini_table, const upb_MiniTableField* field, upb_Message* clone, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(clone)); UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); upb_Array* cloned_array = upb_Array_DeepClone( array, upb_MiniTableField_CType(field), @@ -185,6 +187,7 @@ static bool upb_Clone_ExtensionValue( upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* mini_table, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); upb_StringView empty_string = upb_StringView_FromDataAndSize(NULL, 0); // Only copy message area skipping upb_Message_Internal. memcpy(dst + 1, src + 1, mini_table->UPB_PRIVATE(size) - sizeof(upb_Message)); @@ -291,6 +294,7 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, bool upb_Message_DeepCopy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* mini_table, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); upb_Message_Clear(dst, mini_table); return _upb_Message_Copy(dst, src, mini_table, arena) != NULL; } @@ -307,6 +311,7 @@ upb_Message* upb_Message_DeepClone(const upb_Message* msg, // Performs a shallow copy. TODO: Extend to handle unknown fields. void upb_Message_ShallowCopy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* m) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); memcpy(dst, src, m->UPB_PRIVATE(size)); } From 5ad2ccb058575df47e918b8cb7b7a6cc9ded6aba Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 19:38:38 +0000 Subject: [PATCH 023/179] Auto-generate files after cl/627468915 --- php/ext/google/protobuf/php-upb.c | 22 ++++++++++++++++++++++ ruby/ext/google/protobuf_c/ruby-upb.c | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 880a18876e..9c880a6240 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -3119,6 +3119,7 @@ static upb_MessageValue jsondec_bool(jsondec* d, const upb_FieldDef* f) { /* Composite types (array/message/map) ****************************************/ static void jsondec_array(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Array* arr = upb_Message_Mutable(msg, f, d->arena).array; jsondec_arrstart(d); @@ -3132,6 +3133,7 @@ static void jsondec_array(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { } static void jsondec_map(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Map* map = upb_Message_Mutable(msg, f, d->arena).map; const upb_MessageDef* entry = upb_FieldDef_MessageSubDef(f); const upb_FieldDef* key_f = upb_MessageDef_FindFieldByNumber(entry, 1); @@ -3153,6 +3155,7 @@ static void jsondec_map(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { static void jsondec_tomsg(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (upb_MessageDef_WellKnownType(m) == kUpb_WellKnown_Unspecified) { jsondec_object(d, msg, m); } else { @@ -3173,6 +3176,7 @@ static upb_MessageValue jsondec_msg(jsondec* d, const upb_FieldDef* f) { static void jsondec_field(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView name; const upb_FieldDef* f; const upb_FieldDef* preserved; @@ -3238,6 +3242,7 @@ static void jsondec_field(jsondec* d, upb_Message* msg, static void jsondec_object(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); jsondec_objstart(d); while (jsondec_objnext(d)) { jsondec_field(d, msg, m); @@ -3338,6 +3343,7 @@ static int64_t jsondec_unixtime(int y, int m, int d, int h, int min, int s) { static void jsondec_timestamp(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue seconds; upb_MessageValue nanos; upb_StringView str = jsondec_string(d); @@ -3403,6 +3409,7 @@ malformed: static void jsondec_duration(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue seconds; upb_MessageValue nanos; upb_StringView str = jsondec_string(d); @@ -3435,6 +3442,7 @@ static void jsondec_duration(jsondec* d, upb_Message* msg, static void jsondec_listvalue(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* values_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* value_m = upb_FieldDef_MessageSubDef(values_f); const upb_MiniTable* value_layout = upb_MessageDef_MiniTable(value_m); @@ -3453,6 +3461,7 @@ static void jsondec_listvalue(jsondec* d, upb_Message* msg, static void jsondec_struct(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* fields_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(fields_f); const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(entry_m, 2); @@ -3475,6 +3484,7 @@ static void jsondec_struct(jsondec* d, upb_Message* msg, static void jsondec_wellknownvalue(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue val; const upb_FieldDef* f; upb_Message* submsg; @@ -3563,6 +3573,7 @@ static upb_StringView jsondec_mask(jsondec* d, const char* buf, static void jsondec_fieldmask(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); /* repeated string paths = 1; */ const upb_FieldDef* paths_f = upb_MessageDef_FindFieldByNumber(m, 1); upb_Array* arr = upb_Message_Mutable(msg, paths_f, d->arena).array; @@ -3586,6 +3597,7 @@ static void jsondec_fieldmask(jsondec* d, upb_Message* msg, static void jsondec_anyfield(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (upb_MessageDef_WellKnownType(m) == kUpb_WellKnown_Unspecified) { /* For regular types: {"@type": "[user type]", "f1": , "f2": } * where f1, f2, etc. are the normal fields of this type. */ @@ -3604,6 +3616,7 @@ static void jsondec_anyfield(jsondec* d, upb_Message* msg, static const upb_MessageDef* jsondec_typeurl(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* type_url_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* type_m; upb_StringView type_url = jsondec_string(d); @@ -3633,6 +3646,7 @@ static const upb_MessageDef* jsondec_typeurl(jsondec* d, upb_Message* msg, } static void jsondec_any(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); /* string type_url = 1; * bytes value = 2; */ const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(m, 2); @@ -3701,6 +3715,7 @@ static void jsondec_any(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { static void jsondec_wrapper(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(m, 1); upb_JsonMessageValue val = jsondec_value(d, value_f); UPB_ASSUME(val.ignore == false); // Wrapper cannot be an enum. @@ -3709,6 +3724,7 @@ static void jsondec_wrapper(jsondec* d, upb_Message* msg, static void jsondec_wellknown(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (upb_MessageDef_WellKnownType(m)) { case kUpb_WellKnown_Any: jsondec_any(d, msg, m); @@ -3749,6 +3765,7 @@ static void jsondec_wellknown(jsondec* d, upb_Message* msg, static bool upb_JsonDecoder_Decode(jsondec* const d, upb_Message* const msg, const upb_MessageDef* const m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (UPB_SETJMP(d->err)) return false; jsondec_tomsg(d, msg, m); @@ -6041,6 +6058,7 @@ static upb_Map* upb_Message_Map_DeepClone(const upb_Map* map, const upb_MiniTableField* f, upb_Message* clone, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(clone)); const upb_MiniTable* map_entry_table = upb_MiniTable_MapEntrySubMessage(mini_table, f); UPB_ASSERT(map_entry_table); @@ -6084,6 +6102,7 @@ static bool upb_Message_Array_DeepClone(const upb_Array* array, const upb_MiniTable* mini_table, const upb_MiniTableField* field, upb_Message* clone, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(clone)); UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); upb_Array* cloned_array = upb_Array_DeepClone( array, upb_MiniTableField_CType(field), @@ -6109,6 +6128,7 @@ static bool upb_Clone_ExtensionValue( upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* mini_table, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); upb_StringView empty_string = upb_StringView_FromDataAndSize(NULL, 0); // Only copy message area skipping upb_Message_Internal. memcpy(dst + 1, src + 1, mini_table->UPB_PRIVATE(size) - sizeof(upb_Message)); @@ -6215,6 +6235,7 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, bool upb_Message_DeepCopy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* mini_table, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); upb_Message_Clear(dst, mini_table); return _upb_Message_Copy(dst, src, mini_table, arena) != NULL; } @@ -6231,6 +6252,7 @@ upb_Message* upb_Message_DeepClone(const upb_Message* msg, // Performs a shallow copy. TODO: Extend to handle unknown fields. void upb_Message_ShallowCopy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* m) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); memcpy(dst, src, m->UPB_PRIVATE(size)); } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 2ad33d5618..4caee9dc69 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -2612,6 +2612,7 @@ static upb_MessageValue jsondec_bool(jsondec* d, const upb_FieldDef* f) { /* Composite types (array/message/map) ****************************************/ static void jsondec_array(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Array* arr = upb_Message_Mutable(msg, f, d->arena).array; jsondec_arrstart(d); @@ -2625,6 +2626,7 @@ static void jsondec_array(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { } static void jsondec_map(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Map* map = upb_Message_Mutable(msg, f, d->arena).map; const upb_MessageDef* entry = upb_FieldDef_MessageSubDef(f); const upb_FieldDef* key_f = upb_MessageDef_FindFieldByNumber(entry, 1); @@ -2646,6 +2648,7 @@ static void jsondec_map(jsondec* d, upb_Message* msg, const upb_FieldDef* f) { static void jsondec_tomsg(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (upb_MessageDef_WellKnownType(m) == kUpb_WellKnown_Unspecified) { jsondec_object(d, msg, m); } else { @@ -2666,6 +2669,7 @@ static upb_MessageValue jsondec_msg(jsondec* d, const upb_FieldDef* f) { static void jsondec_field(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView name; const upb_FieldDef* f; const upb_FieldDef* preserved; @@ -2731,6 +2735,7 @@ static void jsondec_field(jsondec* d, upb_Message* msg, static void jsondec_object(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); jsondec_objstart(d); while (jsondec_objnext(d)) { jsondec_field(d, msg, m); @@ -2831,6 +2836,7 @@ static int64_t jsondec_unixtime(int y, int m, int d, int h, int min, int s) { static void jsondec_timestamp(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue seconds; upb_MessageValue nanos; upb_StringView str = jsondec_string(d); @@ -2896,6 +2902,7 @@ malformed: static void jsondec_duration(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue seconds; upb_MessageValue nanos; upb_StringView str = jsondec_string(d); @@ -2928,6 +2935,7 @@ static void jsondec_duration(jsondec* d, upb_Message* msg, static void jsondec_listvalue(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* values_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* value_m = upb_FieldDef_MessageSubDef(values_f); const upb_MiniTable* value_layout = upb_MessageDef_MiniTable(value_m); @@ -2946,6 +2954,7 @@ static void jsondec_listvalue(jsondec* d, upb_Message* msg, static void jsondec_struct(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* fields_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(fields_f); const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(entry_m, 2); @@ -2968,6 +2977,7 @@ static void jsondec_struct(jsondec* d, upb_Message* msg, static void jsondec_wellknownvalue(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_MessageValue val; const upb_FieldDef* f; upb_Message* submsg; @@ -3056,6 +3066,7 @@ static upb_StringView jsondec_mask(jsondec* d, const char* buf, static void jsondec_fieldmask(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); /* repeated string paths = 1; */ const upb_FieldDef* paths_f = upb_MessageDef_FindFieldByNumber(m, 1); upb_Array* arr = upb_Message_Mutable(msg, paths_f, d->arena).array; @@ -3079,6 +3090,7 @@ static void jsondec_fieldmask(jsondec* d, upb_Message* msg, static void jsondec_anyfield(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (upb_MessageDef_WellKnownType(m) == kUpb_WellKnown_Unspecified) { /* For regular types: {"@type": "[user type]", "f1": , "f2": } * where f1, f2, etc. are the normal fields of this type. */ @@ -3097,6 +3109,7 @@ static void jsondec_anyfield(jsondec* d, upb_Message* msg, static const upb_MessageDef* jsondec_typeurl(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* type_url_f = upb_MessageDef_FindFieldByNumber(m, 1); const upb_MessageDef* type_m; upb_StringView type_url = jsondec_string(d); @@ -3126,6 +3139,7 @@ static const upb_MessageDef* jsondec_typeurl(jsondec* d, upb_Message* msg, } static void jsondec_any(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); /* string type_url = 1; * bytes value = 2; */ const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(m, 2); @@ -3194,6 +3208,7 @@ static void jsondec_any(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { static void jsondec_wrapper(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_FieldDef* value_f = upb_MessageDef_FindFieldByNumber(m, 1); upb_JsonMessageValue val = jsondec_value(d, value_f); UPB_ASSUME(val.ignore == false); // Wrapper cannot be an enum. @@ -3202,6 +3217,7 @@ static void jsondec_wrapper(jsondec* d, upb_Message* msg, static void jsondec_wellknown(jsondec* d, upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (upb_MessageDef_WellKnownType(m)) { case kUpb_WellKnown_Any: jsondec_any(d, msg, m); @@ -3242,6 +3258,7 @@ static void jsondec_wellknown(jsondec* d, upb_Message* msg, static bool upb_JsonDecoder_Decode(jsondec* const d, upb_Message* const msg, const upb_MessageDef* const m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); if (UPB_SETJMP(d->err)) return false; jsondec_tomsg(d, msg, m); @@ -5534,6 +5551,7 @@ static upb_Map* upb_Message_Map_DeepClone(const upb_Map* map, const upb_MiniTableField* f, upb_Message* clone, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(clone)); const upb_MiniTable* map_entry_table = upb_MiniTable_MapEntrySubMessage(mini_table, f); UPB_ASSERT(map_entry_table); @@ -5577,6 +5595,7 @@ static bool upb_Message_Array_DeepClone(const upb_Array* array, const upb_MiniTable* mini_table, const upb_MiniTableField* field, upb_Message* clone, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(clone)); UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); upb_Array* cloned_array = upb_Array_DeepClone( array, upb_MiniTableField_CType(field), @@ -5602,6 +5621,7 @@ static bool upb_Clone_ExtensionValue( upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* mini_table, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); upb_StringView empty_string = upb_StringView_FromDataAndSize(NULL, 0); // Only copy message area skipping upb_Message_Internal. memcpy(dst + 1, src + 1, mini_table->UPB_PRIVATE(size) - sizeof(upb_Message)); @@ -5708,6 +5728,7 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, bool upb_Message_DeepCopy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* mini_table, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); upb_Message_Clear(dst, mini_table); return _upb_Message_Copy(dst, src, mini_table, arena) != NULL; } @@ -5724,6 +5745,7 @@ upb_Message* upb_Message_DeepClone(const upb_Message* msg, // Performs a shallow copy. TODO: Extend to handle unknown fields. void upb_Message_ShallowCopy(upb_Message* dst, const upb_Message* src, const upb_MiniTable* m) { + UPB_ASSERT(!upb_Message_IsFrozen(dst)); memcpy(dst, src, m->UPB_PRIVATE(size)); } From bd9f211f9712c97a442735b5709a795ed2bab662 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Apr 2024 13:43:40 -0700 Subject: [PATCH 024/179] Remove IsFrozen checks from upb/conformance PiperOrigin-RevId: 627492449 --- upb/conformance/BUILD | 2 -- upb/conformance/conformance_upb.c | 4 ---- 2 files changed, 6 deletions(-) diff --git a/upb/conformance/BUILD b/upb/conformance/BUILD index e645cc202b..9630c016bc 100644 --- a/upb/conformance/BUILD +++ b/upb/conformance/BUILD @@ -73,7 +73,6 @@ cc_binary( "//editions:test_messages_proto3_editions_upbdefs", "//upb:base", "//upb:json", - "//upb:message", "//upb:port", "//upb:reflection", "//upb:text", @@ -128,7 +127,6 @@ cc_binary( "//editions:test_messages_proto3_editions_upbdefs", "//upb:base", "//upb:json", - "//upb:message", "//upb:port", "//upb:reflection", "//upb:text", diff --git a/upb/conformance/conformance_upb.c b/upb/conformance/conformance_upb.c index 28c713e8c5..349d9ec83f 100644 --- a/upb/conformance/conformance_upb.c +++ b/upb/conformance/conformance_upb.c @@ -23,7 +23,6 @@ #include "upb/base/upcast.h" #include "upb/json/decode.h" #include "upb/json/encode.h" -#include "upb/message/message.h" #include "upb/reflection/message.h" #include "upb/text/encode.h" #include "upb/wire/decode.h" @@ -69,7 +68,6 @@ typedef struct { } ctx; bool parse_proto(upb_Message* msg, const upb_MessageDef* m, const ctx* c) { - UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView proto = conformance_ConformanceRequest_protobuf_payload(c->request); if (upb_Decode(proto.data, proto.size, msg, upb_MessageDef_MiniTable(m), NULL, @@ -119,7 +117,6 @@ void serialize_text(const upb_Message* msg, const upb_MessageDef* m, } bool parse_json(upb_Message* msg, const upb_MessageDef* m, const ctx* c) { - UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_StringView json = conformance_ConformanceRequest_json_payload(c->request); upb_Status status; int opts = 0; @@ -175,7 +172,6 @@ void serialize_json(const upb_Message* msg, const upb_MessageDef* m, } bool parse_input(upb_Message* msg, const upb_MessageDef* m, const ctx* c) { - UPB_ASSERT(!upb_Message_IsFrozen(msg)); switch (conformance_ConformanceRequest_payload_case(c->request)) { case conformance_ConformanceRequest_payload_protobuf_payload: return parse_proto(msg, m, c); From 0e223312edcc61592e8bc9cebf536fe8211e7445 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 15:07:56 -0700 Subject: [PATCH 025/179] Internal change. PiperOrigin-RevId: 627518025 --- src/google/protobuf/text_format.cc | 7 +------ src/google/protobuf/text_format.h | 2 -- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 3f4d3ba260..94efb981cc 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -97,10 +97,6 @@ const char kDebugStringSilentMarkerForDetection[] = "\t "; // Controls insertion of kDebugStringSilentMarker into DebugString() output. PROTOBUF_EXPORT std::atomic enable_debug_text_format_marker; -// Controls insertion of a marker making debug strings non-parseable, and -// redacting annotated fields. -PROTOBUF_EXPORT std::atomic enable_debug_text_redaction{true}; - int64_t GetRedactedFieldCount() { return num_redacted_field.load(std::memory_order_relaxed); } @@ -182,8 +178,7 @@ std::string StringifyMessage(const Message& message, Option option) { break; } printer.SetExpandAny(true); - printer.SetRedactDebugString( - internal::enable_debug_text_redaction.load(std::memory_order_relaxed)); + printer.SetRedactDebugString(true); printer.SetRandomizeDebugString(true); printer.SetReportSensitiveFields(reporter); std::string result; diff --git a/src/google/protobuf/text_format.h b/src/google/protobuf/text_format.h index 05ae3508a2..97d1822619 100644 --- a/src/google/protobuf/text_format.h +++ b/src/google/protobuf/text_format.h @@ -45,8 +45,6 @@ PROTOBUF_EXPORT extern const char kDebugStringSilentMarker[1]; PROTOBUF_EXPORT extern const char kDebugStringSilentMarkerForDetection[3]; PROTOBUF_EXPORT extern std::atomic enable_debug_text_format_marker; -PROTOBUF_EXPORT extern std::atomic enable_debug_text_detection; -PROTOBUF_EXPORT extern std::atomic enable_debug_text_redaction; PROTOBUF_EXPORT int64_t GetRedactedFieldCount(); PROTOBUF_EXPORT bool ShouldRedactField(const FieldDescriptor* field); From 5b5db6666e79198bea8915300c40a90d27795b48 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 23 Apr 2024 16:01:27 -0700 Subject: [PATCH 026/179] Check IsFrozen for internal/message.c and reflection/message.c PiperOrigin-RevId: 627531912 --- upb/message/internal/message.c | 1 + upb/reflection/message.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/upb/message/internal/message.c b/upb/message/internal/message.c index 856fc97016..dc31f13e22 100644 --- a/upb/message/internal/message.c +++ b/upb/message/internal/message.c @@ -23,6 +23,7 @@ const double kUpb_NaN = NAN; bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const size_t overhead = sizeof(upb_Message_Internal); upb_Message_Internal* in = UPB_PRIVATE(_upb_Message_GetInternal)(msg); diff --git a/upb/reflection/message.c b/upb/reflection/message.c index b0c4a401c3..9211738c33 100644 --- a/upb/reflection/message.c +++ b/upb/reflection/message.c @@ -65,6 +65,7 @@ upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, const upb_FieldDef* f, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); UPB_ASSERT(upb_FieldDef_IsSubMessage(f) || upb_FieldDef_IsRepeated(f)); if (upb_FieldDef_HasPresence(f) && !upb_Message_HasFieldByDef(msg, f)) { // We need to skip the upb_Message_GetFieldByDef() call in this case. @@ -103,6 +104,7 @@ make: bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, upb_MessageValue val, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_MiniTableField* m_f = upb_FieldDef_MiniTable(f); if (upb_MiniTableField_IsExtension(m_f)) { @@ -115,6 +117,7 @@ bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, } void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_MiniTableField* m_f = upb_FieldDef_MiniTable(f); if (upb_MiniTableField_IsExtension(m_f)) { @@ -125,6 +128,7 @@ void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { } void upb_Message_ClearByDef(upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Message_Clear(msg, upb_MessageDef_MiniTable(m)); } @@ -186,6 +190,7 @@ bool upb_Message_Next(const upb_Message* msg, const upb_MessageDef* m, bool _upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, int depth) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); size_t iter = kUpb_Message_Begin; const upb_FieldDef* f; upb_MessageValue val; From 322d4d44588f25df15ddd0782c463e195c2a577a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 23:16:57 +0000 Subject: [PATCH 027/179] Auto-generate files after cl/627531912 --- php/ext/google/protobuf/php-upb.c | 6 ++++++ ruby/ext/google/protobuf_c/ruby-upb.c | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 9c880a6240..7b7806928e 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -12021,6 +12021,7 @@ const double kUpb_NaN = NAN; bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const size_t overhead = sizeof(upb_Message_Internal); upb_Message_Internal* in = UPB_PRIVATE(_upb_Message_GetInternal)(msg); @@ -15423,6 +15424,7 @@ upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, const upb_FieldDef* f, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); UPB_ASSERT(upb_FieldDef_IsSubMessage(f) || upb_FieldDef_IsRepeated(f)); if (upb_FieldDef_HasPresence(f) && !upb_Message_HasFieldByDef(msg, f)) { // We need to skip the upb_Message_GetFieldByDef() call in this case. @@ -15461,6 +15463,7 @@ make: bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, upb_MessageValue val, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_MiniTableField* m_f = upb_FieldDef_MiniTable(f); if (upb_MiniTableField_IsExtension(m_f)) { @@ -15473,6 +15476,7 @@ bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, } void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_MiniTableField* m_f = upb_FieldDef_MiniTable(f); if (upb_MiniTableField_IsExtension(m_f)) { @@ -15483,6 +15487,7 @@ void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { } void upb_Message_ClearByDef(upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Message_Clear(msg, upb_MessageDef_MiniTable(m)); } @@ -15544,6 +15549,7 @@ bool upb_Message_Next(const upb_Message* msg, const upb_MessageDef* m, bool _upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, int depth) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); size_t iter = kUpb_Message_Begin; const upb_FieldDef* f; upb_MessageValue val; diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 4caee9dc69..d90b3eea1d 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -11514,6 +11514,7 @@ const double kUpb_NaN = NAN; bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const size_t overhead = sizeof(upb_Message_Internal); upb_Message_Internal* in = UPB_PRIVATE(_upb_Message_GetInternal)(msg); @@ -14916,6 +14917,7 @@ upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, const upb_FieldDef* f, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); UPB_ASSERT(upb_FieldDef_IsSubMessage(f) || upb_FieldDef_IsRepeated(f)); if (upb_FieldDef_HasPresence(f) && !upb_Message_HasFieldByDef(msg, f)) { // We need to skip the upb_Message_GetFieldByDef() call in this case. @@ -14954,6 +14956,7 @@ make: bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, upb_MessageValue val, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_MiniTableField* m_f = upb_FieldDef_MiniTable(f); if (upb_MiniTableField_IsExtension(m_f)) { @@ -14966,6 +14969,7 @@ bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, } void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); const upb_MiniTableField* m_f = upb_FieldDef_MiniTable(f); if (upb_MiniTableField_IsExtension(m_f)) { @@ -14976,6 +14980,7 @@ void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { } void upb_Message_ClearByDef(upb_Message* msg, const upb_MessageDef* m) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Message_Clear(msg, upb_MessageDef_MiniTable(m)); } @@ -15037,6 +15042,7 @@ bool upb_Message_Next(const upb_Message* msg, const upb_MessageDef* m, bool _upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, int depth) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); size_t iter = kUpb_Message_Begin; const upb_FieldDef* f; upb_MessageValue val; From 2c16decbcee58f0ad3c9ca4ac722e02ac22b4bf7 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 23 Apr 2024 19:55:13 -0700 Subject: [PATCH 028/179] Internal change. PiperOrigin-RevId: 627582199 --- src/google/protobuf/text_format.cc | 109 +++++++++++++++++------------ src/google/protobuf/text_format.h | 14 ++-- 2 files changed, 75 insertions(+), 48 deletions(-) diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 94efb981cc..6b3c01bfbb 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -97,12 +97,61 @@ const char kDebugStringSilentMarkerForDetection[] = "\t "; // Controls insertion of kDebugStringSilentMarker into DebugString() output. PROTOBUF_EXPORT std::atomic enable_debug_text_format_marker; +// Controls insertion of a marker making debug strings non-parseable, and +// redacting annotated fields in Protobuf's DebugString APIs. +PROTOBUF_EXPORT std::atomic enable_debug_string_safe_format{false}; + int64_t GetRedactedFieldCount() { return num_redacted_field.load(std::memory_order_relaxed); } + +enum class Option { kNone, kShort, kUTF8 }; + +std::string StringifyMessage(const Message& message, Option option, + FieldReporterLevel reporter_level, + bool enable_safe_format) { + // Indicate all scoped reflection calls are from DebugString function. + ScopedReflectionMode scope(ReflectionMode::kDebugString); + + TextFormat::Printer printer; + internal::FieldReporterLevel reporter = reporter_level; + switch (option) { + case Option::kShort: + printer.SetSingleLineMode(true); + break; + case Option::kUTF8: + printer.SetUseUtf8StringEscaping(true); + break; + case Option::kNone: + break; + } + printer.SetExpandAny(true); + printer.SetRedactDebugString(enable_safe_format); + printer.SetRandomizeDebugString(enable_safe_format); + printer.SetReportSensitiveFields(reporter); + std::string result; + printer.PrintToString(message, &result); + + if (option == Option::kShort) { + TrimTrailingSpace(result); + } + + return result; +} + +PROTOBUF_EXPORT std::string StringifyMessage(const Message& message) { + return StringifyMessage(message, Option::kNone, + FieldReporterLevel::kAbslStringify, true); +} } // namespace internal std::string Message::DebugString() const { + bool enable_safe_format = + internal::enable_debug_string_safe_format.load(std::memory_order_relaxed); + if (enable_safe_format) { + return StringifyMessage(*this, internal::Option::kNone, + FieldReporterLevel::kDebugString, true); + } // Indicate all scoped reflection calls are from DebugString function. ScopedReflectionMode scope(ReflectionMode::kDebugString); std::string debug_string; @@ -119,6 +168,12 @@ std::string Message::DebugString() const { } std::string Message::ShortDebugString() const { + bool enable_safe_format = + internal::enable_debug_string_safe_format.load(std::memory_order_relaxed); + if (enable_safe_format) { + return StringifyMessage(*this, internal::Option::kShort, + FieldReporterLevel::kShortDebugString, true); + } // Indicate all scoped reflection calls are from DebugString function. ScopedReflectionMode scope(ReflectionMode::kDebugString); std::string debug_string; @@ -137,6 +192,12 @@ std::string Message::ShortDebugString() const { } std::string Message::Utf8DebugString() const { + bool enable_safe_format = + internal::enable_debug_string_safe_format.load(std::memory_order_relaxed); + if (enable_safe_format) { + return StringifyMessage(*this, internal::Option::kUTF8, + FieldReporterLevel::kUtf8DebugString, true); + } // Indicate all scoped reflection calls are from DebugString function. ScopedReflectionMode scope(ReflectionMode::kDebugString); std::string debug_string; @@ -155,54 +216,14 @@ std::string Message::Utf8DebugString() const { void Message::PrintDebugString() const { printf("%s", DebugString().c_str()); } -namespace internal { - -enum class Option { kNone, kShort, kUTF8 }; - -std::string StringifyMessage(const Message& message, Option option) { - // Indicate all scoped reflection calls are from DebugString function. - ScopedReflectionMode scope(ReflectionMode::kDebugString); - - TextFormat::Printer printer; - internal::FieldReporterLevel reporter = FieldReporterLevel::kAbslStringify; - switch (option) { - case Option::kShort: - printer.SetSingleLineMode(true); - reporter = FieldReporterLevel::kShortFormat; - break; - case Option::kUTF8: - printer.SetUseUtf8StringEscaping(true); - reporter = FieldReporterLevel::kUtf8Format; - break; - case Option::kNone: - break; - } - printer.SetExpandAny(true); - printer.SetRedactDebugString(true); - printer.SetRandomizeDebugString(true); - printer.SetReportSensitiveFields(reporter); - std::string result; - printer.PrintToString(message, &result); - - if (option == Option::kShort) { - TrimTrailingSpace(result); - } - - return result; -} - -PROTOBUF_EXPORT std::string StringifyMessage(const Message& message) { - return StringifyMessage(message, Option::kNone); -} - -} // namespace internal - PROTOBUF_EXPORT std::string ShortFormat(const Message& message) { - return internal::StringifyMessage(message, internal::Option::kShort); + return internal::StringifyMessage(message, internal::Option::kShort, + FieldReporterLevel::kShortFormat, true); } PROTOBUF_EXPORT std::string Utf8Format(const Message& message) { - return internal::StringifyMessage(message, internal::Option::kUTF8); + return internal::StringifyMessage(message, internal::Option::kUTF8, + FieldReporterLevel::kUtf8Format, true); } diff --git a/src/google/protobuf/text_format.h b/src/google/protobuf/text_format.h index 97d1822619..ba9210f344 100644 --- a/src/google/protobuf/text_format.h +++ b/src/google/protobuf/text_format.h @@ -45,6 +45,7 @@ PROTOBUF_EXPORT extern const char kDebugStringSilentMarker[1]; PROTOBUF_EXPORT extern const char kDebugStringSilentMarkerForDetection[3]; PROTOBUF_EXPORT extern std::atomic enable_debug_text_format_marker; +PROTOBUF_EXPORT extern std::atomic enable_debug_string_safe_format; PROTOBUF_EXPORT int64_t GetRedactedFieldCount(); PROTOBUF_EXPORT bool ShouldRedactField(const FieldDescriptor* field); @@ -86,9 +87,13 @@ namespace internal { // Enum used to set printing options for StringifyMessage. PROTOBUF_EXPORT enum class Option; -// Converts a protobuf message to a string with redaction enabled. +// Converts a protobuf message to a string. If enable_safe_format is true, +// sensitive fields are redacted, and a per-process randomized prefix is +// inserted. PROTOBUF_EXPORT std::string StringifyMessage(const Message& message, - Option option); + Option option, + FieldReporterLevel reporter_level, + bool enable_safe_format); class UnsetFieldsMetadataTextFormatTestUtil; class UnsetFieldsMetadataMessageDifferencerTestUtil; @@ -451,8 +456,9 @@ class PROTOBUF_EXPORT TextFormat { friend std::string Message::DebugString() const; friend std::string Message::ShortDebugString() const; friend std::string Message::Utf8DebugString() const; - friend std::string internal::StringifyMessage(const Message& message, - internal::Option option); + friend std::string internal::StringifyMessage( + const Message& message, internal::Option option, + internal::FieldReporterLevel reporter_level, bool enable_safe_format); // Sets whether silent markers will be inserted. void SetInsertSilentMarker(bool v) { insert_silent_marker_ = v; } From 1a7ce6106115d0ff1c2b82a1a9a10d6603a2c2a1 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Wed, 24 Apr 2024 06:21:54 -0700 Subject: [PATCH 029/179] Split Proxied into Proxied and MutProxied traits. Proxied is not marked as Sized yet, because ProtoStr is still dynamically sized. We will wait for clarity for the string types before marking Proxied Sized. PiperOrigin-RevId: 627707544 --- rust/map.rs | 5 ++- rust/optional.rs | 7 ++++- rust/primitive.rs | 7 ++++- rust/proto_macro.rs | 8 ++--- rust/proxied.rs | 33 ++++++++++++++------ rust/repeated.rs | 8 ++++- rust/shared.rs | 2 +- rust/string.rs | 10 ++++-- rust/vtable.rs | 6 ++-- src/google/protobuf/compiler/rust/enum.cc | 3 ++ src/google/protobuf/compiler/rust/message.cc | 3 ++ 11 files changed, 68 insertions(+), 24 deletions(-) diff --git a/rust/map.rs b/rust/map.rs index 5ce74047b6..367793668f 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -6,7 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd use crate::{ - Mut, MutProxy, Proxied, SettableValue, View, ViewProxy, + Mut, MutProxied, MutProxy, Proxied, SettableValue, View, ViewProxy, __internal::Private, __runtime::{InnerMap, InnerMapMut, RawMap, RawMapIter}, }; @@ -95,6 +95,9 @@ where impl + ?Sized> Proxied for Map { type View<'msg> = MapView<'msg, K, V> where K: 'msg, V: 'msg; +} + +impl + ?Sized> MutProxied for Map { type Mut<'msg> = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } diff --git a/rust/optional.rs b/rust/optional.rs index 0268c61bfd..741f12d4f5 100644 --- a/rust/optional.rs +++ b/rust/optional.rs @@ -10,7 +10,9 @@ #![allow(unused)] use crate::__internal::Private; -use crate::{Mut, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy}; +use crate::{ + Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, +}; use std::convert::{AsMut, AsRef}; use std::fmt::{self, Debug}; use std::panic; @@ -495,6 +497,9 @@ mod tests { impl Proxied for VtableProxied { type View<'msg> = VtableProxiedView; + } + + impl MutProxied for VtableProxied { type Mut<'msg> = VtableProxiedMut<'msg>; } diff --git a/rust/primitive.rs b/rust/primitive.rs index 7c34795c9a..f14863187b 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -10,7 +10,9 @@ use std::fmt::Debug; use crate::__internal::Private; use crate::__runtime::InnerPrimitiveMut; use crate::vtable::{PrimitiveWithRawVTable, ProxiedWithRawVTable, RawVTableOptionalMutatorData}; -use crate::{Mut, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy}; +use crate::{ + Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, +}; /// A mutator for a primitive (numeric or enum) value of `T`. /// @@ -98,6 +100,9 @@ macro_rules! impl_singular_primitives { $( impl Proxied for $t { type View<'msg> = $t; + } + + impl MutProxied for $t { type Mut<'msg> = PrimitiveMut<'msg, $t>; } diff --git a/rust/proto_macro.rs b/rust/proto_macro.rs index d3da607664..95b14232bf 100644 --- a/rust/proto_macro.rs +++ b/rust/proto_macro.rs @@ -53,13 +53,13 @@ macro_rules! proto_internal { // nested message (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $field:ident : $($value:tt)* }) => { { - let mut $msg: <$($msgtype)::+ as $crate::Proxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + let mut $msg: <$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); proto_internal!(@msg $msg $field : $($value)*); } }; (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $field:ident : $($value:tt)* }) => { { - let mut $msg: <::$($msgtype)::+ as $crate::Proxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + let mut $msg: <::$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); proto_internal!(@msg $msg $field : $($value)*); } }; @@ -77,12 +77,12 @@ macro_rules! proto_internal { // empty nested message (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { }) => { { - let mut $msg: <$($msgtype)::+ as $crate::Proxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + let mut $msg: <$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); } }; (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { }) => { { - let mut $msg: <::$($msgtype)::+ as $crate::Proxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + let mut $msg: <::$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); } }; diff --git a/rust/proxied.rs b/rust/proxied.rs index 8d9214d28e..41ddb6a743 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -51,18 +51,25 @@ use std::fmt::Debug; /// A type that can be accessed through a reference-like proxy. /// -/// An instance of a `Proxied` can be accessed -/// immutably via `Proxied::View` and mutably via `Proxied::Mut`. +/// An instance of a `Proxied` can be accessed immutably via `Proxied::View`. /// /// All Protobuf field types implement `Proxied`. pub trait Proxied { /// The proxy type that provides shared access to a `T`, like a `&'msg T`. /// /// Most code should use the type alias [`View`]. - type View<'msg>: ViewProxy<'msg, Proxied = Self> + Copy + Send + SettableValue + type View<'msg>: ViewProxy<'msg, Proxied = Self> + Copy + Send where Self: 'msg; +} +/// A type that can be be accessed through a reference-like proxy. +/// +/// An instance of a `MutProxied` can be accessed mutably via `MutProxied::Mut` +/// and immutably via `MutProxied::View`. +/// +/// `MutProxied` is implemented by message, map and repeated field types. +pub trait MutProxied: Proxied { /// The proxy type that provides exclusive mutable access to a `T`, like a /// `&'msg mut T`. /// @@ -83,7 +90,7 @@ pub type View<'msg, T> = ::View<'msg>; /// /// This is more concise than fully spelling the associated type. #[allow(dead_code)] -pub type Mut<'msg, T> = ::Mut<'msg>; +pub type Mut<'msg, T> = ::Mut<'msg>; /// Declares conversion operations common to all views. /// @@ -147,7 +154,10 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { /// /// This trait is intentionally made non-object-safe to prevent a potential /// future incompatible change. -pub trait MutProxy<'msg>: ViewProxy<'msg> { +pub trait MutProxy<'msg>: ViewProxy<'msg> +where + Self::Proxied: MutProxied, +{ /// Gets an immutable view of this field. This is shorthand for `as_view`. /// /// This provides a shorter lifetime than `into_view` but can also be called @@ -211,7 +221,7 @@ pub trait MutProxy<'msg>: ViewProxy<'msg> { /// /// All scalar and message types implement `ProxiedWithPresence`, while repeated /// types don't. -pub trait ProxiedWithPresence: Proxied { +pub trait ProxiedWithPresence: MutProxied { /// The data necessary to store a present field mutator proxying `Self`. /// This is the contents of `PresentField<'msg, Self>`. type PresentMutData<'msg>: MutProxy<'msg, Proxied = Self>; @@ -233,7 +243,7 @@ pub trait ProxiedWithPresence: Proxied { /// Values that can be used to set a field of `T`. pub trait SettableValue: Sized where - T: Proxied + ?Sized, + T: MutProxied + ?Sized, { /// Consumes `self` to set the given mutator to the value of `self`. #[doc(hidden)] @@ -308,6 +318,9 @@ mod tests { impl Proxied for MyProxied { type View<'msg> = MyProxiedView<'msg>; + } + + impl MutProxied for MyProxied { type Mut<'msg> = MyProxiedMut<'msg>; } @@ -460,7 +473,7 @@ mod tests { y: &'b View<'a, T>, ) -> [View<'b, T>; 2] where - T: Proxied, + T: MutProxied, 'a: 'b, { // `[x, y]` fails to compile because `'a` is not the same as `'b` and the `View` @@ -509,7 +522,7 @@ mod tests { fn reborrow_generic_mut_into_view<'a, 'b, T>(x: Mut<'a, T>, y: View<'b, T>) -> [View<'b, T>; 2] where - T: Proxied, + T: MutProxied, 'a: 'b, { [x.into_view(), y] @@ -529,7 +542,7 @@ mod tests { fn reborrow_generic_mut_into_mut<'a, 'b, T>(x: Mut<'a, T>, y: Mut<'b, T>) -> [Mut<'b, T>; 2] where - T: Proxied, + T: MutProxied, 'a: 'b, { // `[x, y]` fails to compile because `'a` is not the same as `'b` and the `Mut` diff --git a/rust/repeated.rs b/rust/repeated.rs index 068dd9a5ec..7253307fcd 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -15,7 +15,7 @@ use std::iter::FusedIterator; use std::marker::PhantomData; use crate::{ - Mut, MutProxy, Proxied, SettableValue, View, ViewProxy, + Mut, MutProxied, MutProxy, Proxied, SettableValue, View, ViewProxy, __internal::Private, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -314,6 +314,12 @@ where T: ProxiedInRepeated + ?Sized, { type View<'msg> = RepeatedView<'msg, T> where Repeated: 'msg; +} + +impl MutProxied for Repeated +where + T: ProxiedInRepeated + ?Sized, +{ type Mut<'msg> = RepeatedMut<'msg, T> where Repeated: 'msg; } diff --git a/rust/shared.rs b/rust/shared.rs index 531d4d4e82..79ff7e629a 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -28,7 +28,7 @@ pub mod __public { pub use crate::primitive::PrimitiveMut; pub use crate::proto; pub use crate::proxied::{ - Mut, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, + Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, }; pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, diff --git a/rust/string.rs b/rust/string.rs index a6277d4b79..9377a53514 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -15,8 +15,8 @@ use crate::__runtime::{ }; use crate::macros::impl_forwarding_settable_value; use crate::{ - AbsentField, FieldEntry, Mut, MutProxy, Optional, PresentField, Proxied, ProxiedWithPresence, - SettableValue, View, ViewProxy, + AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, Proxied, + ProxiedWithPresence, SettableValue, View, ViewProxy, }; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; @@ -123,6 +123,9 @@ impl AsRef<[u8]> for BytesMut<'_> { impl Proxied for [u8] { type View<'msg> = &'msg [u8]; +} + +impl MutProxied for [u8] { type Mut<'msg> = BytesMut<'msg>; } @@ -465,6 +468,9 @@ impl Ord for ProtoStr { impl Proxied for ProtoStr { type View<'msg> = &'msg ProtoStr; +} + +impl MutProxied for ProtoStr { type Mut<'msg> = ProtoStrMut<'msg>; } diff --git a/rust/vtable.rs b/rust/vtable.rs index 52a0d87bc2..fbcf4c46a2 100644 --- a/rust/vtable.rs +++ b/rust/vtable.rs @@ -11,7 +11,7 @@ use crate::__runtime::{ RawMessage, }; use crate::{ - AbsentField, FieldEntry, Mut, MutProxy, Optional, PresentField, PrimitiveMut, Proxied, + AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, PrimitiveMut, ProxiedWithPresence, View, ViewProxy, }; use std::fmt::{self, Debug}; @@ -23,7 +23,7 @@ use std::ptr::NonNull; /// /// This vtable should consist of `unsafe fn`s that call thunks that operate on /// `RawMessage`. The structure of this vtable is different per proxied type. -pub trait ProxiedWithRawVTable: Proxied { +pub trait ProxiedWithRawVTable: MutProxied { /// The vtable for get/set access, stored in static memory. type VTable: Debug + 'static; @@ -443,7 +443,7 @@ pub trait PrimitiveWithRawVTable: + ProxiedWithPresence + Sync + Send - + for<'msg> Proxied = Self, Mut<'msg> = PrimitiveMut<'msg, Self>> + + for<'msg> MutProxied = Self, Mut<'msg> = PrimitiveMut<'msg, Self>> { } diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 4184ed0975..5ce270b54f 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -393,6 +393,9 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { impl $pb$::Proxied for $name$ { type View<'a> = $name$; + } + + impl $pb$::MutProxied for $name$ { type Mut<'a> = $pb$::PrimitiveMut<'a, $name$>; } diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 6ee718fc18..5700ebb458 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -865,6 +865,9 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { impl $pb$::Proxied for $Msg$ { type View<'msg> = $Msg$View<'msg>; + } + + impl $pb$::MutProxied for $Msg$ { type Mut<'msg> = $Msg$Mut<'msg>; } From 849b975e5fce3d181aea692b412826f48c38e087 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Wed, 24 Apr 2024 08:20:01 -0700 Subject: [PATCH 030/179] Temporarily use AsRef<{ProtoStr, [u8]}> for string/bytes accessors This change removes the only remaining instance of SettableValue in a generated accessor. The principled fix is to implement IntoProxied for ProtoStr/[u8], but this will have to wait until we have agreed on & implemented the 1.0 string types. So we'll use AsRef in the meantime in order to not break any user code while allowing us to make progress on IntoProxied and Proxied changes. PiperOrigin-RevId: 627735404 --- rust/cpp.rs | 16 ++- rust/string.rs | 103 +++--------------- rust/upb.rs | 16 ++- .../rust/accessors/singular_string.cc | 20 +++- 4 files changed, 46 insertions(+), 109 deletions(-) diff --git a/rust/cpp.rs b/rust/cpp.rs index 225c97e519..9e78c09336 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -171,6 +171,13 @@ impl Deref for SerializedData { } } +// TODO: remove after IntoProxied has been implemented for bytes. +impl AsRef<[u8]> for SerializedData { + fn as_ref(&self) -> &[u8] { + self + } +} + impl Drop for SerializedData { fn drop(&mut self) { // SAFETY: `data` was allocated by the Rust global allocator with a @@ -185,15 +192,6 @@ impl fmt::Debug for SerializedData { } } -impl SettableValue<[u8]> for SerializedData { - fn set_on<'msg>(self, _private: Private, mut mutator: Mut<'msg, [u8]>) - where - [u8]: 'msg, - { - mutator.set(self.as_ref()) - } -} - /// A type to transfer an owned Rust string across the FFI boundary: /// * This struct is ABI-compatible with the equivalent C struct. /// * It owns its data but does not drop it. Immediately turn it into a diff --git a/rust/string.rs b/rust/string.rs index 9377a53514..e54ec44bc0 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -185,53 +185,6 @@ impl<'msg> MutProxy<'msg> for BytesMut<'msg> { } } -impl SettableValue<[u8]> for &'_ [u8] { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, [u8]>) - where - [u8]: 'msg, - { - // SAFETY: this is a `bytes` field with no restriction on UTF-8. - unsafe { mutator.inner.set(self) } - } - - fn set_on_absent( - self, - _private: Private, - absent_mutator: <[u8] as ProxiedWithPresence>::AbsentMutData<'_>, - ) -> <[u8] as ProxiedWithPresence>::PresentMutData<'_> { - // SAFETY: this is a `bytes` field with no restriction on UTF-8. - unsafe { absent_mutator.set(self) } - } - - fn set_on_present( - self, - _private: Private, - present_mutator: <[u8] as ProxiedWithPresence>::PresentMutData<'_>, - ) { - // SAFETY: this is a `bytes` field with no restriction on UTF-8. - unsafe { - present_mutator.set(self); - } - } -} - -impl SettableValue<[u8]> for &'_ [u8; N] { - // forward to `self[..]` - impl_forwarding_settable_value!([u8], self => &self[..]); -} - -impl SettableValue<[u8]> for Vec { - // TODO: Investigate taking ownership of this when allowed by the - // runtime. - impl_forwarding_settable_value!([u8], self => &self[..]); -} - -impl SettableValue<[u8]> for Cow<'_, [u8]> { - // TODO: Investigate taking ownership of this when allowed by the - // runtime. - impl_forwarding_settable_value!([u8], self => &self[..]); -} - impl Hash for BytesMut<'_> { fn hash(&self, state: &mut H) { self.deref().hash(state) @@ -701,50 +654,28 @@ impl<'msg> MutProxy<'msg> for ProtoStrMut<'msg> { } } -impl SettableValue for &'_ ProtoStr { - fn set_on<'b>(self, _private: Private, mutator: Mut<'b, ProtoStr>) - where - ProtoStr: 'b, - { - // SAFETY: A `ProtoStr` has the same UTF-8 validity requirement as the runtime. - unsafe { mutator.bytes.inner.set(self.as_bytes()) } - } - - fn set_on_absent( - self, - _private: Private, - absent_mutator: ::AbsentMutData<'_>, - ) -> ::PresentMutData<'_> { - // SAFETY: A `ProtoStr` has the same UTF-8 validity requirement as the runtime. - StrPresentMutData(unsafe { absent_mutator.0.set(self.as_bytes()) }) - } - - fn set_on_present( - self, - _private: Private, - present_mutator: ::PresentMutData<'_>, - ) { - // SAFETY: A `ProtoStr` has the same UTF-8 validity requirement as the runtime. - unsafe { - present_mutator.0.set(self.as_bytes()); - } +// TODO: remove after IntoProxied has been implemented for +// ProtoStr. +impl AsRef for String { + fn as_ref(&self) -> &ProtoStr { + ProtoStr::from_str(self.as_str()) } } -impl SettableValue for &'_ str { - impl_forwarding_settable_value!(ProtoStr, self => ProtoStr::from_str(self)); -} - -impl SettableValue for String { - // TODO: Investigate taking ownership of this when allowed by the - // runtime. - impl_forwarding_settable_value!(ProtoStr, self => ProtoStr::from_str(&self)); +// TODO: remove after IntoProxied has been implemented for +// ProtoStr. +impl AsRef for &str { + fn as_ref(&self) -> &ProtoStr { + ProtoStr::from_str(self) + } } -impl SettableValue for Cow<'_, str> { - // TODO: Investigate taking ownership of this when allowed by the - // runtime. - impl_forwarding_settable_value!(ProtoStr, self => ProtoStr::from_str(&self)); +// TODO: remove after IntoProxied has been implemented for +// ProtoStr. +impl AsRef for &ProtoStr { + fn as_ref(&self) -> &ProtoStr { + self + } } impl Hash for ProtoStrMut<'_> { diff --git a/rust/upb.rs b/rust/upb.rs index 5a0c07e471..3b9a69ba44 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -97,18 +97,16 @@ impl Deref for SerializedData { } } -impl fmt::Debug for SerializedData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(self.deref(), f) +// TODO: remove after IntoProxied has been implemented for bytes. +impl AsRef<[u8]> for SerializedData { + fn as_ref(&self) -> &[u8] { + self } } -impl SettableValue<[u8]> for SerializedData { - fn set_on<'msg>(self, _private: Private, mut mutator: Mut<'msg, [u8]>) - where - [u8]: 'msg, - { - mutator.set(self.as_ref()) +impl fmt::Debug for SerializedData { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) } } diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index afb3592e75..a9a8b27940 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -80,12 +80,22 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, [&] { if (accessor_case == AccessorCase::VIEW) return; ctx.Emit(R"rs( - pub fn set_$raw_field_name$(&mut self, val: impl $pb$::SettableValue<$proxied_type$>) { - //~ TODO: Optimize this to not go through the - //~ FieldEntry. - self.$raw_field_name$_mut().set(val); + // TODO: Use IntoProxied once string/bytes types support it. + pub fn set_$raw_field_name$(&mut self, val: impl std::convert::AsRef<$proxied_type$>) { + let string_view: $pbr$::PtrAndLen = + $pbr$::copy_bytes_in_arena_if_needed_by_runtime( + self.as_mutator_message_ref(), + val.as_ref().into() + ).into(); + + unsafe { + $setter_thunk$( + self.as_mutator_message_ref().msg(), + string_view + ); } - )rs"); + } + )rs"); }}, {"hazzer", [&] { From 55875598f734797804da23c499ba79ca242ef59d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Apr 2024 10:39:37 -0700 Subject: [PATCH 031/179] Add a collision-avoidance behavior to the Rust Proto codegen. The intent is to avoid codegen issues for simple cases where a message of the shape: message M { int32 x = 3 string set_x = 8; } Which would otherwise break due to the first field having a setter whose name collides with the second field's getter. By seeing that 'set_x' matches another field with a common accessor prefixed, it will generate all accessors for that field as though it was named `set_x_8`. This does not avoid all possible collisions, but should mitigate the vast majority of situations of an accidental collision. PiperOrigin-RevId: 627776907 --- rust/test/BUILD | 12 +++---- rust/test/{reserved.proto => bad_names.proto} | 20 +++++++++-- rust/test/shared/BUILD | 26 +++++++------- .../{reserved_test.rs => bad_names_test.rs} | 14 ++++++-- .../protobuf/compiler/rust/accessors/map.cc | 3 +- .../compiler/rust/accessors/repeated_field.cc | 3 +- .../rust/accessors/singular_message.cc | 5 +-- .../rust/accessors/singular_scalar.cc | 5 +-- .../rust/accessors/singular_string.cc | 7 ++-- src/google/protobuf/compiler/rust/naming.cc | 34 +++++++++++++++++++ src/google/protobuf/compiler/rust/naming.h | 19 +++++++++++ src/google/protobuf/compiler/rust/oneof.cc | 3 +- 12 files changed, 119 insertions(+), 32 deletions(-) rename rust/test/{reserved.proto => bad_names.proto} (53%) rename rust/test/shared/{reserved_test.rs => bad_names_test.rs} (68%) diff --git a/rust/test/BUILD b/rust/test/BUILD index d0e86bb03f..3b05958c8a 100644 --- a/rust/test/BUILD +++ b/rust/test/BUILD @@ -342,23 +342,23 @@ rust_cc_proto_library( ) proto_library( - name = "reserved_proto", + name = "bad_names_proto", testonly = True, - srcs = ["reserved.proto"], + srcs = ["bad_names.proto"], ) rust_cc_proto_library( - name = "reserved_cc_rust_proto", + name = "bad_names_cc_rust_proto", testonly = True, visibility = ["//rust/test/shared:__subpackages__"], - deps = [":reserved_proto"], + deps = [":bad_names_proto"], ) rust_upb_proto_library( - name = "reserved_upb_rust_proto", + name = "bad_names_upb_rust_proto", testonly = True, visibility = ["//rust/test/shared:__subpackages__"], - deps = [":reserved_proto"], + deps = [":bad_names_proto"], ) proto_library( diff --git a/rust/test/reserved.proto b/rust/test/bad_names.proto similarity index 53% rename from rust/test/reserved.proto rename to rust/test/bad_names.proto index 1f8c2a937d..21fbf3b7cb 100644 --- a/rust/test/reserved.proto +++ b/rust/test/bad_names.proto @@ -6,8 +6,10 @@ // https://developers.google.com/open-source/licenses/bsd // LINT: LEGACY_NAMES -// The purpose of this file is to be as hostile as possible to reserved words -// to the Rust language and ensure it still works. + +// The purpose of this file is to be hostile on field/message/enum naming and +// ensure that it works (e.g. collisions between names and language keywords, +// collisions between two different field's accessor's names). syntax = "proto2"; @@ -36,3 +38,17 @@ message Ref { .type.type.Pub.Self const = 3; } } + +// A message where the accessors would collide that should still work. Note that +// not all collisions problems are avoided, not least because C++ Proto does not +// avoid all possible collisions (eg a field `x` and `clear_x` will often not +// compile on C++). +message AccessorsCollide { + message X {} + message SetX {} + optional SetX set_x = 2; + optional X x = 3; + oneof o { + bool x_mut = 5; + } +} diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index 7210cc2374..460542bf46 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -159,24 +159,26 @@ rust_test( ) rust_test( - name = "reserved_cpp_test", - srcs = ["reserved_test.rs"], + name = "bad_names_cpp_test", + srcs = ["bad_names_test.rs"], deps = [ - "//rust/test:reserved_cc_rust_proto", + "//rust/test:bad_names_cc_rust_proto", "//rust/test:unittest_cc_rust_proto", "@crate_index//:googletest", ], ) -rust_test( - name = "reserved_upb_test", - srcs = ["reserved_test.rs"], - deps = [ - "//rust/test:reserved_upb_rust_proto", - "//rust/test:unittest_upb_rust_proto", - "@crate_index//:googletest", - ], -) +# TODO: This test currently fails on upb due to the thunk names not correctly matching +# the upb C codegen collision avoidance. +# rust_test( +# name = "bad_names_upb_test", +# srcs = ["bad_names_test.rs"], +# deps = [ +# "@crate_index//:googletest", +# "//rust/test:bad_names_upb_rust_proto", +# "//rust/test:unittest_upb_rust_proto", +# ], +# ) rust_test( name = "nested_types_cpp_test", diff --git a/rust/test/shared/reserved_test.rs b/rust/test/shared/bad_names_test.rs similarity index 68% rename from rust/test/shared/reserved_test.rs rename to rust/test/shared/bad_names_test.rs index 0d2cb52bb6..a4b133b92f 100644 --- a/rust/test/shared/reserved_test.rs +++ b/rust/test/shared/bad_names_test.rs @@ -5,10 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -/// Test covering proto compilation with reserved words. +use bad_names_proto::*; use googletest::prelude::*; -use reserved_proto::Self__mangled_because_ident_isnt_a_legal_raw_identifier; -use reserved_proto::{r#enum, Ref}; #[test] fn test_reserved_keyword_in_accessors() { @@ -22,3 +20,13 @@ fn test_reserved_keyword_in_messages() { let _ = r#enum::new(); let _ = Ref::new().r#const(); } + +#[test] +fn test_collision_in_accessors() { + let mut m = AccessorsCollide::new(); + m.set_x_mut_5(false); + assert_that!(m.x_mut_5(), eq(false)); + assert_that!(m.has_x_mut_5(), eq(true)); + assert_that!(m.has_x(), eq(false)); + assert_that!(m.has_set_x_2(), eq(false)); +} diff --git a/src/google/protobuf/compiler/rust/accessors/map.cc b/src/google/protobuf/compiler/rust/accessors/map.cc index ee0d089784..75aa5007ee 100644 --- a/src/google/protobuf/compiler/rust/accessors/map.cc +++ b/src/google/protobuf/compiler/rust/accessors/map.cc @@ -39,8 +39,9 @@ void Map::InMsgImpl(Context& ctx, const FieldDescriptor& field, AccessorCase accessor_case) const { auto& key_type = *field.message_type()->map_key(); auto& value_type = *field.message_type()->map_value(); + std::string field_name = FieldNameWithCollisionAvoidance(field); - ctx.Emit({{"field", RsSafeName(field.name())}, + ctx.Emit({{"field", RsSafeName(field_name)}, {"Key", RsTypePath(ctx, key_type)}, {"Value", RsTypePath(ctx, value_type)}, {"view_lifetime", ViewLifetime(accessor_case)}, diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index c48b437847..e1e14e646f 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -22,7 +22,8 @@ namespace rust { void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, AccessorCase accessor_case) const { - ctx.Emit({{"field", RsSafeName(field.name())}, + std::string field_name = FieldNameWithCollisionAvoidance(field); + ctx.Emit({{"field", RsSafeName(field_name)}, {"RsType", RsTypePath(ctx, field)}, {"view_lifetime", ViewLifetime(accessor_case)}, {"view_self", ViewReceiver(accessor_case)}, diff --git a/src/google/protobuf/compiler/rust/accessors/singular_message.cc b/src/google/protobuf/compiler/rust/accessors/singular_message.cc index 8f3ea9bfe0..73acd60164 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_message.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_message.cc @@ -24,9 +24,10 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, AccessorCase accessor_case) const { // fully qualified message name with modules prefixed std::string msg_type = RsTypePath(ctx, field); + std::string field_name = FieldNameWithCollisionAvoidance(field); ctx.Emit({{"msg_type", msg_type}, - {"field", RsSafeName(field.name())}, - {"raw_field_name", field.name()}, + {"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, {"view_lifetime", ViewLifetime(accessor_case)}, {"view_self", ViewReceiver(accessor_case)}, {"getter_thunk", ThunkName(ctx, field, "get")}, diff --git a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc index bf8bf622d3..2c3546cbd9 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc @@ -23,10 +23,11 @@ namespace rust { void SingularScalar::InMsgImpl(Context& ctx, const FieldDescriptor& field, AccessorCase accessor_case) const { + std::string field_name = FieldNameWithCollisionAvoidance(field); ctx.Emit( { - {"field", RsSafeName(field.name())}, - {"raw_field_name", field.name()}, // Never r# prefixed + {"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, // Never r# prefixed {"view_self", ViewReceiver(accessor_case)}, {"Scalar", RsTypePath(ctx, field)}, {"hazzer_thunk", ThunkName(ctx, field, "has")}, diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index a9a8b27940..1de8f5b7ae 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -5,6 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#include + #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" @@ -21,10 +23,11 @@ namespace rust { void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, AccessorCase accessor_case) const { + std::string field_name = FieldNameWithCollisionAvoidance(field); ctx.Emit( { - {"field", RsSafeName(field.name())}, - {"raw_field_name", field.name()}, + {"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, {"hazzer_thunk", ThunkName(ctx, field, "has")}, {"getter_thunk", ThunkName(ctx, field, "get")}, {"setter_thunk", ThunkName(ctx, field, "set")}, diff --git a/src/google/protobuf/compiler/rust/naming.cc b/src/google/protobuf/compiler/rust/naming.cc index 6879e4ae45..78d96b8393 100644 --- a/src/google/protobuf/compiler/rust/naming.cc +++ b/src/google/protobuf/compiler/rust/naming.cc @@ -270,6 +270,40 @@ std::string FieldInfoComment(Context& ctx, const FieldDescriptor& field) { return comment; } +static constexpr absl::string_view kAccessorPrefixes[] = {"clear_", "has_", + "set_"}; + +static constexpr absl::string_view kAccessorSuffixes[] = {"_mut", "_opt"}; + +std::string FieldNameWithCollisionAvoidance(const FieldDescriptor& field) { + absl::string_view name = field.name(); + const Descriptor& msg = *field.containing_type(); + + for (absl::string_view prefix : kAccessorPrefixes) { + if (absl::StartsWith(name, prefix)) { + absl::string_view without_prefix = name; + without_prefix.remove_prefix(prefix.size()); + + if (msg.FindFieldByName(without_prefix) != nullptr) { + return absl::StrCat(name, "_", field.number()); + } + } + } + + for (absl::string_view suffix : kAccessorSuffixes) { + if (absl::EndsWith(name, suffix)) { + absl::string_view without_suffix = name; + without_suffix.remove_suffix(suffix.size()); + + if (msg.FindFieldByName(without_suffix) != nullptr) { + return absl::StrCat(name, "_", field.number()); + } + } + } + + return std::string(name); +} + std::string RsSafeName(absl::string_view name) { if (!IsLegalRawIdentifierName(name)) { return absl::StrCat(name, diff --git a/src/google/protobuf/compiler/rust/naming.h b/src/google/protobuf/compiler/rust/naming.h index f3d7af797e..70eebe1b64 100644 --- a/src/google/protobuf/compiler/rust/naming.h +++ b/src/google/protobuf/compiler/rust/naming.h @@ -55,6 +55,25 @@ std::string OneofCaseRsName(const FieldDescriptor& oneof_field); std::string FieldInfoComment(Context& ctx, const FieldDescriptor& field); +// Return how to name a field with 'collision avoidance'. This adds a suffix +// of the field number to the field name if it appears that it will collide with +// another field's non-getter accessor. +// +// For example, for the message: +// message M { bool set_x = 1; int32 x = 2; string x_mut = 8; } +// All accessors for the field `set_x` will be constructed as though the field +// was instead named `set_x_1`, and all accessors for `x_mut` will be as though +// the field was instead named `x_mut_8`. +// +// This is a best-effort heuristic to avoid realistic accidental +// collisions. It is still possible to create a message definition that will +// have a collision, and it may rename a field even if there's no collision (as +// in the case of x_mut in the example). +// +// Note the returned name may still be a rust keyword: RsSafeName() should +// additionally be used if there is no prefix/suffix being appended to the name. +std::string FieldNameWithCollisionAvoidance(const FieldDescriptor& field); + // Returns how to 'spell' the provided name in Rust, which is the provided name // verbatim unless it is a Rust keyword that isn't a legal symbol name. std::string RsSafeName(absl::string_view name); diff --git a/src/google/protobuf/compiler/rust/oneof.cc b/src/google/protobuf/compiler/rust/oneof.cc index 8453af898b..fc8cef2f18 100644 --- a/src/google/protobuf/compiler/rust/oneof.cc +++ b/src/google/protobuf/compiler/rust/oneof.cc @@ -193,10 +193,11 @@ void GenerateOneofAccessors(Context& ctx, const OneofDescriptor& oneof, if (rs_type.empty()) { continue; } + std::string field_name = FieldNameWithCollisionAvoidance(field); ctx.Emit( { {"case", OneofCaseRsName(field)}, - {"rs_getter", RsSafeName(field.name())}, + {"rs_getter", RsSafeName(field_name)}, {"type", rs_type}, }, R"rs( From 8be1312b3c1da1960d2ec7181a253b1819b3e842 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Wed, 24 Apr 2024 11:26:51 -0700 Subject: [PATCH 032/179] upb: fix Dart empty imports PiperOrigin-RevId: 627794151 --- upb/reflection/def.hpp | 1 + upb/reflection/field_def.c | 11 +++++++---- upb/reflection/field_def.h | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/upb/reflection/def.hpp b/upb/reflection/def.hpp index 206f5db744..cf28a38ff9 100644 --- a/upb/reflection/def.hpp +++ b/upb/reflection/def.hpp @@ -105,6 +105,7 @@ class FieldDefPtr { OneofDefPtr real_containing_oneof() const; // Convenient field type tests. + bool IsEnum() const { return upb_FieldDef_IsEnum(ptr_); } bool IsSubMessage() const { return upb_FieldDef_IsSubMessage(ptr_); } bool IsString() const { return upb_FieldDef_IsString(ptr_); } bool IsSequence() const { return upb_FieldDef_IsRepeated(ptr_); } diff --git a/upb/reflection/field_def.c b/upb/reflection/field_def.c index 98fa37092b..d5597f4b2e 100644 --- a/upb/reflection/field_def.c +++ b/upb/reflection/field_def.c @@ -208,11 +208,11 @@ upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f) { } const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f) { - return upb_FieldDef_CType(f) == kUpb_CType_Message ? f->sub.msgdef : NULL; + return upb_FieldDef_IsSubMessage(f) ? f->sub.msgdef : NULL; } const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f) { - return upb_FieldDef_CType(f) == kUpb_CType_Enum ? f->sub.enumdef : NULL; + return upb_FieldDef_IsEnum(f) ? f->sub.enumdef : NULL; } const upb_MiniTableField* upb_FieldDef_MiniTable(const upb_FieldDef* f) { @@ -310,8 +310,11 @@ bool upb_FieldDef_HasDefault(const upb_FieldDef* f) { return f->has_default; } bool upb_FieldDef_HasPresence(const upb_FieldDef* f) { return f->has_presence; } bool upb_FieldDef_HasSubDef(const upb_FieldDef* f) { - return upb_FieldDef_IsSubMessage(f) || - upb_FieldDef_CType(f) == kUpb_CType_Enum; + return upb_FieldDef_IsSubMessage(f) || upb_FieldDef_IsEnum(f); +} + +bool upb_FieldDef_IsEnum(const upb_FieldDef* f) { + return upb_FieldDef_CType(f) == kUpb_CType_Enum; } bool upb_FieldDef_IsMap(const upb_FieldDef* f) { diff --git a/upb/reflection/field_def.h b/upb/reflection/field_def.h index a4019a0aa0..98f4a875d0 100644 --- a/upb/reflection/field_def.h +++ b/upb/reflection/field_def.h @@ -45,6 +45,7 @@ bool upb_FieldDef_HasOptions(const upb_FieldDef* f); UPB_API bool upb_FieldDef_HasPresence(const upb_FieldDef* f); bool upb_FieldDef_HasSubDef(const upb_FieldDef* f); uint32_t upb_FieldDef_Index(const upb_FieldDef* f); +UPB_API bool upb_FieldDef_IsEnum(const upb_FieldDef* f); bool upb_FieldDef_IsExtension(const upb_FieldDef* f); UPB_API bool upb_FieldDef_IsMap(const upb_FieldDef* f); bool upb_FieldDef_IsOptional(const upb_FieldDef* f); From 9cc5be12edd35ed51949427316fe4dbae2709852 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Wed, 24 Apr 2024 11:27:29 -0700 Subject: [PATCH 033/179] Catch all the exceptions in python JSON ParseDict and raise json_format.ParseError PiperOrigin-RevId: 627794381 --- .../protobuf/internal/json_format_test.py | 8 +++++++- python/google/protobuf/json_format.py | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index 6c5a5e32c8..965557eb1e 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py @@ -1439,7 +1439,7 @@ class JsonFormatTest(JsonFormatBase): def testInvalidAny(self): message = any_pb2.Any() text = '{"@type": "type.googleapis.com/google.protobuf.Int32Value"}' - self.assertRaisesRegex(KeyError, 'value', json_format.Parse, text, message) + self.assertRaisesRegex(json_format.ParseError, 'KeyError: \'value\'', json_format.Parse, text, message) text = '{"value": 1234}' self.assertRaisesRegex( json_format.ParseError, @@ -1662,6 +1662,12 @@ class JsonFormatTest(JsonFormatBase): json_format.Parse(json_string, new_parsed_message) self.assertEqual(new_message, new_parsed_message) + def testOtherParseErrors(self): + self.CheckError( + '9', + "Failed to parse JSON: TypeError: 'int' object is not iterable.", + ) + if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index 8a2e24975b..2a6bba9391 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py @@ -451,13 +451,22 @@ def Parse( """ if not isinstance(text, str): text = text.decode('utf-8') + try: js = json.loads(text, object_pairs_hook=_DuplicateChecker) - except ValueError as e: + except Exception as e: raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e - return ParseDict( - js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth - ) + + try: + return ParseDict( + js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth + ) + except ParseError as e: + raise e + except Exception as e: + raise ParseError( + 'Failed to parse JSON: {0}: {1}.'.format(type(e).__name__, str(e)) + ) from e def ParseDict( From ef020875349635ba2bdefe4ef294dcee54500766 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Apr 2024 18:42:18 +0000 Subject: [PATCH 034/179] Auto-generate files after cl/627794151 --- php/ext/google/protobuf/php-upb.c | 11 +++++++---- php/ext/google/protobuf/php-upb.h | 1 + ruby/ext/google/protobuf_c/ruby-upb.c | 11 +++++++---- ruby/ext/google/protobuf_c/ruby-upb.h | 1 + 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 7b7806928e..3879fa0201 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -13696,11 +13696,11 @@ upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f) { } const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f) { - return upb_FieldDef_CType(f) == kUpb_CType_Message ? f->sub.msgdef : NULL; + return upb_FieldDef_IsSubMessage(f) ? f->sub.msgdef : NULL; } const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f) { - return upb_FieldDef_CType(f) == kUpb_CType_Enum ? f->sub.enumdef : NULL; + return upb_FieldDef_IsEnum(f) ? f->sub.enumdef : NULL; } const upb_MiniTableField* upb_FieldDef_MiniTable(const upb_FieldDef* f) { @@ -13798,8 +13798,11 @@ bool upb_FieldDef_HasDefault(const upb_FieldDef* f) { return f->has_default; } bool upb_FieldDef_HasPresence(const upb_FieldDef* f) { return f->has_presence; } bool upb_FieldDef_HasSubDef(const upb_FieldDef* f) { - return upb_FieldDef_IsSubMessage(f) || - upb_FieldDef_CType(f) == kUpb_CType_Enum; + return upb_FieldDef_IsSubMessage(f) || upb_FieldDef_IsEnum(f); +} + +bool upb_FieldDef_IsEnum(const upb_FieldDef* f) { + return upb_FieldDef_CType(f) == kUpb_CType_Enum; } bool upb_FieldDef_IsMap(const upb_FieldDef* f) { diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index fa63a63297..b97bd1db2b 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -11469,6 +11469,7 @@ bool upb_FieldDef_HasOptions(const upb_FieldDef* f); UPB_API bool upb_FieldDef_HasPresence(const upb_FieldDef* f); bool upb_FieldDef_HasSubDef(const upb_FieldDef* f); uint32_t upb_FieldDef_Index(const upb_FieldDef* f); +UPB_API bool upb_FieldDef_IsEnum(const upb_FieldDef* f); bool upb_FieldDef_IsExtension(const upb_FieldDef* f); UPB_API bool upb_FieldDef_IsMap(const upb_FieldDef* f); bool upb_FieldDef_IsOptional(const upb_FieldDef* f); diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index d90b3eea1d..a426daf84f 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -13189,11 +13189,11 @@ upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f) { } const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f) { - return upb_FieldDef_CType(f) == kUpb_CType_Message ? f->sub.msgdef : NULL; + return upb_FieldDef_IsSubMessage(f) ? f->sub.msgdef : NULL; } const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f) { - return upb_FieldDef_CType(f) == kUpb_CType_Enum ? f->sub.enumdef : NULL; + return upb_FieldDef_IsEnum(f) ? f->sub.enumdef : NULL; } const upb_MiniTableField* upb_FieldDef_MiniTable(const upb_FieldDef* f) { @@ -13291,8 +13291,11 @@ bool upb_FieldDef_HasDefault(const upb_FieldDef* f) { return f->has_default; } bool upb_FieldDef_HasPresence(const upb_FieldDef* f) { return f->has_presence; } bool upb_FieldDef_HasSubDef(const upb_FieldDef* f) { - return upb_FieldDef_IsSubMessage(f) || - upb_FieldDef_CType(f) == kUpb_CType_Enum; + return upb_FieldDef_IsSubMessage(f) || upb_FieldDef_IsEnum(f); +} + +bool upb_FieldDef_IsEnum(const upb_FieldDef* f) { + return upb_FieldDef_CType(f) == kUpb_CType_Enum; } bool upb_FieldDef_IsMap(const upb_FieldDef* f) { diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 76b6af3e2b..7307f5e252 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -11871,6 +11871,7 @@ bool upb_FieldDef_HasOptions(const upb_FieldDef* f); UPB_API bool upb_FieldDef_HasPresence(const upb_FieldDef* f); bool upb_FieldDef_HasSubDef(const upb_FieldDef* f); uint32_t upb_FieldDef_Index(const upb_FieldDef* f); +UPB_API bool upb_FieldDef_IsEnum(const upb_FieldDef* f); bool upb_FieldDef_IsExtension(const upb_FieldDef* f); UPB_API bool upb_FieldDef_IsMap(const upb_FieldDef* f); bool upb_FieldDef_IsOptional(const upb_FieldDef* f); From 734729afc2ace9dc33e41b91a34fd16bb53594d9 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 24 Apr 2024 12:25:37 -0700 Subject: [PATCH 035/179] Create the concept of 'owned data' in upb/rust as a generalization of the upb.rs SerializedData (which is a arena + data for arbitrary types, both thin and wide ref types), use that for the wire parse/serialize path. PiperOrigin-RevId: 627814154 --- rust/upb.rs | 68 +----------- rust/upb/BUILD | 1 + rust/upb/arena.rs | 49 +++++++- rust/upb/lib.rs | 11 +- rust/upb/message.rs | 4 + rust/upb/owned_arena_box.rs | 111 +++++++++++++++++++ rust/upb/wire.rs | 66 ++++++++++- src/google/protobuf/compiler/rust/message.cc | 40 ++----- 8 files changed, 246 insertions(+), 104 deletions(-) create mode 100644 rust/upb/owned_arena_box.rs diff --git a/rust/upb.rs b/rust/upb.rs index 3b9a69ba44..e697a27bf8 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -14,9 +14,7 @@ use crate::{ }; use core::fmt::Debug; use std::alloc::Layout; -use std::fmt; use std::mem::{size_of, ManuallyDrop, MaybeUninit}; -use std::ops::Deref; use std::ptr::{self, NonNull}; use std::slice; use std::sync::OnceLock; @@ -60,55 +58,7 @@ impl ScratchSpace { } } -/// Serialized Protobuf wire format data. -/// -/// It's typically produced by `::serialize()`. -pub struct SerializedData { - data: NonNull, - len: usize, - - // The arena that owns `data`. - _arena: Arena, -} - -impl SerializedData { - /// Construct `SerializedData` from raw pointers and its owning arena. - /// - /// # Safety - /// - `arena` must be have allocated `data` - /// - `data` must be readable for `len` bytes and not mutate while this - /// struct exists - pub unsafe fn from_raw_parts(arena: Arena, data: NonNull, len: usize) -> Self { - SerializedData { _arena: arena, data, len } - } - - /// Gets a raw slice pointer. - pub fn as_ptr(&self) -> *const [u8] { - ptr::slice_from_raw_parts(self.data.as_ptr(), self.len) - } -} - -impl Deref for SerializedData { - type Target = [u8]; - fn deref(&self) -> &Self::Target { - // SAFETY: `data` is valid for `len` bytes as promised by - // the caller of `SerializedData::from_raw_parts`. - unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) } - } -} - -// TODO: remove after IntoProxied has been implemented for bytes. -impl AsRef<[u8]> for SerializedData { - fn as_ref(&self) -> &[u8] { - self - } -} - -impl fmt::Debug for SerializedData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(self.deref(), f) - } -} +pub type SerializedData = upb::OwnedArenaBox<[u8]>; // TODO: Investigate replacing this with direct access to UPB bits. pub type MessagePresentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutatorData<'msg, T>; @@ -812,22 +762,6 @@ mod tests { use super::*; use googletest::prelude::*; - #[test] - fn test_serialized_data_roundtrip() { - let arena = Arena::new(); - let original_data = b"Hello world"; - let len = original_data.len(); - - let serialized_data = unsafe { - SerializedData::from_raw_parts( - arena, - NonNull::new(original_data as *const _ as *mut _).unwrap(), - len, - ) - }; - assert_that!(&*serialized_data, eq(b"Hello world")); - } - #[test] fn assert_c_type_sizes() { // TODO: add these same asserts in C++. diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 894d4950e0..13cea4d7f5 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -23,6 +23,7 @@ rust_library( "message_value.rs", "mini_table.rs", "opaque_pointee.rs", + "owned_arena_box.rs", "string_view.rs", "wire.rs", ], diff --git a/rust/upb/arena.rs b/rust/upb/arena.rs index 8610d4de56..281ea2f83c 100644 --- a/rust/upb/arena.rs +++ b/rust/upb/arena.rs @@ -3,7 +3,7 @@ use std::alloc::{self, Layout}; use std::cell::UnsafeCell; use std::marker::PhantomData; use std::mem::{align_of, MaybeUninit}; -use std::ptr::NonNull; +use std::ptr::{self, NonNull}; use std::slice; opaque_pointee!(upb_Arena); @@ -95,6 +95,53 @@ impl Arena { // `UPB_MALLOC_ALIGN` boundary. unsafe { slice::from_raw_parts_mut(ptr.cast(), layout.size()) } } + + /// Same as alloc() but panics if `layout.align() > UPB_MALLOC_ALIGN`. + #[allow(clippy::mut_from_ref)] + #[inline] + pub fn checked_alloc(&self, layout: Layout) -> &mut [MaybeUninit] { + assert!(layout.align() <= UPB_MALLOC_ALIGN); + // SAFETY: layout.align() <= UPB_MALLOC_ALIGN asserted. + unsafe { self.alloc(layout) } + } + + /// Copies the T into this arena and returns a pointer to the T data inside + /// the arena. + pub fn copy_in<'a, T: Copy>(&'a self, data: &T) -> &'a T { + let layout = Layout::for_value(data); + let alloc = self.checked_alloc(layout); + + // SAFETY: + // - alloc is valid for `layout.len()` bytes and is the uninit bytes are written + // to not read from until written. + // - T is copy so copying the bytes of the value is sound. + unsafe { + let alloc = alloc.as_mut_ptr().cast::>(); + // let data = (data as *const T).cast::>(); + (*alloc).write(*data) + } + } + + pub fn copy_str_in<'a>(&'a self, s: &str) -> &'a str { + let copied_bytes = self.copy_slice_in(s.as_bytes()); + // SAFETY: `copied_bytes` has same contents as `s` and so must meet &str + // criteria. + unsafe { std::str::from_utf8_unchecked(copied_bytes) } + } + + pub fn copy_slice_in<'a, T: Copy>(&'a self, data: &[T]) -> &'a [T] { + let layout = Layout::for_value(data); + let alloc: *mut T = self.checked_alloc(layout).as_mut_ptr().cast(); + + // SAFETY: + // - uninit_alloc is valid for `layout.len()` bytes and is the uninit bytes are + // written to not read from until written. + // - T is copy so copying the bytes of the values is sound. + unsafe { + ptr::copy_nonoverlapping(data.as_ptr(), alloc, data.len()); + slice::from_raw_parts_mut(alloc, data.len()) + } + } } impl Default for Arena { diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index f557d1b024..040d98d736 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -21,7 +21,9 @@ pub use map::{ }; mod message; -pub use message::{upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, RawMessage}; +pub use message::{ + upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_New, RawMessage, +}; mod message_value; pub use message_value::{upb_MessageValue, upb_MutableMessageValue}; @@ -31,8 +33,11 @@ pub use mini_table::{upb_MiniTable, RawMiniTable}; mod opaque_pointee; +mod owned_arena_box; +pub use owned_arena_box::OwnedArenaBox; + mod string_view; pub use string_view::StringView; -mod wire; -pub use wire::{upb_Decode, upb_Encode, DecodeStatus, EncodeStatus}; +pub mod wire; +pub use wire::{upb_Decode, DecodeStatus, EncodeStatus}; diff --git a/rust/upb/message.rs b/rust/upb/message.rs index 2a9fe91e30..fd831d7b6d 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -6,6 +6,10 @@ opaque_pointee!(upb_Message); pub type RawMessage = NonNull; extern "C" { + /// SAFETY: No constraints. + pub fn upb_Message_New(mini_table: *const upb_MiniTable, arena: RawArena) + -> Option; + pub fn upb_Message_DeepCopy( dst: RawMessage, src: RawMessage, diff --git a/rust/upb/owned_arena_box.rs b/rust/upb/owned_arena_box.rs new file mode 100644 index 0000000000..9f10accc49 --- /dev/null +++ b/rust/upb/owned_arena_box.rs @@ -0,0 +1,111 @@ +use crate::Arena; +use std::fmt::{self, Debug}; +use std::ops::{Deref, DerefMut}; +use std::ptr::NonNull; + +/// An 'owned' T, similar to a Box where the T is data +/// held in a upb Arena. By holding the data pointer and a corresponding arena +/// together the data liveness is be maintained. +/// +/// This struct is conceptually self-referential, where `data` points at memory +/// inside `arena`. This avoids typical concerns of self-referential data +/// structures because `arena` modifications (other than drop) will never +/// invalidate `data`, and `data` and `arena` are both behind indirections which +/// avoids any concern with std::mem::swap. +pub struct OwnedArenaBox { + data: NonNull, + arena: Arena, +} + +impl OwnedArenaBox { + /// Construct `OwnedArenaBox` from raw pointers and its owning arena. + /// + /// # Safety + /// - `data` must satisfy the safety constraints of pointer::as_mut::<'a>() + /// where 'a is the passed arena's lifetime (`data` should be valid and + /// not mutated while this struct is live). + /// - `data` should be a pointer into a block from a previous allocation on + /// `arena`, or to another arena fused to it, or should be pointing at + /// 'static data (and if it is pointing at any struct like upb_Message, + /// all data transitively reachable should similarly be kept live by + /// `arena` or be 'static). + pub unsafe fn new(data: NonNull, arena: Arena) -> Self { + OwnedArenaBox { arena, data } + } + + pub fn data(&self) -> *const T { + self.data.as_ptr() + } + + pub fn into_parts(self) -> (NonNull, Arena) { + (self.data, self.arena) + } +} + +impl Deref for OwnedArenaBox { + type Target = T; + fn deref(&self) -> &Self::Target { + self.as_ref() + } +} + +impl DerefMut for OwnedArenaBox { + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_mut() + } +} + +impl AsRef for OwnedArenaBox { + fn as_ref(&self) -> &T { + // SAFETY: + // - `data` is valid under the conditions set on ::new(). + unsafe { self.data.as_ref() } + } +} + +impl AsMut for OwnedArenaBox { + fn as_mut(&mut self) -> &mut T { + // SAFETY: + // - `data` is valid under the conditions set on ::new(). + unsafe { self.data.as_mut() } + } +} + +impl Debug for OwnedArenaBox { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_tuple("OwnedArenaBox").field(self.deref()).finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str; + + #[test] + fn test_byte_slice_pointer_roundtrip() { + let arena = Arena::new(); + let original_data: &'static [u8] = b"Hello world"; + let owned_data = unsafe { OwnedArenaBox::new(original_data.into(), arena) }; + assert_eq!(&*owned_data, b"Hello world"); + } + + #[test] + fn test_alloc_str_roundtrip() { + let arena = Arena::new(); + let s: &str = "Hello"; + let arena_alloc_str: NonNull = arena.copy_str_in(s).into(); + let owned_data = unsafe { OwnedArenaBox::new(arena_alloc_str, arena) }; + assert_eq!(&*owned_data, s); + } + + #[test] + fn test_sized_type_roundtrip() { + let arena = Arena::new(); + let arena_alloc_u32: NonNull = arena.copy_in(&7u32).into(); + let mut owned_data = unsafe { OwnedArenaBox::new(arena_alloc_u32, arena) }; + assert_eq!(*owned_data, 7); + *owned_data = 8; + assert_eq!(*owned_data, 8); + } +} diff --git a/rust/upb/wire.rs b/rust/upb/wire.rs index 2b68145cc2..aa335cc020 100644 --- a/rust/upb/wire.rs +++ b/rust/upb/wire.rs @@ -1,8 +1,9 @@ -use crate::{upb_ExtensionRegistry, upb_MiniTable, RawArena, RawMessage}; +use crate::{upb_ExtensionRegistry, upb_MiniTable, Arena, OwnedArenaBox, RawArena, RawMessage}; +use std::ptr::NonNull; // LINT.IfChange(encode_status) #[repr(C)] -#[derive(PartialEq, Eq, Copy, Clone)] +#[derive(PartialEq, Eq, Copy, Clone, Debug)] pub enum EncodeStatus { Ok = 0, OutOfMemory = 1, @@ -13,7 +14,7 @@ pub enum EncodeStatus { // LINT.IfChange(decode_status) #[repr(C)] -#[derive(PartialEq, Eq, Copy, Clone)] +#[derive(PartialEq, Eq, Copy, Clone, Debug)] pub enum DecodeStatus { Ok = 0, Malformed = 1, @@ -25,7 +26,62 @@ pub enum DecodeStatus { } // LINT.ThenChange() +/// If Err, then EncodeStatus != Ok. +/// +/// SAFETY: +/// - `msg` must be associated with `mini_table`. +pub unsafe fn encode( + msg: RawMessage, + mini_table: *const upb_MiniTable, +) -> Result, EncodeStatus> { + let arena = Arena::new(); + let mut buf: *mut u8 = std::ptr::null_mut(); + let mut len = 0usize; + + // SAFETY: + // - `mini_table` is the one associated with `msg`. + // - `buf` and `buf_size` are legally writable. + let status = upb_Encode(msg, mini_table, 0, arena.raw(), &mut buf, &mut len); + + if status == EncodeStatus::Ok { + assert!(!buf.is_null()); // EncodeStatus Ok should never return NULL data, even for len=0. + // SAFETY: upb guarantees that `buf` is valid to read for `len`. + let slice = NonNull::new_unchecked(std::ptr::slice_from_raw_parts_mut(buf, len)); + Ok(OwnedArenaBox::new(slice, arena)) + } else { + Err(status) + } +} + +/// Decodes into the provided message (merge semantics). If Err, then +/// DecodeStatus != Ok. +/// +/// SAFETY: +/// - `msg` must be mutable. +/// - `msg` must be associated with `mini_table`. +pub unsafe fn decode( + buf: &[u8], + msg: RawMessage, + mini_table: *const upb_MiniTable, + arena: &Arena, +) -> Result<(), DecodeStatus> { + let len = buf.len(); + let buf = buf.as_ptr(); + // SAFETY: + // - `mini_table` is the one associated with `msg` + // - `buf` is legally readable for at least `buf_size` bytes. + // - `extreg` is null. + let status = upb_Decode(buf, len, msg, mini_table, std::ptr::null(), 0, arena.raw()); + match status { + DecodeStatus::Ok => Ok(()), + _ => Err(status), + } +} + extern "C" { + // SAFETY: + // - `mini_table` is the one associated with `msg` + // - `buf` and `buf_size` are legally writable. pub fn upb_Encode( msg: RawMessage, mini_table: *const upb_MiniTable, @@ -35,6 +91,10 @@ extern "C" { buf_size: *mut usize, ) -> EncodeStatus; + // SAFETY: + // - `mini_table` is the one associated with `msg` + // - `buf` is legally readable for at least `buf_size` bytes. + // - `extreg` is either null or points at a valid upb_ExtensionRegistry. pub fn upb_Decode( buf: *const u8, buf_size: usize, diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 5700ebb458..5ccf2541ac 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -68,35 +68,17 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( - let arena = $pbr$::Arena::new(); // SAFETY: $minitable$ is a static of a const object. let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; - let options = 0; - let mut buf: *mut u8 = std::ptr::null_mut(); - let mut len = 0; - - // SAFETY: `mini_table` is the corresponding one that was used to - // construct `self.raw_msg()`. - let status = unsafe { - $pbr$::upb_Encode(self.raw_msg(), mini_table, options, arena.raw(), - &mut buf, &mut len) + // SAFETY: $minitable$ is the one associated with raw_msg(). + let encoded = unsafe { + $pbr$::wire::encode(self.raw_msg(), mini_table) }; //~ TODO: Currently serialize() on the Rust API is an //~ infallible fn, so if upb signals an error here we can only panic. - assert!(status == $pbr$::EncodeStatus::Ok); - let data = if len == 0 { - std::ptr::NonNull::dangling() - } else { - std::ptr::NonNull::new(buf).unwrap() - }; - - // SAFETY: - // - `arena` allocated `data`. - // - `data` is valid for reads up to `len` and will not be mutated. - unsafe { - $pbr$::SerializedData::from_raw_parts(arena, data, len) - } + let serialized = encoded.expect("serialize is not allowed to fail"); + serialized )rs"); return; } @@ -131,27 +113,25 @@ void MessageClearAndParse(Context& ctx, const Descriptor& msg) { let mut msg = Self::new(); // SAFETY: $minitable$ is a static of a const object. let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; - let ext_reg = std::ptr::null(); - let options = 0; // SAFETY: // - `data.as_ptr()` is valid to read for `data.len()` // - `mini_table` is the one used to construct `msg.raw_msg()` // - `msg.arena().raw()` is held for the same lifetime as `msg`. let status = unsafe { - $pbr$::upb_Decode( - data.as_ptr(), data.len(), msg.raw_msg(), - mini_table, ext_reg, options, msg.arena().raw()) + $pbr$::wire::decode( + data, msg.raw_msg(), + mini_table, msg.arena()) }; match status { - $pbr$::DecodeStatus::Ok => { + Ok(_) => { //~ This swap causes the old self.inner.arena to be moved into `msg` //~ which we immediately drop, which will release any previous //~ message that was held here. std::mem::swap(self, &mut msg); Ok(()) } - _ => Err($pb$::ParseError) + Err(_) => Err($pb$::ParseError) } )rs"); return; From 59d8afde6b580d3824e8d52a8bdfbbf45949552b Mon Sep 17 00:00:00 2001 From: Marcel Hlopko Date: Wed, 24 Apr 2024 13:30:38 -0700 Subject: [PATCH 036/179] Use the main/proper proto_common PiperOrigin-RevId: 627833411 --- rust/aspects.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/aspects.bzl b/rust/aspects.bzl index 7a714ea6b8..e1695379d4 100644 --- a/rust/aspects.bzl +++ b/rust/aspects.bzl @@ -4,6 +4,7 @@ Disclaimer: This project is experimental, under heavy development, and should no be used yet.""" load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") # buildifier: disable=bzl-visibility load("@rules_rust//rust/private:providers.bzl", "CrateInfo", "DepInfo", "DepVariantInfo") @@ -12,8 +13,6 @@ load("@rules_rust//rust/private:providers.bzl", "CrateInfo", "DepInfo", "DepVari load("@rules_rust//rust/private:rustc.bzl", "rustc_compile_action") load("//bazel:upb_proto_library.bzl", "UpbWrappedCcInfo", "upb_proto_library_aspect") -proto_common = proto_common_do_not_use - visibility(["//rust/..."]) CrateMappingInfo = provider( From dcc5db4c567a7cca31c3797e2644443e9921ba5d Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Thu, 25 Apr 2024 00:36:14 -0700 Subject: [PATCH 037/179] Implement Arena::fuse() PiperOrigin-RevId: 627983487 --- rust/upb/arena.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/rust/upb/arena.rs b/rust/upb/arena.rs index 281ea2f83c..76bb9e1ca0 100644 --- a/rust/upb/arena.rs +++ b/rust/upb/arena.rs @@ -142,6 +142,25 @@ impl Arena { slice::from_raw_parts_mut(alloc, data.len()) } } + + /// Fuse two arenas so they share the same lifetime. + /// + /// `fuse` will make it so that the memory allocated by `self` or `other` is + /// guaranteed to last until both `self` and `other` have been dropped. + /// The pointers returned by `Arena::alloc` will continue to be valid so + /// long as either `self` or `other` has not been dropped. + /// + pub fn fuse(&self, other: &Arena) { + // SAFETY: `self.raw()` and `other.raw()` are both valid UPB arenas. + let success = unsafe { upb_Arena_Fuse(self.raw(), other.raw()) }; + if !success { + // Fusing can fail if any of the arenas has an initial block i.e. the arena is + // backed by a preallocated chunk of memory that it doesn't own and thus cannot + // lifetime extend. This function panics because this is typically not a + // recoverable error but a logic bug in a program. + panic!("Could not fuse two UPB arenas."); + } + } } impl Default for Arena { @@ -164,6 +183,7 @@ extern "C" { fn upb_Arena_New() -> Option; fn upb_Arena_Free(arena: RawArena); fn upb_Arena_Malloc(arena: RawArena, size: usize) -> *mut u8; + fn upb_Arena_Fuse(arena1: RawArena, arena2: RawArena) -> bool; } #[cfg(test)] From 9ea682d1eea5489717588341a126d368da5fbe17 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Apr 2024 07:45:51 -0700 Subject: [PATCH 038/179] Add a TODO to use Layout::WithStaticSizes in SerialArenaChunk when we update the abseil release. PiperOrigin-RevId: 628073678 --- protobuf_deps.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 558540cc7f..19c51746e0 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -43,6 +43,8 @@ def protobuf_deps(): _github_archive( name = "com_google_absl", repo = "https://github.com/abseil/abseil-cpp", + # TODO: use Layout::WithStaticSizes in SerialArenaChunk when we update + # abseil to new release. commit = "4a2c63365eff8823a5221db86ef490e828306f9d", # Abseil LTS 20240116.0 sha256 = "f49929d22751bf70dd61922fb1fd05eb7aec5e7a7f870beece79a6e28f0a06c1", ) From 047fc7673eb2a7373f7ad8f5b094de201d737b36 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Apr 2024 08:39:30 -0700 Subject: [PATCH 039/179] Introduce `MessageDescriptor.IsMapType` in C# reflection to make it easier to tell whether a given message was generated by protoc for a map field. PiperOrigin-RevId: 628087118 --- .../Google.Protobuf.Test/Reflection/DescriptorsTest.cs | 8 ++++++++ csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs | 2 +- .../src/Google.Protobuf/Reflection/MessageDescriptor.cs | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs index 4bd857aacb..f908b2815c 100644 --- a/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs +++ b/csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs @@ -204,6 +204,14 @@ namespace Google.Protobuf.Reflection TestDescriptorToProto(messageType.ToProto, messageType.Proto); } + [Test] + public void MessageDescriptor_IsMapEntry() + { + var testMapMessage = TestMap.Descriptor; + Assert.False(testMapMessage.IsMapEntry); + Assert.True(testMapMessage.Fields[1].MessageType.IsMapEntry); + } + [Test] public void FieldDescriptor_GeneratedCode() { diff --git a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs index 9d41930838..d9bf6b373f 100644 --- a/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs @@ -242,7 +242,7 @@ namespace Google.Protobuf.Reflection /// /// Returns true if this field is a map field; false otherwise. /// - public bool IsMap => FieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry; + public bool IsMap => FieldType == FieldType.Message && messageType.IsMapEntry; /// /// Returns true if this field is a packed, repeated field; false otherwise. diff --git a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs index ead4676a5c..b4b69ad6e3 100644 --- a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs @@ -202,6 +202,12 @@ namespace Google.Protobuf.Reflection /// internal bool IsWrapperType => File.Package == "google.protobuf" && File.Name == "google/protobuf/wrappers.proto"; + /// + /// Returns whether this message was synthetically-created to store key/value pairs in a + /// map field. + /// + public bool IsMapEntry => Proto.Options?.MapEntry == true; + /// /// If this is a nested type, get the outer descriptor, otherwise null. /// From 7a1c926ef38b040fff86f4fb80d066c28ffea0c3 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Thu, 25 Apr 2024 08:53:50 -0700 Subject: [PATCH 040/179] upb: begin to delete the old Dart v2 plugin code PiperOrigin-RevId: 628090572 --- upb_generator/BUILD | 24 -- upb_generator/protoc-gen-upbdev.cc | 71 ----- upb_generator/subprocess.cc | 444 ----------------------------- upb_generator/subprocess.h | 81 ------ upb_generator/upbdev.c | 5 +- upb_generator/upbdev.h | 5 - 6 files changed, 3 insertions(+), 627 deletions(-) delete mode 100644 upb_generator/protoc-gen-upbdev.cc delete mode 100644 upb_generator/subprocess.cc delete mode 100644 upb_generator/subprocess.h diff --git a/upb_generator/BUILD b/upb_generator/BUILD index 56b211d748..757d840af5 100644 --- a/upb_generator/BUILD +++ b/upb_generator/BUILD @@ -409,27 +409,3 @@ proto_lang_toolchain( runtime = "//upb:generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", visibility = ["//visibility:public"], ) - -cc_binary( - name = "protoc-gen-upbdev", - srcs = [ - "protoc-gen-upbdev.cc", - "subprocess.cc", - "subprocess.h", - ], - copts = UPB_DEFAULT_CPPOPTS, - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":plugin_upb_proto", - ":upbdev", - "//upb:base", - "//upb:mem", - "//upb:port", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/strings", - ], -) diff --git a/upb_generator/protoc-gen-upbdev.cc b/upb_generator/protoc-gen-upbdev.cc deleted file mode 100644 index 801a2c8438..0000000000 --- a/upb_generator/protoc-gen-upbdev.cc +++ /dev/null @@ -1,71 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#include -#include - -#include "google/protobuf/compiler/plugin.upb.h" -#include "upb/base/status.h" -#include "upb/base/string_view.h" -#include "upb/mem/arena.h" -#include "upb_generator/subprocess.h" -#include "upb_generator/upbdev.h" - -static constexpr char kDefaultPlugin[] = "protoc_dart_plugin"; - -int main() { - upb_Arena* a = upb_Arena_New(); - upb_Status status; - upb_Status_Clear(&status); - - // Read (binary) stdin into a string. - const std::string input = {std::istreambuf_iterator(std::cin), - std::istreambuf_iterator()}; - - // Parse the request. - auto inner_request = google_protobuf_compiler_CodeGeneratorRequest_parse( - input.c_str(), input.size(), a); - - // Check the request for a plugin name. - std::string plugin = kDefaultPlugin; - if (google_protobuf_compiler_CodeGeneratorRequest_has_parameter(inner_request)) { - auto param = google_protobuf_compiler_CodeGeneratorRequest_parameter(inner_request); - plugin = std::string(param.data, param.size); - } - - // Wrap the request inside a upb_CodeGeneratorRequest and JSON-encode it. - const upb_StringView sv = - upbdev_ProcessInput(input.data(), input.size(), a, &status); - if (!upb_Status_IsOk(&status)) { - std::cerr << status.msg << '\n'; - return -1; - } - - // Launch the subprocess. - upb::generator::Subprocess subprocess; - subprocess.Start(plugin, upb::generator::Subprocess::SEARCH_PATH); - - // Exchange JSON strings with the subprocess. - const std::string json_request = std::string(sv.data, sv.size); - std::string json_response, error; - const bool ok = subprocess.Communicate(json_request, &json_response, &error); - if (!ok) { - // Dump the JSON request to stderr if we can't launch the next plugin. - std::cerr << json_request << '\n'; - return -1; - } - - // Decode, serialize, and write the JSON response. - upbdev_ProcessOutput(json_response.data(), json_response.size(), a, &status); - if (!upb_Status_IsOk(&status)) { - std::cerr << status.msg << '\n'; - return -1; - } - - upb_Arena_Free(a); - return 0; -} diff --git a/upb_generator/subprocess.cc b/upb_generator/subprocess.cc deleted file mode 100644 index 6831f656b7..0000000000 --- a/upb_generator/subprocess.cc +++ /dev/null @@ -1,444 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -// Shamelessly copied from the protobuf compiler's subprocess.cc -// except this version passes strings instead of Messages. - -#include "upb_generator/subprocess.h" - -#include -#include -#include - -#ifndef _MSVC_LANG -#include -#include -#include -#include -#endif - -#include "absl/log/absl_log.h" -#include "absl/strings/substitute.h" - -// Must be last. -#include "upb/port/def.inc" - -namespace upb { -namespace generator { - -namespace { -char* portable_strdup(const char* s) { - char* ns = (char*)malloc(strlen(s) + 1); - if (ns != nullptr) { - strcpy(ns, s); - } - return ns; -} -} // namespace - -#ifdef _WIN32 - -static void CloseHandleOrDie(HANDLE handle) { - if (!CloseHandle(handle)) { - ABSL_LOG(FATAL) << "CloseHandle: " - << Subprocess::Win32ErrorMessage(GetLastError()); - } -} - -Subprocess::Subprocess() - : process_start_error_(ERROR_SUCCESS), - child_handle_(nullptr), - child_stdin_(nullptr), - child_stdout_(nullptr) {} - -Subprocess::~Subprocess() { - if (child_stdin_ != nullptr) { - CloseHandleOrDie(child_stdin_); - } - if (child_stdout_ != nullptr) { - CloseHandleOrDie(child_stdout_); - } -} - -void Subprocess::Start(const std::string& program, SearchMode search_mode) { - // Create the pipes. - HANDLE stdin_pipe_read; - HANDLE stdin_pipe_write; - HANDLE stdout_pipe_read; - HANDLE stdout_pipe_write; - - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, nullptr, 0)) { - ABSL_LOG(FATAL) << "CreatePipe: " << Win32ErrorMessage(GetLastError()); - } - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, nullptr, 0)) { - ABSL_LOG(FATAL) << "CreatePipe: " << Win32ErrorMessage(GetLastError()); - } - - // Make child side of the pipes inheritable. - if (!SetHandleInformation(stdin_pipe_read, HANDLE_FLAG_INHERIT, - HANDLE_FLAG_INHERIT)) { - ABSL_LOG(FATAL) << "SetHandleInformation: " - << Win32ErrorMessage(GetLastError()); - } - if (!SetHandleInformation(stdout_pipe_write, HANDLE_FLAG_INHERIT, - HANDLE_FLAG_INHERIT)) { - ABSL_LOG(FATAL) << "SetHandleInformation: " - << Win32ErrorMessage(GetLastError()); - } - - // Setup STARTUPINFO to redirect handles. - STARTUPINFOA startup_info; - ZeroMemory(&startup_info, sizeof(startup_info)); - startup_info.cb = sizeof(startup_info); - startup_info.dwFlags = STARTF_USESTDHANDLES; - startup_info.hStdInput = stdin_pipe_read; - startup_info.hStdOutput = stdout_pipe_write; - startup_info.hStdError = GetStdHandle(STD_ERROR_HANDLE); - - if (startup_info.hStdError == INVALID_HANDLE_VALUE) { - ABSL_LOG(FATAL) << "GetStdHandle: " << Win32ErrorMessage(GetLastError()); - } - - // Invoking cmd.exe allows for '.bat' files from the path as well as '.exe'. - // Using a malloc'ed string because CreateProcess() can mutate its second - // parameter. - char* command_line = - portable_strdup(("cmd.exe /c \"" + program + "\"").c_str()); - - // Create the process. - PROCESS_INFORMATION process_info; - - if (CreateProcessA((search_mode == SEARCH_PATH) ? nullptr : program.c_str(), - (search_mode == SEARCH_PATH) ? command_line : nullptr, - nullptr, // process security attributes - nullptr, // thread security attributes - TRUE, // inherit handles? - 0, // obscure creation flags - nullptr, // environment (inherit from parent) - nullptr, // current directory (inherit from parent) - &startup_info, &process_info)) { - child_handle_ = process_info.hProcess; - CloseHandleOrDie(process_info.hThread); - child_stdin_ = stdin_pipe_write; - child_stdout_ = stdout_pipe_read; - } else { - process_start_error_ = GetLastError(); - CloseHandleOrDie(stdin_pipe_write); - CloseHandleOrDie(stdout_pipe_read); - } - - CloseHandleOrDie(stdin_pipe_read); - CloseHandleOrDie(stdout_pipe_write); - free(command_line); -} - -bool Subprocess::Communicate(const std::string& input_data, - std::string* output_data, std::string* error) { - if (process_start_error_ != ERROR_SUCCESS) { - *error = Win32ErrorMessage(process_start_error_); - return false; - } - - GOOGLE_CHECK(child_handle_ != nullptr) << "Must call Start() first."; - - int input_pos = 0; - - while (child_stdout_ != nullptr) { - HANDLE handles[2]; - int handle_count = 0; - - if (child_stdin_ != nullptr) { - handles[handle_count++] = child_stdin_; - } - if (child_stdout_ != nullptr) { - handles[handle_count++] = child_stdout_; - } - - DWORD wait_result = - WaitForMultipleObjects(handle_count, handles, FALSE, INFINITE); - - HANDLE signaled_handle = nullptr; - if (wait_result >= WAIT_OBJECT_0 && - wait_result < WAIT_OBJECT_0 + handle_count) { - signaled_handle = handles[wait_result - WAIT_OBJECT_0]; - } else if (wait_result == WAIT_FAILED) { - ABSL_LOG(FATAL) << "WaitForMultipleObjects: " - << Win32ErrorMessage(GetLastError()); - } else { - ABSL_LOG(FATAL) << "WaitForMultipleObjects: Unexpected return code: " - << wait_result; - } - - if (signaled_handle == child_stdin_) { - DWORD n; - if (!WriteFile(child_stdin_, input_data.data() + input_pos, - input_data.size() - input_pos, &n, nullptr)) { - // Child closed pipe. Presumably it will report an error later. - // Pretend we're done for now. - input_pos = input_data.size(); - } else { - input_pos += n; - } - - if (input_pos == input_data.size()) { - // We're done writing. Close. - CloseHandleOrDie(child_stdin_); - child_stdin_ = nullptr; - } - } else if (signaled_handle == child_stdout_) { - char buffer[4096]; - DWORD n; - - if (!ReadFile(child_stdout_, buffer, sizeof(buffer), &n, nullptr)) { - // We're done reading. Close. - CloseHandleOrDie(child_stdout_); - child_stdout_ = nullptr; - } else { - output_data->append(buffer, n); - } - } - } - - if (child_stdin_ != nullptr) { - // Child did not finish reading input before it closed the output. - // Presumably it exited with an error. - CloseHandleOrDie(child_stdin_); - child_stdin_ = nullptr; - } - - DWORD wait_result = WaitForSingleObject(child_handle_, INFINITE); - - if (wait_result == WAIT_FAILED) { - ABSL_LOG(FATAL) << "WaitForSingleObject: " - << Win32ErrorMessage(GetLastError()); - } else if (wait_result != WAIT_OBJECT_0) { - ABSL_LOG(FATAL) << "WaitForSingleObject: Unexpected return code: " - << wait_result; - } - - DWORD exit_code; - if (!GetExitCodeProcess(child_handle_, &exit_code)) { - ABSL_LOG(FATAL) << "GetExitCodeProcess: " - << Win32ErrorMessage(GetLastError()); - } - - CloseHandleOrDie(child_handle_); - child_handle_ = nullptr; - - if (exit_code != 0) { - *error = absl::Substitute("Plugin failed with status code $0.", exit_code); - return false; - } - - return true; -} - -std::string Subprocess::Win32ErrorMessage(DWORD error_code) { - char* message; - - // WTF? - FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - nullptr, error_code, - MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), - (LPSTR)&message, // NOT A BUG! - 0, nullptr); - - std::string result = message; - LocalFree(message); - return result; -} - -// =================================================================== - -#else // _WIN32 - -Subprocess::Subprocess() - : child_pid_(-1), child_stdin_(-1), child_stdout_(-1) {} - -Subprocess::~Subprocess() { - if (child_stdin_ != -1) { - close(child_stdin_); - } - if (child_stdout_ != -1) { - close(child_stdout_); - } -} - -void Subprocess::Start(const std::string& program, SearchMode search_mode) { - // Note that we assume that there are no other threads, thus we don't have to - // do crazy stuff like using socket pairs or avoiding libc locks. - - // [0] is read end, [1] is write end. - int stdin_pipe[2]; - int stdout_pipe[2]; - - int p0 = pipe(stdin_pipe); - int p1 = pipe(stdout_pipe); - UPB_ASSERT(p0 != -1); - UPB_ASSERT(p1 != -1); - - char* argv[2] = {portable_strdup(program.c_str()), nullptr}; - - child_pid_ = fork(); - if (child_pid_ == -1) { - std::cerr << "fork: " << strerror(errno); - } else if (child_pid_ == 0) { - // We are the child. - dup2(stdin_pipe[0], STDIN_FILENO); - dup2(stdout_pipe[1], STDOUT_FILENO); - - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - switch (search_mode) { - case SEARCH_PATH: - execvp(argv[0], argv); - break; - case EXACT_NAME: - execv(argv[0], argv); - break; - } - - // Write directly to STDERR_FILENO to avoid stdio code paths that may do - // stuff that is unsafe here. - int ignored; - ignored = write(STDERR_FILENO, argv[0], strlen(argv[0])); - const char* message = - ": program not found or is not executable\n" - "Please specify a program using absolute path or make sure " - "the program is available in your PATH system variable\n"; - ignored = write(STDERR_FILENO, message, strlen(message)); - (void)ignored; - - // Must use _exit() rather than exit() to avoid flushing output buffers - // that will also be flushed by the parent. - _exit(1); - } else { - free(argv[0]); - - close(stdin_pipe[0]); - close(stdout_pipe[1]); - - child_stdin_ = stdin_pipe[1]; - child_stdout_ = stdout_pipe[0]; - } -} - -bool Subprocess::Communicate(const std::string& input_data, - std::string* output_data, std::string* error) { - if (child_stdin_ == -1) { - std::cerr << "Must call Start() first." << '\n'; - UPB_ASSERT(child_stdin_ != -1); - } - - // The "sighandler_t" typedef is GNU-specific, so define our own. - typedef void SignalHandler(int); - - // Make sure SIGPIPE is disabled so that if the child dies it doesn't kill us. - SignalHandler* old_pipe_handler = signal(SIGPIPE, SIG_IGN); - - int input_pos = 0; - int max_fd = std::max(child_stdin_, child_stdout_); - - while (child_stdout_ != -1) { - fd_set read_fds; - fd_set write_fds; - FD_ZERO(&read_fds); - FD_ZERO(&write_fds); - if (child_stdout_ != -1) { - FD_SET(child_stdout_, &read_fds); - } - if (child_stdin_ != -1) { - FD_SET(child_stdin_, &write_fds); - } - - if (select(max_fd + 1, &read_fds, &write_fds, nullptr, nullptr) < 0) { - if (errno == EINTR) { - // Interrupted by signal. Try again. - continue; - } else { - std::cerr << "select: " << strerror(errno) << '\n'; - UPB_ASSERT(0); - } - } - - if (child_stdin_ != -1 && FD_ISSET(child_stdin_, &write_fds)) { - int n = write(child_stdin_, input_data.data() + input_pos, - input_data.size() - input_pos); - if (n < 0) { - // Child closed pipe. Presumably it will report an error later. - // Pretend we're done for now. - input_pos = input_data.size(); - } else { - input_pos += n; - } - - if (input_pos == (int)input_data.size()) { - // We're done writing. Close. - close(child_stdin_); - child_stdin_ = -1; - } - } - - if (child_stdout_ != -1 && FD_ISSET(child_stdout_, &read_fds)) { - char buffer[4096]; - int n = read(child_stdout_, buffer, sizeof(buffer)); - - if (n > 0) { - output_data->append(buffer, (size_t)n); - } else { - // We're done reading. Close. - close(child_stdout_); - child_stdout_ = -1; - } - } - } - - if (child_stdin_ != -1) { - // Child did not finish reading input before it closed the output. - // Presumably it exited with an error. - close(child_stdin_); - child_stdin_ = -1; - } - - int status; - while (waitpid(child_pid_, &status, 0) == -1) { - if (errno != EINTR) { - std::cerr << "waitpid: " << strerror(errno) << '\n'; - UPB_ASSERT(0); - } - } - - // Restore SIGPIPE handling. - signal(SIGPIPE, old_pipe_handler); - - if (WIFEXITED(status)) { - if (WEXITSTATUS(status) != 0) { - int error_code = WEXITSTATUS(status); - *error = - absl::Substitute("Plugin failed with status code $0.", error_code); - return false; - } - } else if (WIFSIGNALED(status)) { - int signal = WTERMSIG(status); - *error = absl::Substitute("Plugin killed by signal $0.", signal); - return false; - } else { - *error = "Neither WEXITSTATUS nor WTERMSIG is true?"; - return false; - } - - return true; -} - -#endif // !_WIN32 - -} // namespace generator -} // namespace upb diff --git a/upb_generator/subprocess.h b/upb_generator/subprocess.h deleted file mode 100644 index 731969083f..0000000000 --- a/upb_generator/subprocess.h +++ /dev/null @@ -1,81 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -// Shamelessly copied from the protobuf compiler's subprocess.h -// except this version passes strings instead of Messages. - -#ifndef THIRD_PARTY_UPB_UPB_GENERATOR_H_ -#define THIRD_PARTY_UPB_UPB_GENERATOR_H_ - -#ifdef _WIN32 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN // right... -#endif -#include -#else // _WIN32 -#include -#include -#endif // !_WIN32 -#include - -namespace upb { -namespace generator { - -// Utility class for launching sub-processes. -class Subprocess { - public: - Subprocess(); - ~Subprocess(); - - enum SearchMode { - SEARCH_PATH, // Use PATH environment variable. - EXACT_NAME // Program is an exact file name; don't use the PATH. - }; - - // Start the subprocess. Currently we don't provide a way to specify - // arguments as protoc plugins don't have any. - void Start(const std::string& program, SearchMode search_mode); - - // Pipe the input message to the subprocess's stdin, then close the pipe. - // Meanwhile, read from the subprocess's stdout and copy into *output. - // All this is done carefully to avoid deadlocks. - // Returns true if successful. On any sort of error, returns false and sets - // *error to a description of the problem. - bool Communicate(const std::string& input_data, std::string* output_data, - std::string* error); - -#ifdef _WIN32 - // Given an error code, returns a human-readable error message. This is - // defined here so that CommandLineInterface can share it. - static std::string Win32ErrorMessage(DWORD error_code); -#endif - - private: -#ifdef _WIN32 - DWORD process_start_error_; - HANDLE child_handle_; - - // The file handles for our end of the child's pipes. We close each and - // set it to NULL when no longer needed. - HANDLE child_stdin_; - HANDLE child_stdout_; - -#else // _WIN32 - pid_t child_pid_; - - // The file descriptors for our end of the child's pipes. We close each and - // set it to -1 when no longer needed. - int child_stdin_; - int child_stdout_; - -#endif // !_WIN32 -}; - -} // namespace generator -} // namespace upb - -#endif // THIRD_PARTY_UPB_UPB_GENERATOR_H_ diff --git a/upb_generator/upbdev.c b/upb_generator/upbdev.c index 1a48621a9b..4246919564 100644 --- a/upb_generator/upbdev.c +++ b/upb_generator/upbdev.c @@ -83,8 +83,9 @@ upb_StringView upbdev_ProcessInput(const char* buf, size_t size, return upbc_JsonEncode(outer_request, arena, status); } -upb_StringView upbdev_ProcessOutput(const char* buf, size_t size, - upb_Arena* arena, upb_Status* status) { +static upb_StringView upbdev_ProcessOutput(const char* buf, size_t size, + upb_Arena* arena, + upb_Status* status) { upb_StringView out = {.data = NULL, .size = 0}; const google_protobuf_compiler_CodeGeneratorResponse* response = diff --git a/upb_generator/upbdev.h b/upb_generator/upbdev.h index ee5b0a8aa9..a5fd0f12aa 100644 --- a/upb_generator/upbdev.h +++ b/upb_generator/upbdev.h @@ -25,11 +25,6 @@ UPB_API upb_StringView upbdev_ProcessInput(const char* buf, size_t size, upb_Arena* arena, upb_Status* status); -// Decode |buf| from JSON, serialize to wire format, and return it. -UPB_API upb_StringView upbdev_ProcessOutput(const char* buf, size_t size, - upb_Arena* arena, - upb_Status* status); - // Decode |buf| from JSON, serialize to wire format, and write it to stdout. UPB_API void upbdev_ProcessStdout(const char* buf, size_t size, upb_Arena* arena, upb_Status* status); From e0fc56659694cefa1cd96e8b867db7520fc5eaa0 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Thu, 25 Apr 2024 09:12:58 -0700 Subject: [PATCH 041/179] Ensure the appropriate upb functions taking non-const pointers assert that the message is not frozen. Fixes message/promote.c and internal/extension.c. PiperOrigin-RevId: 628095780 --- upb/message/internal/extension.c | 1 + upb/message/promote.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/upb/message/internal/extension.c b/upb/message/internal/extension.c index 85a09fd2d0..f55253d495 100644 --- a/upb/message/internal/extension.c +++ b/upb/message/internal/extension.c @@ -49,6 +49,7 @@ const upb_Extension* UPB_PRIVATE(_upb_Message_Getexts)( upb_Extension* UPB_PRIVATE(_upb_Message_GetOrCreateExtension)( struct upb_Message* msg, const upb_MiniTableExtension* e, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Extension* ext = (upb_Extension*)UPB_PRIVATE(_upb_Message_Getext)(msg, e); if (ext) return ext; if (!UPB_PRIVATE(_upb_Message_Realloc)(msg, sizeof(upb_Extension), a)) diff --git a/upb/message/promote.c b/upb/message/promote.c index d4b1ad1dee..6496e23a68 100644 --- a/upb/message/promote.c +++ b/upb/message/promote.c @@ -67,6 +67,7 @@ static upb_UnknownToMessageRet upb_MiniTable_ParseUnknownMessage( upb_GetExtension_Status upb_Message_GetOrPromoteExtension( upb_Message* msg, const upb_MiniTableExtension* ext_table, int decode_options, upb_Arena* arena, upb_MessageValue* value) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); UPB_ASSERT(upb_MiniTableExtension_CType(ext_table) == kUpb_CType_Message); const upb_Extension* extension = UPB_PRIVATE(_upb_Message_Getext)(msg, ext_table); @@ -177,6 +178,7 @@ upb_DecodeStatus upb_Message_PromoteMessage(upb_Message* parent, int decode_options, upb_Arena* arena, upb_Message** promoted) { + UPB_ASSERT(!upb_Message_IsFrozen(parent)); const upb_MiniTable* sub_table = upb_MiniTable_GetSubMessageTable(mini_table, field); UPB_ASSERT(sub_table); @@ -233,6 +235,7 @@ upb_UnknownToMessageRet upb_MiniTable_PromoteUnknownToMessage( upb_Message* msg, const upb_MiniTable* mini_table, const upb_MiniTableField* field, const upb_MiniTable* sub_mini_table, int decode_options, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_FindUnknownRet unknown; // We need to loop and merge unknowns that have matching tag field->number. upb_Message* message = NULL; @@ -292,6 +295,8 @@ upb_UnknownToMessageRet upb_MiniTable_PromoteUnknownToMessage( upb_UnknownToMessage_Status upb_MiniTable_PromoteUnknownToMessageArray( upb_Message* msg, const upb_MiniTableField* field, const upb_MiniTable* mini_table, int decode_options, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + upb_Array* repeated_messages = upb_Message_GetMutableArray(msg, field); // Find all unknowns with given field number and parse. upb_FindUnknownRet unknown; @@ -327,6 +332,8 @@ upb_UnknownToMessage_Status upb_MiniTable_PromoteUnknownToMessageArray( upb_UnknownToMessage_Status upb_MiniTable_PromoteUnknownToMap( upb_Message* msg, const upb_MiniTable* mini_table, const upb_MiniTableField* field, int decode_options, upb_Arena* arena) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + const upb_MiniTable* map_entry_mini_table = upb_MiniTable_MapEntrySubMessage(mini_table, field); UPB_ASSERT(upb_MiniTable_FieldCount(map_entry_mini_table) == 2); From 08c7b1ca92df7f56e2296eb3dcedc742636173f2 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Apr 2024 16:28:26 +0000 Subject: [PATCH 042/179] Auto-generate files after cl/628095780 --- php/ext/google/protobuf/php-upb.c | 1 + ruby/ext/google/protobuf_c/ruby-upb.c | 1 + 2 files changed, 2 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 3879fa0201..508b7f46dc 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -11996,6 +11996,7 @@ const upb_Extension* UPB_PRIVATE(_upb_Message_Getexts)( upb_Extension* UPB_PRIVATE(_upb_Message_GetOrCreateExtension)( struct upb_Message* msg, const upb_MiniTableExtension* e, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Extension* ext = (upb_Extension*)UPB_PRIVATE(_upb_Message_Getext)(msg, e); if (ext) return ext; if (!UPB_PRIVATE(_upb_Message_Realloc)(msg, sizeof(upb_Extension), a)) diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index a426daf84f..9578e8e976 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -11489,6 +11489,7 @@ const upb_Extension* UPB_PRIVATE(_upb_Message_Getexts)( upb_Extension* UPB_PRIVATE(_upb_Message_GetOrCreateExtension)( struct upb_Message* msg, const upb_MiniTableExtension* e, upb_Arena* a) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); upb_Extension* ext = (upb_Extension*)UPB_PRIVATE(_upb_Message_Getext)(msg, e); if (ext) return ext; if (!UPB_PRIVATE(_upb_Message_Realloc)(msg, sizeof(upb_Extension), a)) From 419824d26bb7a579b672aeb107770626bc883d13 Mon Sep 17 00:00:00 2001 From: Marcel Hlopko Date: Thu, 25 Apr 2024 12:11:39 -0700 Subject: [PATCH 043/179] Remove obsolete logic related to cc_proto_library PiperOrigin-RevId: 628151973 --- rust/aspects.bzl | 11 ----- rust/defs.bzl | 2 +- rust/test/BUILD | 41 +++---------------- .../rust_proto_library_unit_test/defs.bzl | 28 +------------ .../rust_proto_library_unit_test.bzl | 4 +- 5 files changed, 9 insertions(+), 77 deletions(-) diff --git a/rust/aspects.bzl b/rust/aspects.bzl index e1695379d4..9e50ec781e 100644 --- a/rust/aspects.bzl +++ b/rust/aspects.bzl @@ -265,10 +265,6 @@ def _compile_rust(ctx, attr, src, extra_srcs, deps): build_info = None, ) -def _is_cc_proto_library(rule): - """Detects if the current rule is a cc_proto_library.""" - return rule.kind == "cc_proto_library" - def _rust_upb_proto_aspect_impl(target, ctx): """Implements the Rust protobuf aspect logic for UPB kernel.""" return _rust_proto_aspect_common(target, ctx, is_upb = True) @@ -287,12 +283,6 @@ def _rust_proto_aspect_common(target, ctx, is_upb): if RustProtoInfo in target: return [] - if _is_cc_proto_library(ctx.rule): - # This is cc_proto_library, but we need the RustProtoInfo provider of the proto_library that - # this aspect provides. Luckily this aspect has already been attached on the proto_library - # so we can just read the provider. - return [ctx.rule.attr.deps[0][RustProtoInfo]] - proto_lang_toolchain = ctx.attr._proto_lang_toolchain[proto_common.ProtoLangToolchainInfo] cc_toolchain = find_cpp_toolchain(ctx) @@ -371,7 +361,6 @@ def _make_proto_library_aspect(is_upb): implementation = (_rust_upb_proto_aspect_impl if is_upb else _rust_cc_proto_aspect_impl), attr_aspects = ["deps"], requires = ([upb_proto_library_aspect] if is_upb else [cc_proto_aspect]), - required_aspect_providers = ([] if is_upb else [CcInfo]), attrs = { "_cc_toolchain": attr.label( doc = ( diff --git a/rust/defs.bzl b/rust/defs.bzl index 64c4022ed0..365c21ba56 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -79,7 +79,7 @@ def _make_rust_proto_library(is_upb): attrs = { "deps": attr.label_list( mandatory = True, - providers = [ProtoInfo] if is_upb else [CcInfo], + providers = [ProtoInfo], aspects = [rust_upb_proto_library_aspect if is_upb else rust_cc_proto_library_aspect], ), }, diff --git a/rust/test/BUILD b/rust/test/BUILD index 3b05958c8a..898bb075bc 100644 --- a/rust/test/BUILD +++ b/rust/test/BUILD @@ -1,4 +1,3 @@ -load("//bazel:cc_proto_library.bzl", "cc_proto_library") load( "//rust:defs.bzl", "rust_cc_proto_library", @@ -224,17 +223,11 @@ proto_library( ], ) -cc_proto_library( - name = "import_public_cc_proto", - testonly = True, - deps = [":import_public_proto"], -) - rust_cc_proto_library( name = "import_public_cc_rust_proto", testonly = True, visibility = ["//rust/test/shared:__subpackages__"], - deps = [":import_public_cc_proto"], + deps = [":import_public_proto"], ) rust_upb_proto_library( @@ -328,17 +321,11 @@ rust_upb_proto_library( deps = [":package_disabiguation_proto"], ) -cc_proto_library( - name = "package_disabiguation_cc_proto", - testonly = True, - deps = [":package_disabiguation_proto"], -) - rust_cc_proto_library( name = "package_disabiguation_cc_rust_proto", testonly = True, visibility = ["//rust/test/shared:__subpackages__"], - deps = [":package_disabiguation_cc_proto"], + deps = [":package_disabiguation_proto"], ) proto_library( @@ -411,19 +398,13 @@ proto_library( srcs = ["struct.proto"], ) -cc_proto_library( - name = "struct_cc_proto", - testonly = True, - deps = [":struct"], -) - rust_cc_proto_library( name = "struct_cc_rust_proto", testonly = True, visibility = [ "//rust/test/shared:__subpackages__", ], - deps = [":struct_cc_proto"], + deps = [":struct"], ) rust_upb_proto_library( @@ -441,19 +422,13 @@ proto_library( srcs = ["imported_types.proto"], ) -cc_proto_library( - name = "imported_types_cc_proto", - testonly = True, - deps = [":imported_types_proto"], -) - rust_cc_proto_library( name = "imported_types_cc_rust_proto", testonly = True, visibility = [ "//rust/test/shared:__subpackages__", ], - deps = [":imported_types_cc_proto"], + deps = [":imported_types_proto"], ) rust_upb_proto_library( @@ -473,19 +448,13 @@ proto_library( deps = [":imported_types_proto"], ) -cc_proto_library( - name = "fields_with_imported_types_cc_proto", - testonly = True, - deps = [":fields-with-imported-types_proto"], -) - rust_cc_proto_library( name = "fields_with_imported_types_cc_rust_proto", testonly = True, visibility = [ "//rust/test/shared:__subpackages__", ], - deps = [":fields_with_imported_types_cc_proto"], + deps = [":fields-with-imported-types_proto"], ) rust_upb_proto_library( diff --git a/rust/test/rust_proto_library_unit_test/defs.bzl b/rust/test/rust_proto_library_unit_test/defs.bzl index 9c4c1ec838..aed8ef4cb3 100644 --- a/rust/test/rust_proto_library_unit_test/defs.bzl +++ b/rust/test/rust_proto_library_unit_test/defs.bzl @@ -23,36 +23,12 @@ attach_upb_aspect = rule( }, ) -CcAspectHelperInfo = provider( - fields = { - "rust_proto_info": "RustProtoInfo from the proto_library", - "actions_info": "Actions of the proto_library", - }, - doc = "A provider passing data from proto_library through cc_proto_library", -) - -def _cc_aspect_helper_impl(_target, ctx): - if ctx.rule.kind == "cc_proto_library": - return CcAspectHelperInfo( - rust_proto_info = ctx.rule.attr.deps[0][RustProtoInfo], - actions_info = ActionsInfo(actions = ctx.rule.attr.deps[0].actions), - ) - - return [] - -_cc_aspect_helper = aspect( - implementation = _cc_aspect_helper_impl, - requires = [rust_cc_proto_library_aspect], - attr_aspects = ["deps"], -) - def _attach_cc_aspect_impl(ctx): - helper = ctx.attr.dep[CcAspectHelperInfo] - return [helper.rust_proto_info, helper.actions_info] + return [ctx.attr.dep[RustProtoInfo], ActionsInfo(actions = ctx.attr.dep.actions)] attach_cc_aspect = rule( implementation = _attach_cc_aspect_impl, attrs = { - "dep": attr.label(aspects = [_cc_aspect_helper]), + "dep": attr.label(aspects = [rust_cc_proto_library_aspect]), }, ) diff --git a/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl b/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl index 15e72fd613..610747666b 100644 --- a/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl +++ b/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl @@ -1,7 +1,6 @@ """This module contains unit tests for rust_proto_library and its aspect.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") -load("//bazel:cc_proto_library.bzl", "cc_proto_library") load("//rust:aspects.bzl", "RustProtoInfo") load("//rust:defs.bzl", "rust_cc_proto_library", "rust_upb_proto_library") load(":defs.bzl", "ActionsInfo", "attach_cc_aspect", "attach_upb_aspect") @@ -160,7 +159,7 @@ def _rust_cc_aspect_test_impl(ctx): rust_cc_aspect_test = analysistest.make(_rust_cc_aspect_test_impl) def _test_cc_aspect(): - attach_cc_aspect(name = "child_proto_with_cc_aspect", dep = ":child_cc_proto") + attach_cc_aspect(name = "child_proto_with_cc_aspect", dep = ":child_proto") rust_cc_aspect_test( name = "rust_cc_aspect_test", @@ -230,7 +229,6 @@ def rust_proto_library_unit_test(name): srcs = ["child.proto"], deps = [":parent_proto", ":parent2_proto"], ) - cc_proto_library(name = "child_cc_proto", deps = [":child_proto"]) _test_upb_aspect() _test_cc_aspect() From 62f2b143d529c3d899bda4c91fce12da0fdeabd7 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 25 Apr 2024 12:46:17 -0700 Subject: [PATCH 044/179] Change the arena.hpp ::Fuse return type to bool to surface whether the fuse failed. PiperOrigin-RevId: 628161481 --- upb/mem/arena.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/upb/mem/arena.hpp b/upb/mem/arena.hpp index e552ceb167..420adfa463 100644 --- a/upb/mem/arena.hpp +++ b/upb/mem/arena.hpp @@ -26,7 +26,11 @@ class Arena { upb_Arena* ptr() const { return ptr_.get(); } - void Fuse(Arena& other) { upb_Arena_Fuse(ptr(), other.ptr()); } + // Fuses the arenas together. + // This operation can only be performed on arenas with no initial blocks. Will + // return false if the fuse failed due to either arena having an initial + // block. + bool Fuse(Arena& other) { return upb_Arena_Fuse(ptr(), other.ptr()); } protected: std::unique_ptr ptr_; From 93bd4bb0122688aa7da8727bd61154e5cc730c38 Mon Sep 17 00:00:00 2001 From: Marcel Hlopko Date: Thu, 25 Apr 2024 23:44:09 -0700 Subject: [PATCH 045/179] Ensure that the rust_proto_library is in the same package as proto_library PiperOrigin-RevId: 628308230 --- rust/BUILD | 5 +++++ rust/aspects.bzl | 10 +++++++--- rust/defs.bzl | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/rust/BUILD b/rust/BUILD index 4aa9cb0280..a86ec69668 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -172,6 +172,11 @@ proto_lang_toolchain( visibility = ["//visibility:public"], ) +package_group( + name = "rust_proto_library_allowed_in_different_package", + packages = ["//rust/test"], # for unittest proto_libraries +) + # This flag controls what kernel all Rust Protobufs are using in the current build. string_flag( name = "rust_proto_library_kernel", diff --git a/rust/aspects.bzl b/rust/aspects.bzl index 9e50ec781e..ab72dc5aa9 100644 --- a/rust/aspects.bzl +++ b/rust/aspects.bzl @@ -33,6 +33,12 @@ RustProtoInfo = provider( }, ) +def proto_rust_toolchain_label(is_upb): + if is_upb: + return "//rust:proto_rust_upb_toolchain" + else: + return "//rust:proto_rust_cpp_toolchain" + def _register_crate_mapping_write_action(name, actions, crate_mappings): """Registers an action that generates a crate mapping for a proto_library. @@ -401,9 +407,7 @@ def _make_proto_library_aspect(is_upb): cfg = "exec", ), "_proto_lang_toolchain": attr.label( - default = Label( - "//rust:proto_rust_upb_toolchain" if is_upb else "//rust:proto_rust_cpp_toolchain", - ), + default = Label(proto_rust_toolchain_label(is_upb)), ), }, fragments = ["cpp"], diff --git a/rust/defs.bzl b/rust/defs.bzl index 365c21ba56..da65623139 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -3,9 +3,11 @@ Disclaimer: This project is experimental, under heavy development, and should not be used yet.""" +load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") load( "//rust:aspects.bzl", "RustProtoInfo", + "proto_rust_toolchain_label", "rust_cc_proto_library_aspect", "rust_upb_proto_library_aspect", ) @@ -65,6 +67,7 @@ def _rust_proto_library_impl(ctx): dep = deps[0] rust_proto_info = dep[RustProtoInfo] + dep_variant_info = rust_proto_info.dep_variant_info return [ dep_variant_info.crate_info, @@ -82,6 +85,9 @@ def _make_rust_proto_library(is_upb): providers = [ProtoInfo], aspects = [rust_upb_proto_library_aspect if is_upb else rust_cc_proto_library_aspect], ), + "_proto_lang_toolchain": attr.label( + default = Label(proto_rust_toolchain_label(is_upb)), + ), }, ) From ea4c77ca48711d5583796ca1f0efc2917c19b29f Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Fri, 26 Apr 2024 11:06:21 -0700 Subject: [PATCH 046/179] upb: implement .mergeFromBuffer() on GeneratedMessage in Dart PiperOrigin-RevId: 628455567 --- conformance/conformance_dart.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conformance/conformance_dart.dart b/conformance/conformance_dart.dart index 06b19b6e29..87d4359196 100644 --- a/conformance/conformance_dart.dart +++ b/conformance/conformance_dart.dart @@ -65,7 +65,7 @@ ConformanceResponse doTest(ConformanceRequest request) { switch (request.requestedOutputFormat) { case WireFormat.PROTOBUF: try { - response.protobufPayload = pb.GeneratedMessage.toBuffer(testMessage); + response.protobufPayload = pb.GeneratedMessage.toBinary(testMessage); } catch (e) { response.serializeError = '$e'; } @@ -86,7 +86,7 @@ Future doTestIo() async { } final request = ConformanceRequest.fromBuffer(serializedMsg); final response = doTest(request); - final serializedOutput = pb.GeneratedMessage.toBuffer(response); + final serializedOutput = pb.GeneratedMessage.toBinary(response); writeLittleEndianIntToStdout(serializedOutput.length); stdout.add(serializedOutput); await stdout.flush(); From b3e7a0077418cd9cb9b820c0197c3b8edb7007dd Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Sun, 28 Apr 2024 17:17:31 -0700 Subject: [PATCH 047/179] Add unit test enum showing allocation in repeated enum lists TestPackedTypes' field is unused, it's just there to make the messages symmetric between packed and unpacked versions. PiperOrigin-RevId: 628913968 --- .../protobuf/internal/descriptor_test.py | 4 +++ .../protobuf/internal/reflection_test.py | 30 +++++++++++++++---- src/google/protobuf/compiler/cpp/unittest.inc | 4 +-- src/google/protobuf/unittest.proto | 1 + src/google/protobuf/unittest_proto3.proto | 3 +- .../protobuf/unittest_proto3_arena.proto | 1 + 6 files changed, 34 insertions(+), 9 deletions(-) diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index ce16d0363a..2c8423efac 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -884,6 +884,10 @@ class DescriptorCopyToProtoTest(unittest.TestCase): name: 'FOREIGN_BAX' number: 32 > + value: < + name: 'FOREIGN_LARGE' + number: 123456 + > """ self._InternalTestCopyToProto( diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 459bef345e..171b9cda81 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -594,19 +594,37 @@ class ReflectionTest(unittest.TestCase): def testEnum_KeysAndValues(self, message_module): if message_module == unittest_pb2: - keys = ['FOREIGN_FOO', 'FOREIGN_BAR', 'FOREIGN_BAZ', 'FOREIGN_BAX'] - values = [4, 5, 6, 32] + keys = [ + 'FOREIGN_FOO', + 'FOREIGN_BAR', + 'FOREIGN_BAZ', + 'FOREIGN_BAX', + 'FOREIGN_LARGE', + ] + values = [4, 5, 6, 32, 123456] items = [ ('FOREIGN_FOO', 4), ('FOREIGN_BAR', 5), ('FOREIGN_BAZ', 6), ('FOREIGN_BAX', 32), + ('FOREIGN_LARGE', 123456), ] else: - keys = ['FOREIGN_ZERO', 'FOREIGN_FOO', 'FOREIGN_BAR', 'FOREIGN_BAZ'] - values = [0, 4, 5, 6] - items = [('FOREIGN_ZERO', 0), ('FOREIGN_FOO', 4), - ('FOREIGN_BAR', 5), ('FOREIGN_BAZ', 6)] + keys = [ + 'FOREIGN_ZERO', + 'FOREIGN_FOO', + 'FOREIGN_BAR', + 'FOREIGN_BAZ', + 'FOREIGN_LARGE', + ] + values = [0, 4, 5, 6, 123456] + items = [ + ('FOREIGN_ZERO', 0), + ('FOREIGN_FOO', 4), + ('FOREIGN_BAR', 5), + ('FOREIGN_BAZ', 6), + ('FOREIGN_LARGE', 123456), + ] self.assertEqual(keys, list(message_module.ForeignEnum.keys())) self.assertEqual(values, diff --git a/src/google/protobuf/compiler/cpp/unittest.inc b/src/google/protobuf/compiler/cpp/unittest.inc index c4b27ae94a..8da0de29ea 100644 --- a/src/google/protobuf/compiler/cpp/unittest.inc +++ b/src/google/protobuf/compiler/cpp/unittest.inc @@ -1022,8 +1022,8 @@ TEST(GENERATED_ENUM_TEST_NAME, MinAndMax) { EXPECT_EQ(4, UNITTEST::TestAllTypes::NestedEnum_ARRAYSIZE); EXPECT_EQ(UNITTEST::FOREIGN_FOO, UNITTEST::ForeignEnum_MIN); - EXPECT_EQ(UNITTEST::FOREIGN_BAX, UNITTEST::ForeignEnum_MAX); - EXPECT_EQ(33, UNITTEST::ForeignEnum_ARRAYSIZE); + EXPECT_EQ(UNITTEST::FOREIGN_LARGE, UNITTEST::ForeignEnum_MAX); + EXPECT_EQ(123457, UNITTEST::ForeignEnum_ARRAYSIZE); EXPECT_EQ(1, UNITTEST::TestEnumWithDupValue_MIN); EXPECT_EQ(3, UNITTEST::TestEnumWithDupValue_MAX); diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 434325f4d1..3d9dc3a775 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -198,6 +198,7 @@ enum ForeignEnum { FOREIGN_BAR = 5; FOREIGN_BAZ = 6; FOREIGN_BAX = 32; // (1 << 32) to generate a 64b bitmask would be incorrect. + FOREIGN_LARGE = 123456; // Large enough to escape the Boxed Integer cache. } message TestReservedFields { diff --git a/src/google/protobuf/unittest_proto3.proto b/src/google/protobuf/unittest_proto3.proto index 1c0f65bc9a..9067728ae2 100644 --- a/src/google/protobuf/unittest_proto3.proto +++ b/src/google/protobuf/unittest_proto3.proto @@ -114,7 +114,7 @@ message TestAllTypes { repeated string repeated_string_piece = 54 [ctype = STRING_PIECE]; repeated string repeated_cord = 55 [ctype = CORD]; - repeated NestedMessage repeated_lazy_message = 57 ; + repeated NestedMessage repeated_lazy_message = 57; oneof oneof_field { uint32 oneof_uint32 = 111; @@ -178,6 +178,7 @@ enum ForeignEnum { FOREIGN_FOO = 4; FOREIGN_BAR = 5; FOREIGN_BAZ = 6; + FOREIGN_LARGE = 123456; // Large enough to escape the Boxed Integer cache. } // TestEmptyMessage is used to test behavior of unknown fields. diff --git a/src/google/protobuf/unittest_proto3_arena.proto b/src/google/protobuf/unittest_proto3_arena.proto index 032ce6ff67..96a9ea85e2 100644 --- a/src/google/protobuf/unittest_proto3_arena.proto +++ b/src/google/protobuf/unittest_proto3_arena.proto @@ -197,6 +197,7 @@ enum ForeignEnum { FOREIGN_FOO = 4; FOREIGN_BAR = 5; FOREIGN_BAZ = 6; + FOREIGN_LARGE = 123456; // Large enough to escape the Boxed Integer cache. } // TestEmptyMessage is used to test behavior of unknown fields. From de6aba9f639ced3dce03310121aa69c914eb7b0e Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Sun, 28 Apr 2024 17:27:38 -0700 Subject: [PATCH 048/179] Java Proto Lite: Serialize repeated enums without allocating Serialize repeated enums without allocating PiperOrigin-RevId: 628915124 --- .../protobuf/CodedOutputStreamWriter.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java index 5368f5dede..a20a579e3e 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java @@ -557,6 +557,38 @@ final class CodedOutputStreamWriter implements Writer { @Override public void writeEnumList(int fieldNumber, List value, boolean packed) throws IOException { + if (value instanceof IntArrayList) { + writeEnumListInternal(fieldNumber, (IntArrayList) value, packed); + } else { + writeEnumListInternal(fieldNumber, value, packed); + } + } + + private void writeEnumListInternal(int fieldNumber, IntArrayList value, boolean packed) + throws IOException { + if (packed) { + output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); + + // Compute and write the length of the data. + int dataSize = 0; + for (int i = 0; i < value.size(); ++i) { + dataSize += CodedOutputStream.computeEnumSizeNoTag(value.getInt(i)); + } + output.writeUInt32NoTag(dataSize); + + // Write the data itself, without any tags. + for (int i = 0; i < value.size(); ++i) { + output.writeEnumNoTag(value.getInt(i)); + } + } else { + for (int i = 0; i < value.size(); ++i) { + output.writeEnum(fieldNumber, value.getInt(i)); + } + } + } + + private void writeEnumListInternal(int fieldNumber, List value, boolean packed) + throws IOException { if (packed) { output.writeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED); From 1d0028dddad380489fe1c93c8dd25cf25dbc5688 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 29 Apr 2024 00:00:16 -0700 Subject: [PATCH 049/179] Fix unused warnings in Rust Proto PiperOrigin-RevId: 628973552 --- rust/BUILD | 1 - rust/cpp.rs | 2 +- rust/macros.rs | 47 ----------------------------------------------- rust/shared.rs | 1 - rust/string.rs | 1 - rust/upb.rs | 2 +- 6 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 rust/macros.rs diff --git a/rust/BUILD b/rust/BUILD index a86ec69668..24dce3acce 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -48,7 +48,6 @@ rust_library( PROTOBUF_SHARED = [ "enum.rs", "internal.rs", - "macros.rs", "optional.rs", "primitive.rs", "proxied.rs", diff --git a/rust/cpp.rs b/rust/cpp.rs index 9e78c09336..3b70dff6ce 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -10,7 +10,7 @@ use crate::__internal::{Enum, Private}; use crate::{ Map, MapIter, Mut, ProtoStr, Proxied, ProxiedInMapValue, ProxiedInRepeated, Repeated, - RepeatedMut, RepeatedView, SettableValue, View, + RepeatedMut, RepeatedView, View, }; use core::fmt::Debug; use paste::paste; diff --git a/rust/macros.rs b/rust/macros.rs deleted file mode 100644 index 4f8334933e..0000000000 --- a/rust/macros.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -//! Runtime-internal macros - -/// Defines a `impl SettableValue<$proxied> for SomeType` body that forwards to -/// another implementation. -/// -/// # Example -/// ```ignore -/// impl<'a, const N: usize> SettableValue<[u8]> for &'a [u8; N] { -/// // Use the `SettableValue<[u8]>` implementation for `&[u8]`: -/// impl_forwarding_settable_value!([u8], self => &self[..]); -/// } -/// ``` -macro_rules! impl_forwarding_settable_value { - ($proxied:ty, $self:ident => $self_forwarding_expr:expr) => { - fn set_on<'b>( - $self, - _private: $crate::__internal::Private, - mutator: $crate::Mut<'b, $proxied>, - ) where $proxied: 'b { - ($self_forwarding_expr).set_on(Private, mutator) - } - - fn set_on_absent( - $self, - _private: $crate::__internal::Private, - absent_mutator: <$proxied as $crate::ProxiedWithPresence>::AbsentMutData<'_>, - ) -> <$proxied as $crate::ProxiedWithPresence>::PresentMutData<'_> { - ($self_forwarding_expr).set_on_absent($crate::__internal::Private, absent_mutator) - } - - fn set_on_present( - $self, - _private: $crate::__internal::Private, - present_mutator: <$proxied as $crate::ProxiedWithPresence>::PresentMutData<'_>, - ) { - ($self_forwarding_expr).set_on_present($crate::__internal::Private, present_mutator) - } - }; -} -pub(crate) use impl_forwarding_settable_value; diff --git a/rust/shared.rs b/rust/shared.rs index 79ff7e629a..26d606a4ca 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -56,7 +56,6 @@ pub mod __runtime; #[path = "enum.rs"] mod r#enum; -mod macros; mod map; mod optional; mod primitive; diff --git a/rust/string.rs b/rust/string.rs index e54ec44bc0..9711f87bf1 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -13,7 +13,6 @@ use crate::__internal::Private; use crate::__runtime::{ BytesAbsentMutData, BytesPresentMutData, InnerBytesMut, PtrAndLen, RawMessage, }; -use crate::macros::impl_forwarding_settable_value; use crate::{ AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, diff --git a/rust/upb.rs b/rust/upb.rs index e697a27bf8..413f9851e1 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -10,7 +10,7 @@ use crate::__internal::{Enum, Private}; use crate::{ Map, MapIter, MapMut, MapView, Mut, ProtoStr, Proxied, ProxiedInMapValue, ProxiedInRepeated, - Repeated, RepeatedMut, RepeatedView, SettableValue, View, ViewProxy, + Repeated, RepeatedMut, RepeatedView, View, ViewProxy, }; use core::fmt::Debug; use std::alloc::Layout; From d76fdc56bb2e9eb6005d09e5adb5518e2d67f2d7 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 29 Apr 2024 01:31:58 -0700 Subject: [PATCH 050/179] Implement IntoProxied for messages PiperOrigin-RevId: 628992357 --- rust/proxied.rs | 15 +++- rust/shared.rs | 3 +- rust/test/shared/accessors_test.rs | 19 ----- .../rust/accessors/singular_message.cc | 49 +++++++++++-- .../rust/accessors/singular_string.cc | 8 +-- src/google/protobuf/compiler/rust/message.cc | 69 +++++++++++-------- 6 files changed, 105 insertions(+), 58 deletions(-) diff --git a/rust/proxied.rs b/rust/proxied.rs index 41ddb6a743..beef53c15d 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -134,7 +134,7 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { /// y: View<'b, T>, /// ) -> [View<'b, T>; 2] /// where - /// T: Proxied, + /// T: MutProxied, /// 'a: 'b, /// { /// // `[x, y]` fails to compile because `'a` is not the same as `'b` and the `View` @@ -295,6 +295,19 @@ where } } +/// A value to `Proxied`-value conversion that consumes the input value. +/// +/// All setter functions accept types that implement `IntoProxied`. The purpose +/// of `IntoProxied` is to allow setting arbitrary values on Protobuf fields +/// with the minimal number of copies. +/// +/// This trait must not be implemented on types outside the Protobuf codegen and +/// runtime. We expect it to change in backwards incompatible ways in the +/// future. +pub trait IntoProxied { + fn into(self, _private: Private) -> T; +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/shared.rs b/rust/shared.rs index 26d606a4ca..4eacf7c767 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -28,7 +28,8 @@ pub mod __public { pub use crate::primitive::PrimitiveMut; pub use crate::proto; pub use crate::proxied::{ - Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, + IntoProxied, Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, + ViewProxy, }; pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index fa11b86d07..36caf8b00c 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -507,16 +507,9 @@ fn test_singular_msg_field() { assert_that!(msg.has_optional_nested_message(), eq(false)); let mut nested_msg_mut = msg.optional_nested_message_mut(); - // test reading an int inside a mut assert_that!(nested_msg_mut.bb(), eq(0)); - // Test setting an owned NestedMessage onto another message. - let mut new_nested = NestedMessage::new(); - new_nested.set_bb(7); - nested_msg_mut.set(new_nested); - assert_that!(nested_msg_mut.bb(), eq(7)); - assert_that!(msg.has_optional_nested_message(), eq(true)); } @@ -762,18 +755,6 @@ fn test_msg_oneof_default_accessors() { // TODO: Add tests covering a message-type field in a oneof. } -#[test] -fn test_set_message_from_view() { - use protobuf::MutProxy; - - let mut m1 = TestAllTypes::new(); - m1.set_optional_int32(1); - let mut m2 = TestAllTypes::new(); - m2.as_mut().set(m1.as_view()); - - assert_that!(m2.optional_int32(), eq(1i32)); -} - #[test] fn test_group() { let mut m = TestAllTypes::new(); diff --git a/src/google/protobuf/compiler/rust/accessors/singular_message.cc b/src/google/protobuf/compiler/rust/accessors/singular_message.cc index 73acd60164..347f1d5dcd 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_message.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_message.cc @@ -34,6 +34,7 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, {"clearer_thunk", ThunkName(ctx, field, "clear")}, {"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"set_allocated_thunk", ThunkName(ctx, field, "set")}, { "getter_body", [&] { @@ -95,7 +96,7 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, unsafe { let has = self.has_$raw_field_name$(); $pbi$::new_vtable_field_entry($pbi$::Private, - self.as_mutator_message_ref(), + self.as_mutator_message_ref($pbi$::Private), &VTABLE, has) } @@ -112,14 +113,44 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, } )rs"); }}, + {"setter_body", + [&] { + if (accessor_case == AccessorCase::VIEW) return; + if (ctx.is_upb()) { + ctx.Emit({}, R"rs( + // The message and arena are dropped after the setter. The + // memory remains allocated as we fuse the arena with the + // parent message's arena. + let mut msg = val.into($pbi$::Private); + self.as_mutator_message_ref($pbi$::Private) + .arena($pbi$::Private) + .fuse(msg.as_mutator_message_ref($pbi$::Private).arena($pbi$::Private)); + + unsafe { + $set_allocated_thunk$(self.as_mutator_message_ref($pbi$::Private).msg(), + msg.as_mutator_message_ref($pbi$::Private).msg()); + } + )rs"); + } else { + ctx.Emit({}, R"rs( + // Prevent the memory from being deallocated. The setter + // transfers ownership of the memory to the parent message. + let mut msg = std::mem::ManuallyDrop::new(val.into($pbi$::Private)); + unsafe { + $set_allocated_thunk$(self.as_mutator_message_ref($pbi$::Private).msg(), + msg.as_mutator_message_ref($pbi$::Private).msg()); + } + )rs"); + } + }}, {"setter", [&] { if (accessor_case == AccessorCase::VIEW) return; ctx.Emit(R"rs( - pub fn set_$raw_field_name$(&mut self, val: impl $pb$::SettableValue<$msg_type$>) { - //~ TODO: Optimize this to not go through the - //~ FieldEntry. - self.$raw_field_name$_entry().set(val); + pub fn set_$raw_field_name$(&mut self, + val: impl $pb$::IntoProxied<$msg_type$>) { + + $setter_body$ } )rs"); }}, @@ -157,6 +188,7 @@ void SingularMessage::InExternC(Context& ctx, {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, {"clearer_thunk", ThunkName(ctx, field, "clear")}, {"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"set_allocated_thunk", ThunkName(ctx, field, "set")}, {"getter_mut", [&] { if (ctx.is_cpp()) { @@ -188,12 +220,16 @@ void SingularMessage::InExternC(Context& ctx, $getter_mut$ fn $clearer_thunk$(raw_msg: $pbr$::RawMessage); fn $hazzer_thunk$(raw_msg: $pbr$::RawMessage) -> bool; + fn $set_allocated_thunk$(raw_msg: $pbr$::RawMessage, + field_msg: $pbr$::RawMessage); )rs"); } void SingularMessage::InThunkCc(Context& ctx, const FieldDescriptor& field) const { ctx.Emit({{"QualifiedMsg", cpp::QualifiedClassName(field.containing_type())}, + {"FieldMsg", cpp::QualifiedClassName(field.message_type())}, + {"set_allocated_thunk", ThunkName(ctx, field, "set")}, {"getter_thunk", ThunkName(ctx, field, "get")}, {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, {"clearer_thunk", ThunkName(ctx, field, "clear")}, @@ -208,6 +244,9 @@ void SingularMessage::InThunkCc(Context& ctx, } void $clearer_thunk$($QualifiedMsg$* msg) { msg->clear_$field$(); } bool $hazzer_thunk$($QualifiedMsg$* msg) { return msg->has_$field$(); } + void $set_allocated_thunk$($QualifiedMsg$* msg, $FieldMsg$* sub_msg) { + msg->set_allocated_$field$(sub_msg); + } )cc"); } diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index 1de8f5b7ae..ee45d71efb 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -87,13 +87,13 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, pub fn set_$raw_field_name$(&mut self, val: impl std::convert::AsRef<$proxied_type$>) { let string_view: $pbr$::PtrAndLen = $pbr$::copy_bytes_in_arena_if_needed_by_runtime( - self.as_mutator_message_ref(), + self.as_mutator_message_ref($pbi$::Private), val.as_ref().into() ).into(); unsafe { $setter_thunk$( - self.as_mutator_message_ref().msg(), + self.as_mutator_message_ref($pbi$::Private).msg(), string_view ); } @@ -160,7 +160,7 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, let has = $hazzer_thunk$(self.raw_msg()); $pbi$::new_vtable_field_entry( $pbi$::Private, - self.as_mutator_message_ref(), + self.as_mutator_message_ref($pbi$::Private), $Msg$::$vtable_name$, has, ) @@ -176,7 +176,7 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, $pbi$::Private, $pbi$::RawVTableMutator::new( $pbi$::Private, - self.as_mutator_message_ref(), + self.as_mutator_message_ref($pbi$::Private), $Msg$::$vtable_name$, ) ) diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 5ccf2541ac..67c569f4f5 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -226,33 +226,56 @@ void MessageDrop(Context& ctx, const Descriptor& msg) { )rs"); } -void MessageSettableValueForView(Context& ctx, const Descriptor& msg) { +void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: ctx.Emit({{"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}}, R"rs( - impl<'msg> $pb$::SettableValue<$Msg$> for $Msg$View<'msg> { - fn set_on<'dst>( - self, _private: $pbi$::Private, mutator: $pb$::Mut<'dst, $Msg$>) - where $Msg$: 'dst { - unsafe { $copy_from_thunk$(mutator.inner.msg(), self.msg) }; + impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$View<'msg> { + fn into(self, _private: $pbi$::Private) -> $Msg$ { + let dst = $Msg$::new(); + unsafe { $copy_from_thunk$(dst.inner.msg, self.msg) }; + dst + } + } + + impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { + fn into(self, _private: $pbi$::Private) -> $Msg$ { + $pb$::IntoProxied::into($pb$::ViewProxy::into_view(self), _private) + } + } + + impl $pb$::IntoProxied<$Msg$> for $Msg$ { + fn into(self, _private: $pbi$::Private) -> $Msg$ { + self } } )rs"); return; case Kernel::kUpb: - // TODO: Add owned SettableValue impl for upb messages. ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( - impl<'msg> $pb$::SettableValue<$Msg$> for $Msg$View<'msg> { - fn set_on<'dst>( - self, _private: $pbi$::Private, mutator: $pb$::Mut<'dst, $Msg$>) - where $Msg$: 'dst { + impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$View<'msg> { + fn into(self, _private: $pbi$::Private) -> $Msg$ { + let dst = $Msg$::new(); unsafe { $pbr$::upb_Message_DeepCopy( - mutator.inner.msg(), + dst.inner.msg, self.msg, $std$::ptr::addr_of!($minitable$), - mutator.inner.arena($pbi$::Private).raw(), + dst.inner.arena.raw(), ) }; + dst + } + } + + impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { + fn into(self, _private: $pbi$::Private) -> $Msg$ { + $pb$::IntoProxied::into($pb$::ViewProxy::into_view(self), _private) + } + } + + impl $pb$::IntoProxied<$Msg$> for $Msg$ { + fn into(self, _private: $pbi$::Private) -> $Msg$ { + self } } )rs"); @@ -797,8 +820,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { AccessorCase::MUT); } }}, - {"settable_impl_for_view", - [&] { MessageSettableValueForView(ctx, msg); }}, + {"into_proxied_impl", [&] { IntoProxiedForMessage(ctx, msg); }}, {"repeated_impl", [&] { MessageProxiedInRepeated(ctx, msg); }}, {"map_value_impl", [&] { MessageProxiedInMapValue(ctx, msg); }}, {"unwrap_upb", @@ -960,17 +982,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } - $settable_impl_for_view$ - - impl $pb$::SettableValue<$Msg$> for $Msg$ { - fn set_on<'dst>( - self, _private: $pbi$::Private, mutator: $pb$::Mut<'dst, $Msg$>) - where $Msg$: 'dst { - //~ TODO: b/320701507 - This current will copy the message and then - //~ drop it, this copy would be avoided on upb kernel. - self.as_view().set_on($pbi$::Private, mutator); - } - } + $into_proxied_impl$ $repeated_impl$ $map_value_impl$ @@ -1013,7 +1025,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.inner.msg() } - fn as_mutator_message_ref(&mut self) -> $pbr$::MutatorMessageRef<'msg> { + pub fn as_mutator_message_ref(&mut self, _private: $pbi$::Private) + -> $pbr$::MutatorMessageRef<'msg> { self.inner } @@ -1059,7 +1072,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.inner.msg } - fn as_mutator_message_ref(&mut self) -> $pbr$::MutatorMessageRef { + pub fn as_mutator_message_ref(&mut self, _private: $pbi$::Private) -> $pbr$::MutatorMessageRef { $pbr$::MutatorMessageRef::new($pbi$::Private, &mut self.inner) } From 1a84c9c0374c912e21fecdaa298bb2cd29b19741 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 29 Apr 2024 02:14:36 -0700 Subject: [PATCH 051/179] Remove gencode that uses vtables for string/bytes field accessors It's now unused. PiperOrigin-RevId: 629000604 --- .../rust/accessors/singular_string.cc | 83 ------------------- 1 file changed, 83 deletions(-) diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index ee45d71efb..dd4de8ce7e 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -11,7 +11,6 @@ #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" #include "google/protobuf/compiler/rust/accessors/accessor_generator.h" -#include "google/protobuf/compiler/rust/accessors/helpers.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" @@ -45,18 +44,6 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, } }) .WithSuffix(""), // This lets `$transform_view$,` work. - {"transform_field_entry", - [&] { - if (field.type() == FieldDescriptor::TYPE_STRING) { - ctx.Emit(R"rs( - $pb$::ProtoStrMut::field_entry_from_bytes( - $pbi$::Private, out - ) - )rs"); - } else { - ctx.Emit("out"); - } - }}, {"view_lifetime", ViewLifetime(accessor_case)}, {"view_self", ViewReceiver(accessor_case)}, {"getter", @@ -117,74 +104,6 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, unsafe { $clearer_thunk$(self.raw_msg()) } })rs"); }}, - {"vtable_name", VTableName(field)}, - {"vtable", - [&] { - if (accessor_case != AccessorCase::OWNED) { - return; - } - if (field.has_presence()) { - ctx.Emit({{"default_value", DefaultValue(ctx, field)}}, - R"rs( - // SAFETY: for `string` fields, the default value is verified as valid UTF-8 - const $vtable_name$: &'static $pbi$::BytesOptionalMutVTable = &unsafe { - $pbi$::BytesOptionalMutVTable::new( - $pbi$::Private, - $getter_thunk$, - $setter_thunk$, - $clearer_thunk$, - $default_value$, - ) - }; - )rs"); - } else { - ctx.Emit(R"rs( - const $vtable_name$: &'static $pbi$::BytesMutVTable = - &$pbi$::BytesMutVTable::new( - $pbi$::Private, - $getter_thunk$, - $setter_thunk$, - ); - )rs"); - } - }}, - {"field_mutator_getter", - [&] { - if (accessor_case == AccessorCase::VIEW) { - return; - } - if (field.has_presence()) { - ctx.Emit(R"rs( - fn $raw_field_name$_mut(&mut self) -> $pb$::FieldEntry<'_, $proxied_type$> { - let out = unsafe { - let has = $hazzer_thunk$(self.raw_msg()); - $pbi$::new_vtable_field_entry( - $pbi$::Private, - self.as_mutator_message_ref($pbi$::Private), - $Msg$::$vtable_name$, - has, - ) - }; - $transform_field_entry$ - } - )rs"); - } else { - ctx.Emit(R"rs( - fn $raw_field_name$_mut(&mut self) -> $pb$::Mut<'_, $proxied_type$> { - unsafe { - <$pb$::Mut<$proxied_type$>>::from_inner( - $pbi$::Private, - $pbi$::RawVTableMutator::new( - $pbi$::Private, - self.as_mutator_message_ref($pbi$::Private), - $Msg$::$vtable_name$, - ) - ) - } - } - )rs"); - } - }}, }, R"rs( $getter$ @@ -192,8 +111,6 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, $setter$ $hazzer$ $clearer$ - $vtable$ - $field_mutator_getter$ )rs"); } From 1980e025eb315ed6c8d1c0ba2c5653022d889189 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 29 Apr 2024 03:39:57 -0700 Subject: [PATCH 052/179] Remove PrimitiveMut and related vtable types The last callside that used PrimitiveMut was in our enums code. This change makes it so that enums nolonger implement MutProxied and thus no longer need the PrimitiveMut type. PiperOrigin-RevId: 629017282 --- rust/BUILD | 2 +- rust/cpp.rs | 1 - rust/internal.rs | 5 +- rust/primitive.rs | 129 +-------------------- rust/shared.rs | 1 - rust/upb.rs | 1 - rust/vtable.rs | 135 +--------------------- src/google/protobuf/compiler/rust/enum.cc | 33 ------ 8 files changed, 6 insertions(+), 301 deletions(-) diff --git a/rust/BUILD b/rust/BUILD index 24dce3acce..21fe525d4b 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -48,8 +48,8 @@ rust_library( PROTOBUF_SHARED = [ "enum.rs", "internal.rs", - "optional.rs", "primitive.rs", + "optional.rs", "proxied.rs", "repeated.rs", "shared.rs", diff --git a/rust/cpp.rs b/rust/cpp.rs index 3b70dff6ce..06b45f6f92 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -238,7 +238,6 @@ pub type MessageAbsentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutator pub type BytesPresentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; pub type BytesAbsentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; pub type InnerBytesMut<'msg> = crate::vtable::RawVTableMutator<'msg, [u8]>; -pub type InnerPrimitiveMut<'msg, T> = crate::vtable::RawVTableMutator<'msg, T>; pub type RawMapIter = UntypedMapIterator; #[derive(Debug)] diff --git a/rust/internal.rs b/rust/internal.rs index 4a9486e91a..742423bc10 100644 --- a/rust/internal.rs +++ b/rust/internal.rs @@ -14,9 +14,8 @@ pub use paste::paste; pub use crate::r#enum::Enum; pub use crate::vtable::{ - new_vtable_field_entry, BytesMutVTable, BytesOptionalMutVTable, PrimitiveOptionalMutVTable, - PrimitiveVTable, PrimitiveWithRawVTable, ProxiedWithRawOptionalVTable, ProxiedWithRawVTable, - RawVTableMutator, RawVTableOptionalMutatorData, + new_vtable_field_entry, BytesMutVTable, BytesOptionalMutVTable, ProxiedWithRawOptionalVTable, + ProxiedWithRawVTable, RawVTableMutator, RawVTableOptionalMutatorData, }; pub use crate::ProtoStr; diff --git a/rust/primitive.rs b/rust/primitive.rs index f14863187b..75665eca2b 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -4,96 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd - -use std::fmt::Debug; - -use crate::__internal::Private; -use crate::__runtime::InnerPrimitiveMut; -use crate::vtable::{PrimitiveWithRawVTable, ProxiedWithRawVTable, RawVTableOptionalMutatorData}; -use crate::{ - Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, -}; - -/// A mutator for a primitive (numeric or enum) value of `T`. -/// -/// This type is `protobuf::Mut<'msg, T>`. -pub struct PrimitiveMut<'msg, T> { - inner: InnerPrimitiveMut<'msg, T>, -} - -impl<'msg, T> Debug for PrimitiveMut<'msg, T> -where - T: PrimitiveWithRawVTable, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PrimitiveMut").field("inner", &self.inner).finish() - } -} - -impl<'msg, T> PrimitiveMut<'msg, T> { - /// # Safety - /// `inner` must be valid and non-aliased for `T` for `'msg` - #[doc(hidden)] - pub unsafe fn from_inner(_private: Private, inner: InnerPrimitiveMut<'msg, T>) -> Self { - Self { inner } - } -} - -// SAFETY: all `T` that can perform mutations don't mutate through a shared -// reference. -unsafe impl<'msg, T> Sync for PrimitiveMut<'msg, T> {} - -impl<'msg, T> PrimitiveMut<'msg, T> -where - T: PrimitiveWithRawVTable, -{ - /// Gets the current value of the field. - pub fn get(&self) -> View<'_, T> { - T::make_view(Private, self.inner) - } - - /// Sets a new value for the field. - pub fn set(&mut self, val: impl SettableValue) { - val.set_on(Private, self.as_mut()) - } - - #[doc(hidden)] - pub fn set_primitive(&mut self, _private: Private, value: T) { - // SAFETY: the raw mutator is valid for `'msg` as enforced by `Mut` - unsafe { self.inner.set(value) } - } -} - -impl<'msg, T> ViewProxy<'msg> for PrimitiveMut<'msg, T> -where - T: PrimitiveWithRawVTable, -{ - type Proxied = T; - - fn as_view(&self) -> View<'_, Self::Proxied> { - self.get() - } - - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> { - self.get() - } -} - -impl<'msg, T> MutProxy<'msg> for PrimitiveMut<'msg, T> -where - T: PrimitiveWithRawVTable, -{ - fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { - PrimitiveMut { inner: self.inner } - } - - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> - where - 'msg: 'shorter, - { - self - } -} +use crate::{Proxied, View, ViewProxy}; macro_rules! impl_singular_primitives { ($($t:ty),*) => { @@ -102,10 +13,6 @@ macro_rules! impl_singular_primitives { type View<'msg> = $t; } - impl MutProxied for $t { - type Mut<'msg> = PrimitiveMut<'msg, $t>; - } - impl<'msg> ViewProxy<'msg> for $t { type Proxied = $t; @@ -118,40 +25,6 @@ macro_rules! impl_singular_primitives { } } - impl SettableValue<$t> for $t { - fn set_on<'msg>(self, private: Private, mut mutator: Mut<'msg, $t>) where $t: 'msg { - mutator.set_primitive(private, self) - } - - fn set_on_absent( - self, - _private: Private, - absent_mutator: <$t as ProxiedWithPresence>::PresentMutData<'_>, - ) -> <$t as ProxiedWithPresence>::AbsentMutData<'_> - { - absent_mutator.set(Private, self) - } - } - - impl ProxiedWithPresence for $t { - type PresentMutData<'msg> = RawVTableOptionalMutatorData<'msg, $t>; - type AbsentMutData<'msg> = RawVTableOptionalMutatorData<'msg, $t>; - - fn clear_present_field( - present_mutator: Self::PresentMutData<'_>, - ) -> Self::AbsentMutData<'_> { - present_mutator.clear(Private) - } - - fn set_absent_to_default( - absent_mutator: Self::AbsentMutData<'_>, - ) -> Self::PresentMutData<'_> { - absent_mutator.set_absent_to_default(Private) - } - } - - impl PrimitiveWithRawVTable for $t {} - // ProxiedInRepeated is implemented in {cpp,upb}.rs )* } diff --git a/rust/shared.rs b/rust/shared.rs index 4eacf7c767..8ea0684a39 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -25,7 +25,6 @@ pub mod __public { pub use crate::r#enum::UnknownEnumValue; pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; pub use crate::optional::{AbsentField, FieldEntry, Optional, PresentField}; - pub use crate::primitive::PrimitiveMut; pub use crate::proto; pub use crate::proxied::{ IntoProxied, Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, diff --git a/rust/upb.rs b/rust/upb.rs index 413f9851e1..98e1c849c1 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -66,7 +66,6 @@ pub type MessageAbsentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutator pub type BytesPresentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; pub type BytesAbsentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; pub type InnerBytesMut<'msg> = crate::vtable::RawVTableMutator<'msg, [u8]>; -pub type InnerPrimitiveMut<'msg, T> = crate::vtable::RawVTableMutator<'msg, T>; #[derive(Debug)] pub struct MessageVTable { diff --git a/rust/vtable.rs b/rust/vtable.rs index fbcf4c46a2..27f14db922 100644 --- a/rust/vtable.rs +++ b/rust/vtable.rs @@ -7,11 +7,10 @@ use crate::__internal::Private; use crate::__runtime::{ - copy_bytes_in_arena_if_needed_by_runtime, InnerPrimitiveMut, MutatorMessageRef, PtrAndLen, - RawMessage, + copy_bytes_in_arena_if_needed_by_runtime, MutatorMessageRef, PtrAndLen, RawMessage, }; use crate::{ - AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, PrimitiveMut, + AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, ProxiedWithPresence, View, ViewProxy, }; use std::fmt::{self, Debug}; @@ -281,47 +280,6 @@ impl ProxiedWithRawOptionalVTable for [u8] { } } -/// A generic thunk vtable for mutating a present primitive field. -#[doc(hidden)] -#[derive(Debug)] -pub struct PrimitiveVTable { - pub(crate) setter: unsafe extern "C" fn(msg: RawMessage, val: T), - pub(crate) getter: unsafe extern "C" fn(msg: RawMessage) -> T, -} - -#[doc(hidden)] -#[derive(Debug)] -/// A generic thunk vtable for mutating an `optional` primitive field. -pub struct PrimitiveOptionalMutVTable { - pub(crate) base: PrimitiveVTable, - pub(crate) clearer: unsafe extern "C" fn(msg: RawMessage), - pub(crate) default: T, -} - -impl PrimitiveVTable { - #[doc(hidden)] - pub const fn new( - _private: Private, - getter: unsafe extern "C" fn(msg: RawMessage) -> T, - setter: unsafe extern "C" fn(msg: RawMessage, val: T), - ) -> Self { - Self { getter, setter } - } -} - -impl PrimitiveOptionalMutVTable { - #[doc(hidden)] - pub const fn new( - _private: Private, - getter: unsafe extern "C" fn(msg: RawMessage) -> T, - setter: unsafe extern "C" fn(msg: RawMessage, val: T), - clearer: unsafe extern "C" fn(msg: RawMessage), - default: T, - ) -> Self { - Self { base: PrimitiveVTable { getter, setter }, clearer, default } - } -} - /// A generic thunk vtable for mutating a present `bytes` or `string` field. #[doc(hidden)] #[derive(Debug)] @@ -425,92 +383,3 @@ impl<'msg> RawVTableOptionalMutatorData<'msg, [u8]> { self } } - -/// Primitive types using a vtable for message access that are trivial to copy -/// and have a `'static` lifetime. -/// -/// Implementing this trait automatically implements `ProxiedWithRawVTable`, -/// `ProxiedWithRawOptionalVTable`, and get/set/clear methods on -/// `RawVTableMutator` and `RawVTableOptionalMutatorData` that use the vtable. -/// -/// It doesn't implement `Proxied`, `ViewProxy`, `SettableValue` or -/// `ProxiedWithPresence` for `Self` to avoid future conflicting blanket impls -/// on those traits. -pub trait PrimitiveWithRawVTable: - Copy - + Debug - + 'static - + ProxiedWithPresence - + Sync - + Send - + for<'msg> MutProxied = Self, Mut<'msg> = PrimitiveMut<'msg, Self>> -{ -} - -impl ProxiedWithRawVTable for T { - type VTable = PrimitiveVTable; - - fn make_view(_private: Private, mut_inner: InnerPrimitiveMut<'_, Self>) -> Self { - mut_inner.get() - } - - fn make_mut(_private: Private, inner: InnerPrimitiveMut<'_, Self>) -> PrimitiveMut<'_, Self> { - // SAFETY: `inner` is valid for the necessary lifetime and `T` as promised by - // the caller of `InnerPrimitiveMut::new`. - unsafe { PrimitiveMut::from_inner(Private, inner) } - } -} - -impl ProxiedWithRawOptionalVTable for T { - type OptionalVTable = PrimitiveOptionalMutVTable; - - fn upcast_vtable( - _private: Private, - optional_vtable: &'static Self::OptionalVTable, - ) -> &'static Self::VTable { - &optional_vtable.base - } -} - -impl RawVTableMutator<'_, T> { - pub(crate) fn get(self) -> T { - // SAFETY: - // - `msg_ref` is valid for the lifetime of `RawVTableMutator` as promised by - // the caller of `new`. - unsafe { (self.vtable().getter)(self.msg_ref.msg()) } - } - - /// # Safety - /// - `msg_ref` must be valid for the lifetime of `RawVTableMutator`. - pub(crate) unsafe fn set(self, val: T) { - // SAFETY: - // - `msg_ref` is valid for the lifetime of `RawVTableMutator` as promised by - // the caller of `new`. - unsafe { (self.vtable().setter)(self.msg_ref.msg(), val) } - } -} - -impl<'msg, T: PrimitiveWithRawVTable> RawVTableOptionalMutatorData<'msg, T> { - pub fn set_absent_to_default(self, private: Private) -> Self { - // SAFETY: - // - `msg_ref` is valid for the lifetime of `RawVTableOptionalMutatorData` as - // promised by the caller of `new`. - self.set(private, self.optional_vtable().default) - } - - pub fn set(self, _private: Private, val: T) -> Self { - // SAFETY: - // - `msg_ref` is valid for the lifetime of `RawVTableOptionalMutatorData` as - // promised by the caller of `new`. - unsafe { (self.optional_vtable().base.setter)(self.msg_ref.msg(), val) } - self - } - - pub fn clear(self, _private: Private) -> Self { - // SAFETY: - // - `msg_ref` is valid for the lifetime of `RawVTableOptionalMutatorData` as - // promised by the caller of `new`. - unsafe { (self.optional_vtable().clearer)(self.msg_ref.msg()) } - self - } -} diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 5ce270b54f..521dcd3ce3 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -395,10 +395,6 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { type View<'a> = $name$; } - impl $pb$::MutProxied for $name$ { - type Mut<'a> = $pb$::PrimitiveMut<'a, $name$>; - } - impl $pb$::ViewProxy<'_> for $name$ { type Proxied = $name$; @@ -411,33 +407,6 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } } - impl $pb$::SettableValue<$name$> for $name$ { - fn set_on<'msg>( - self, - private: $pbi$::Private, - mut mutator: $pb$::Mut<'msg, $name$> - ) where $name$: 'msg { - mutator.set_primitive(private, self) - } - } - - impl $pb$::ProxiedWithPresence for $name$ { - type PresentMutData<'a> = $pbi$::RawVTableOptionalMutatorData<'a, $name$>; - type AbsentMutData<'a> = $pbi$::RawVTableOptionalMutatorData<'a, $name$>; - - fn clear_present_field( - present_mutator: Self::PresentMutData<'_>, - ) -> Self::AbsentMutData<'_> { - present_mutator.clear($pbi$::Private) - } - - fn set_absent_to_default( - absent_mutator: Self::AbsentMutData<'_>, - ) -> Self::PresentMutData<'_> { - absent_mutator.set_absent_to_default($pbi$::Private) - } - } - unsafe impl $pb$::ProxiedInRepeated for $name$ { fn repeated_len(r: $pb$::View<$pb$::Repeated>) -> usize { $pbr$::cast_enum_repeated_view($pbi$::Private, r).len() @@ -485,8 +454,6 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } } - impl $pbi$::PrimitiveWithRawVTable for $name$ {} - // SAFETY: this is an enum type unsafe impl $pbi$::Enum for $name$ { const NAME: &'static str = "$name$"; From f9ed22055e42c8d5c7ca013e0217ff38e428c117 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 29 Apr 2024 04:13:53 -0700 Subject: [PATCH 053/179] Remove MutProxied for bytes/string fields. This change then also makes the BytesMut/ProtoStrMut types unused. It removes them and related code. PiperOrigin-RevId: 629023886 --- rust/internal.rs | 4 +- rust/shared.rs | 2 +- rust/string.rs | 402 ----------------------------------------------- rust/vtable.rs | 130 +-------------- 4 files changed, 4 insertions(+), 534 deletions(-) diff --git a/rust/internal.rs b/rust/internal.rs index 742423bc10..d9fa408ff1 100644 --- a/rust/internal.rs +++ b/rust/internal.rs @@ -14,8 +14,8 @@ pub use paste::paste; pub use crate::r#enum::Enum; pub use crate::vtable::{ - new_vtable_field_entry, BytesMutVTable, BytesOptionalMutVTable, ProxiedWithRawOptionalVTable, - ProxiedWithRawVTable, RawVTableMutator, RawVTableOptionalMutatorData, + new_vtable_field_entry, ProxiedWithRawOptionalVTable, ProxiedWithRawVTable, RawVTableMutator, + RawVTableOptionalMutatorData, }; pub use crate::ProtoStr; diff --git a/rust/shared.rs b/rust/shared.rs index 8ea0684a39..c4aaef7906 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -33,7 +33,7 @@ pub mod __public { pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, }; - pub use crate::string::{BytesMut, ProtoStr, ProtoStrMut}; + pub use crate::string::ProtoStr; pub use crate::ParseError; } pub use __public::*; diff --git a/rust/string.rs b/rust/string.rs index 9711f87bf1..ed1cfdc3ba 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -26,121 +26,10 @@ use std::iter; use std::ops::{Deref, DerefMut}; use utf8::Utf8Chunks; -/// A mutator for `bytes` fields - this type is `protobuf::Mut<'msg, [u8]>`. -/// -/// This type implements `Deref`, so many operations are -/// provided through that, including indexing and slicing. -/// -/// Conceptually, this type is like a `&'msg mut &'msg str`, though the actual -/// implementation is dependent on runtime and `'msg` is covariant. -/// -/// Unlike `Vec`, this type has no in-place concatenation functions like -/// `extend_from_slice`. -/// -/// `BytesMut` is not intended to be grown and reallocated like a `Vec`. It's -/// recommended to instead build a `Vec` or `String` and pass that directly -/// to `set`, which will reuse the allocation if supported by the runtime. -#[derive(Debug)] -pub struct BytesMut<'msg> { - inner: InnerBytesMut<'msg>, -} - -// SAFETY: -// - Protobuf Rust messages don't allow shared mutation across threads. -// - Protobuf Rust messages don't share arenas. -// - All access that touches an arena occurs behind a `&mut`. -// - All mutators that store an arena are `!Send`. -unsafe impl Sync for BytesMut<'_> {} - -impl<'msg> BytesMut<'msg> { - /// Constructs a new `BytesMut` from its internal, runtime-dependent part. - #[doc(hidden)] - pub fn from_inner(_private: Private, inner: InnerBytesMut<'msg>) -> Self { - Self { inner } - } - - /// Gets the current value of the field. - pub fn get(&self) -> &[u8] { - self.as_view() - } - - /// Sets the byte string to the given `val`, cloning any borrowed data. - /// - /// This method accepts both owned and borrowed byte strings; if the runtime - /// supports it, an owned value will not reallocate when setting the - /// string. - pub fn set(&mut self, val: impl SettableValue<[u8]>) { - val.set_on(Private, MutProxy::as_mut(self)) - } - - /// Truncates the byte string. - /// - /// Has no effect if `new_len` is larger than the current `len`. - pub fn truncate(&mut self, new_len: usize) { - self.inner.truncate(new_len) - } - - /// Clears the byte string to the empty string. - /// - /// # Compared with `FieldEntry::clear` - /// - /// Note that this is different than marking an `optional bytes` field as - /// absent; if these `bytes` are in an `optional`, `FieldEntry::is_set` - /// will still return `true` after this method is invoked. - /// - /// This also means that if the field has a non-empty default, - /// `BytesMut::clear` results in the accessor returning an empty string - /// while `FieldEntry::clear` results in the non-empty default. - /// - /// However, for a proto3 `bytes` that has implicit presence, there is no - /// distinction between these states: unset `bytes` is the same as empty - /// `bytes` and the default is always the empty string. - /// - /// In the C++ API, this is the difference between `msg.clear_bytes_field()` - /// and `msg.mutable_bytes_field()->clear()`. - /// - /// Having the same name and signature as `FieldEntry::clear` makes code - /// that calls `field_mut().clear()` easier to migrate from implicit - /// to explicit presence. - pub fn clear(&mut self) { - self.truncate(0); - } -} - -impl Deref for BytesMut<'_> { - type Target = [u8]; - fn deref(&self) -> &[u8] { - self.as_ref() - } -} - -impl AsRef<[u8]> for BytesMut<'_> { - fn as_ref(&self) -> &[u8] { - unsafe { self.inner.get() } - } -} - impl Proxied for [u8] { type View<'msg> = &'msg [u8]; } -impl MutProxied for [u8] { - type Mut<'msg> = BytesMut<'msg>; -} - -impl ProxiedWithPresence for [u8] { - type PresentMutData<'msg> = BytesPresentMutData<'msg>; - type AbsentMutData<'msg> = BytesAbsentMutData<'msg>; - - fn clear_present_field(present_mutator: Self::PresentMutData<'_>) -> Self::AbsentMutData<'_> { - present_mutator.clear() - } - - fn set_absent_to_default(absent_mutator: Self::AbsentMutData<'_>) -> Self::PresentMutData<'_> { - absent_mutator.set_absent_to_default() - } -} - impl<'msg> ViewProxy<'msg> for &'msg [u8] { type Proxied = [u8]; @@ -156,47 +45,6 @@ impl<'msg> ViewProxy<'msg> for &'msg [u8] { } } -impl<'msg> ViewProxy<'msg> for BytesMut<'msg> { - type Proxied = [u8]; - - fn as_view(&self) -> &[u8] { - self.as_ref() - } - - fn into_view<'shorter>(self) -> &'shorter [u8] - where - 'msg: 'shorter, - { - self.inner.get() - } -} - -impl<'msg> MutProxy<'msg> for BytesMut<'msg> { - fn as_mut(&mut self) -> BytesMut<'_> { - BytesMut { inner: self.inner } - } - - fn into_mut<'shorter>(self) -> BytesMut<'shorter> - where - 'msg: 'shorter, - { - BytesMut { inner: self.inner } - } -} - -impl Hash for BytesMut<'_> { - fn hash(&self, state: &mut H) { - self.deref().hash(state) - } -} - -impl Eq for BytesMut<'_> {} -impl<'msg> Ord for BytesMut<'msg> { - fn cmp(&self, other: &BytesMut<'msg>) -> Ordering { - self.deref().cmp(other.deref()) - } -} - /// The bytes were not valid UTF-8. #[derive(Debug, PartialEq)] pub struct Utf8Error(pub(crate) ()); @@ -422,23 +270,6 @@ impl Proxied for ProtoStr { type View<'msg> = &'msg ProtoStr; } -impl MutProxied for ProtoStr { - type Mut<'msg> = ProtoStrMut<'msg>; -} - -impl ProxiedWithPresence for ProtoStr { - type PresentMutData<'msg> = StrPresentMutData<'msg>; - type AbsentMutData<'msg> = StrAbsentMutData<'msg>; - - fn clear_present_field(present_mutator: Self::PresentMutData<'_>) -> Self::AbsentMutData<'_> { - StrAbsentMutData(present_mutator.0.clear()) - } - - fn set_absent_to_default(absent_mutator: Self::AbsentMutData<'_>) -> Self::PresentMutData<'_> { - StrPresentMutData(absent_mutator.0.set_absent_to_default()) - } -} - impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { type Proxied = ProtoStr; @@ -454,205 +285,6 @@ impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { } } -/// Non-exported newtype for `ProxiedWithPresence::PresentData` -#[derive(Debug)] -pub struct StrPresentMutData<'msg>(BytesPresentMutData<'msg>); - -impl<'msg> ViewProxy<'msg> for StrPresentMutData<'msg> { - type Proxied = ProtoStr; - - fn as_view(&self) -> View<'_, ProtoStr> { - // SAFETY: The `ProtoStr` API guards against non-UTF-8 data. The runtime does - // not require `ProtoStr` to be UTF-8 if it could be mutated outside of these - // guards, such as through FFI. - unsafe { ProtoStr::from_utf8_unchecked(self.0.as_view()) } - } - - fn into_view<'shorter>(self) -> View<'shorter, ProtoStr> - where - 'msg: 'shorter, - { - // SAFETY: The `ProtoStr` API guards against non-UTF-8 data. The runtime does - // not require `ProtoStr` to be UTF-8 if it could be mutated outside of these - // guards, such as through FFI. - unsafe { ProtoStr::from_utf8_unchecked(self.0.into_view()) } - } -} - -impl<'msg> MutProxy<'msg> for StrPresentMutData<'msg> { - fn as_mut(&mut self) -> Mut<'_, ProtoStr> { - ProtoStrMut { bytes: self.0.as_mut() } - } - - fn into_mut<'shorter>(self) -> Mut<'shorter, ProtoStr> - where - 'msg: 'shorter, - { - ProtoStrMut { bytes: self.0.into_mut() } - } -} - -/// Non-exported newtype for `ProxiedWithPresence::AbsentData` -#[derive(Debug)] -pub struct StrAbsentMutData<'msg>(BytesAbsentMutData<'msg>); - -impl<'msg> ViewProxy<'msg> for StrAbsentMutData<'msg> { - type Proxied = ProtoStr; - - fn as_view(&self) -> View<'_, ProtoStr> { - // SAFETY: The `ProtoStr` API guards against non-UTF-8 data. The runtime does - // not require `ProtoStr` to be UTF-8 if it could be mutated outside of these - // guards, such as through FFI. - unsafe { ProtoStr::from_utf8_unchecked(self.0.as_view()) } - } - - fn into_view<'shorter>(self) -> View<'shorter, ProtoStr> - where - 'msg: 'shorter, - { - // SAFETY: The `ProtoStr` API guards against non-UTF-8 data. The runtime does - // not require `ProtoStr` to be UTF-8 if it could be mutated outside of these - // guards, such as through FFI. - unsafe { ProtoStr::from_utf8_unchecked(self.0.into_view()) } - } -} - -#[derive(Debug)] -pub struct ProtoStrMut<'msg> { - bytes: BytesMut<'msg>, -} - -impl<'msg> ProtoStrMut<'msg> { - /// Constructs a new `ProtoStrMut` from its internal, runtime-dependent - /// part. - #[doc(hidden)] - pub fn from_inner(_private: Private, inner: InnerBytesMut<'msg>) -> Self { - Self { bytes: BytesMut { inner } } - } - - /// Converts a `bytes` `FieldEntry` into a `string` one. Used by gencode. - #[doc(hidden)] - pub fn field_entry_from_bytes( - _private: Private, - field_entry: FieldEntry<'_, [u8]>, - ) -> FieldEntry { - match field_entry { - Optional::Set(present) => { - Optional::Set(PresentField::from_inner(Private, StrPresentMutData(present.inner))) - } - Optional::Unset(absent) => { - Optional::Unset(AbsentField::from_inner(Private, StrAbsentMutData(absent.inner))) - } - } - } - - /// Gets the current value of the field. - pub fn get(&self) -> &ProtoStr { - self.as_view() - } - - /// Sets the string to the given `val`, cloning any borrowed data. - /// - /// This method accepts both owned and borrowed strings; if the runtime - /// supports it, an owned value will not reallocate when setting the - /// string. - pub fn set(&mut self, val: impl SettableValue) { - val.set_on(Private, MutProxy::as_mut(self)) - } - - /// Truncates the string. - /// - /// Has no effect if `new_len` is larger than the current `len`. - /// - /// If `new_len` does not lie on a UTF-8 `char` boundary, behavior is - /// runtime-dependent. If this occurs, the runtime may: - /// - /// - Panic - /// - Truncate the string further to be on a `char` boundary. - /// - Truncate to `new_len`, resulting in a `ProtoStr` with a non-UTF8 tail. - pub fn truncate(&mut self, new_len: usize) { - self.bytes.truncate(new_len) - } - - /// Clears the string, setting it to the empty string. - /// - /// # Compared with `FieldEntry::clear` - /// - /// Note that this is different than marking an `optional string` field as - /// absent; if this cleared `string` is in an `optional`, - /// `FieldEntry::is_set` will still return `true` after this method is - /// invoked. - /// - /// This also means that if the field has a non-empty default, - /// `ProtoStrMut::clear` results in the accessor returning an empty string - /// while `FieldEntry::clear` results in the non-empty default. - /// - /// However, for a proto3 `string` that has implicit presence, there is no - /// distinction between these states: unset `string` is the same as empty - /// `string` and the default is always the empty string. - /// - /// In the C++ API, this is the difference between - /// `msg.clear_string_field()` - /// and `msg.mutable_string_field()->clear()`. - /// - /// Having the same name and signature as `FieldEntry::clear` makes code - /// that calls `field_mut().clear()` easier to migrate from implicit - /// to explicit presence. - pub fn clear(&mut self) { - self.truncate(0); - } -} - -impl Deref for ProtoStrMut<'_> { - type Target = ProtoStr; - fn deref(&self) -> &ProtoStr { - self.as_view() - } -} - -impl AsRef for ProtoStrMut<'_> { - fn as_ref(&self) -> &ProtoStr { - self.as_view() - } -} - -impl AsRef<[u8]> for ProtoStrMut<'_> { - fn as_ref(&self) -> &[u8] { - self.as_view().as_bytes() - } -} - -impl<'msg> ViewProxy<'msg> for ProtoStrMut<'msg> { - type Proxied = ProtoStr; - - fn as_view(&self) -> &ProtoStr { - // SAFETY: The `ProtoStr` API guards against non-UTF-8 data. The runtime does - // not require `ProtoStr` to be UTF-8 if it could be mutated outside of these - // guards, such as through FFI. - unsafe { ProtoStr::from_utf8_unchecked(self.bytes.as_view()) } - } - - fn into_view<'shorter>(self) -> &'shorter ProtoStr - where - 'msg: 'shorter, - { - unsafe { ProtoStr::from_utf8_unchecked(self.bytes.into_view()) } - } -} - -impl<'msg> MutProxy<'msg> for ProtoStrMut<'msg> { - fn as_mut(&mut self) -> ProtoStrMut<'_> { - ProtoStrMut { bytes: BytesMut { inner: self.bytes.inner } } - } - - fn into_mut<'shorter>(self) -> ProtoStrMut<'shorter> - where - 'msg: 'shorter, - { - ProtoStrMut { bytes: BytesMut { inner: self.bytes.inner } } - } -} - // TODO: remove after IntoProxied has been implemented for // ProtoStr. impl AsRef for String { @@ -677,19 +309,6 @@ impl AsRef for &ProtoStr { } } -impl Hash for ProtoStrMut<'_> { - fn hash(&self, state: &mut H) { - self.deref().hash(state) - } -} - -impl Eq for ProtoStrMut<'_> {} -impl<'msg> Ord for ProtoStrMut<'msg> { - fn cmp(&self, other: &ProtoStrMut<'msg>) -> Ordering { - self.deref().cmp(other.deref()) - } -} - /// Implements `PartialCmp` and `PartialEq` for the `lhs` against the `rhs` /// using `AsRef<[u8]>`. // TODO: consider improving to not require a `<()>` if no generics are @@ -712,33 +331,12 @@ macro_rules! impl_bytes_partial_cmp { } impl_bytes_partial_cmp!( - // Should `BytesMut` compare with `str` and `ProtoStr[Mut]` with `[u8]`? - // `[u8]` and `str` do not compare with each other in the stdlib. - - // `BytesMut` against protobuf types - <('a, 'b)> BytesMut<'a> => BytesMut<'b>, - - // `BytesMut` against foreign types - <('a)> BytesMut<'a> => [u8], - <('a)> [u8] => BytesMut<'a>, - <('a, const N: usize)> BytesMut<'a> => [u8; N], - <('a, const N: usize)> [u8; N] => BytesMut<'a>, - // `ProtoStr` against protobuf types <()> ProtoStr => ProtoStr, - <('a)> ProtoStr => ProtoStrMut<'a>, // `ProtoStr` against foreign types <()> ProtoStr => str, <()> str => ProtoStr, - - // `ProtoStrMut` against protobuf types - <('a, 'b)> ProtoStrMut<'a> => ProtoStrMut<'b>, - <('a)> ProtoStrMut<'a> => ProtoStr, - - // `ProtoStrMut` against foreign types - <('a)> ProtoStrMut<'a> => str, - <('a)> str => ProtoStrMut<'a>, ); #[cfg(test)] diff --git a/rust/vtable.rs b/rust/vtable.rs index 27f14db922..83d6c0592c 100644 --- a/rust/vtable.rs +++ b/rust/vtable.rs @@ -6,9 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd use crate::__internal::Private; -use crate::__runtime::{ - copy_bytes_in_arena_if_needed_by_runtime, MutatorMessageRef, PtrAndLen, RawMessage, -}; +use crate::__runtime::MutatorMessageRef; use crate::{ AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, ProxiedWithPresence, View, ViewProxy, @@ -257,129 +255,3 @@ impl<'msg, T: ProxiedWithRawOptionalVTable + ?Sized + 'msg> MutProxy<'msg> T::make_mut(Private, self.into_raw_mut()) } } - -impl ProxiedWithRawVTable for [u8] { - type VTable = BytesMutVTable; - - fn make_view(_private: Private, mut_inner: RawVTableMutator<'_, Self>) -> View<'_, Self> { - mut_inner.get() - } - - fn make_mut(_private: Private, inner: RawVTableMutator<'_, Self>) -> Mut<'_, Self> { - crate::string::BytesMut::from_inner(Private, inner) - } -} - -impl ProxiedWithRawOptionalVTable for [u8] { - type OptionalVTable = BytesOptionalMutVTable; - fn upcast_vtable( - _private: Private, - optional_vtable: &'static Self::OptionalVTable, - ) -> &'static Self::VTable { - &optional_vtable.base - } -} - -/// A generic thunk vtable for mutating a present `bytes` or `string` field. -#[doc(hidden)] -#[derive(Debug)] -pub struct BytesMutVTable { - pub(crate) setter: unsafe extern "C" fn(msg: RawMessage, val: PtrAndLen), - pub(crate) getter: unsafe extern "C" fn(msg: RawMessage) -> PtrAndLen, -} - -/// A generic thunk vtable for mutating an `optional` `bytes` or `string` field. -#[derive(Debug)] -pub struct BytesOptionalMutVTable { - pub(crate) base: BytesMutVTable, - pub(crate) clearer: unsafe extern "C" fn(msg: RawMessage), - pub(crate) default: &'static [u8], -} - -impl BytesMutVTable { - #[doc(hidden)] - pub const fn new( - _private: Private, - getter: unsafe extern "C" fn(msg: RawMessage) -> PtrAndLen, - setter: unsafe extern "C" fn(msg: RawMessage, val: PtrAndLen), - ) -> Self { - Self { getter, setter } - } -} - -impl BytesOptionalMutVTable { - /// # Safety - /// The `default` value must be UTF-8 if required by - /// the runtime and this is for a `string` field. - #[doc(hidden)] - pub const unsafe fn new( - _private: Private, - getter: unsafe extern "C" fn(msg: RawMessage) -> PtrAndLen, - setter: unsafe extern "C" fn(msg: RawMessage, val: PtrAndLen), - clearer: unsafe extern "C" fn(msg: RawMessage), - default: &'static [u8], - ) -> Self { - Self { base: BytesMutVTable { getter, setter }, clearer, default } - } -} - -impl<'msg> RawVTableMutator<'msg, [u8]> { - pub(crate) fn get(self) -> &'msg [u8] { - // SAFETY: - // - `msg_ref` is valid for `'msg` as promised by the caller of `new`. - // - The caller of `BytesMutVTable` promised that the returned `PtrAndLen` is - // valid for `'msg`. - unsafe { (self.vtable().getter)(self.msg_ref.msg()).as_ref() } - } - - /// # Safety - /// - `msg_ref` must be valid for `'msg` - /// - If this is for a `string` field, `val` must be valid UTF-8 if the - /// runtime requires it. - pub(crate) unsafe fn set(self, val: &[u8]) { - let val = copy_bytes_in_arena_if_needed_by_runtime(self.msg_ref, val); - // SAFETY: - // - `msg_ref` is valid for `'msg` as promised by the caller of `new`. - unsafe { (self.vtable().setter)(self.msg_ref.msg(), val.into()) } - } - - pub(crate) fn truncate(&self, len: usize) { - if len == 0 { - // SAFETY: The empty string is valid UTF-8. - unsafe { - self.set(b""); - } - return; - } - todo!("b/294252563") - } -} - -impl<'msg> RawVTableOptionalMutatorData<'msg, [u8]> { - /// Sets an absent `bytes`/`string` field to its default value. - pub(crate) fn set_absent_to_default(self) -> Self { - // SAFETY: The default value is UTF-8 if required by the - // runtime as promised by the caller of `BytesOptionalMutVTable::new`. - unsafe { self.set(self.optional_vtable().default) } - } - - /// # Safety - /// - If this is a `string` field, `val` must be valid UTF-8 if required by - /// the runtime. - pub(crate) unsafe fn set(self, val: &[u8]) -> Self { - let val = copy_bytes_in_arena_if_needed_by_runtime(self.msg_ref, val); - // SAFETY: - // - `msg_ref` is valid for `'msg` as promised by the caller. - unsafe { (self.optional_vtable().base.setter)(self.msg_ref.msg(), val.into()) } - self - } - - pub(crate) fn clear(self) -> Self { - // SAFETY: - // - `msg_ref` is valid for `'msg` as promised by the caller. - // - The caller of `new` promised that the returned `PtrAndLen` is valid for - // `'msg`. - unsafe { (self.optional_vtable().clearer)(self.msg_ref.msg()) } - self - } -} From b7a145df95315e5a83098c662b79eeea8a5fb9d3 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 29 Apr 2024 05:50:45 -0700 Subject: [PATCH 054/179] Implement _mut() accessors for messages without FieldEntry PiperOrigin-RevId: 629041337 --- .../rust/accessors/singular_message.cc | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/google/protobuf/compiler/rust/accessors/singular_message.cc b/src/google/protobuf/compiler/rust/accessors/singular_message.cc index 347f1d5dcd..c5f77c2ca2 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_message.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_message.cc @@ -69,39 +69,34 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, } )rs"); }}, - {"getter_mut", + {"getter_mut_body", [&] { - if (accessor_case == AccessorCase::VIEW) { - return; - } - ctx.Emit({}, R"rs( - pub fn $raw_field_name$_mut(&mut self) -> $msg_type$Mut<'_> { - self.$raw_field_name$_entry().or_default() - } + if (ctx.is_cpp()) { + ctx.Emit({}, R"rs( + let raw_msg = unsafe { $getter_mut_thunk$(self.raw_msg()) }; + $msg_type$Mut::from_parent($pbi$::Private, + self.as_mutator_message_ref($pbi$::Private), raw_msg) + )rs"); + } else { + ctx.Emit({}, R"rs( + let raw_msg = unsafe { + $getter_mut_thunk$(self.raw_msg(), self.arena().raw()) + }; + $msg_type$Mut::from_parent($pbi$::Private, + self.as_mutator_message_ref($pbi$::Private), raw_msg) )rs"); + } }}, - {"private_getter_entry", + {"getter_mut", [&] { if (accessor_case == AccessorCase::VIEW) { return; } + ctx.Emit({}, R"rs( - fn $raw_field_name$_entry(&mut self) - -> $pb$::FieldEntry<'_, $msg_type$> { - static VTABLE: $pbr$::MessageVTable = - $pbr$::MessageVTable::new($pbi$::Private, - $getter_thunk$, - $getter_mut_thunk$, - $clearer_thunk$); - unsafe { - let has = self.has_$raw_field_name$(); - $pbi$::new_vtable_field_entry($pbi$::Private, - self.as_mutator_message_ref($pbi$::Private), - &VTABLE, - has) - } - } - )rs"); + pub fn $raw_field_name$_mut(&mut self) -> $msg_type$Mut<'_> { + $getter_mut_body$ + })rs"); }}, {"getter_opt", [&] { @@ -172,7 +167,6 @@ void SingularMessage::InMsgImpl(Context& ctx, const FieldDescriptor& field, R"rs( $getter$ $getter_mut$ - $private_getter_entry$ $getter_opt$ $setter$ $hazzer$ From 501067aa8095022bb69c9cafbcffc433caad09ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Mon, 29 Apr 2024 12:27:30 -0700 Subject: [PATCH 055/179] Expose `MiniDescriptorEncode` for `FieldDefPtr`. We already expose equivalent methods for `MessageDefPtr` and `EnumDefPtr`. PiperOrigin-RevId: 629150393 --- upb/reflection/def.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/upb/reflection/def.hpp b/upb/reflection/def.hpp index cf28a38ff9..452ddc6e32 100644 --- a/upb/reflection/def.hpp +++ b/upb/reflection/def.hpp @@ -61,6 +61,13 @@ class FieldDefPtr { return upb_FieldDef_MiniTable(ptr_); } + std::string MiniDescriptorEncode() const { + upb::Arena arena; + upb_StringView md; + upb_FieldDef_MiniDescriptorEncode(ptr_, arena.ptr(), &md); + return std::string(md.data, md.size); + } + const UPB_DESC(FieldOptions) * options() const { return upb_FieldDef_Options(ptr_); } From f2e1ad315212eccb9cb457a8ff75c7579dcf964c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 29 Apr 2024 13:04:33 -0700 Subject: [PATCH 056/179] Internal change. PiperOrigin-RevId: 629161453 --- src/google/protobuf/text_format_unittest.cc | 57 +++++++++++++-------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/src/google/protobuf/text_format_unittest.cc b/src/google/protobuf/text_format_unittest.cc index 02b1d38167..886d65695a 100644 --- a/src/google/protobuf/text_format_unittest.cc +++ b/src/google/protobuf/text_format_unittest.cc @@ -93,7 +93,20 @@ constexpr absl::string_view kEscapeTestStringEscaped = constexpr absl::string_view value_replacement = "\\[REDACTED\\]"; -class TextFormatTest : public testing::Test { +class TextFormatTestBase : public testing::Test { + public: + void SetUp() override { + single_line_debug_format_prefix_ = ""; + multi_line_debug_format_prefix_ = proto_.DebugString(); + } + + protected: + unittest::TestAllTypes proto_; + std::string single_line_debug_format_prefix_; + std::string multi_line_debug_format_prefix_; +}; + +class TextFormatTest : public TextFormatTestBase { public: static void SetUpTestSuite() { ABSL_CHECK_OK(File::GetContents( @@ -108,8 +121,6 @@ class TextFormatTest : public testing::Test { protected: // Text format read from text_format_unittest_data.txt. const std::string proto_text_format_; - unittest::TestAllTypes proto_; - private: static std::string static_proto_text_format_; }; @@ -157,8 +168,8 @@ TEST_F(TextFormatTest, ShortDebugString) { proto_.mutable_optional_foreign_message(); EXPECT_EQ(proto_.ShortDebugString(), - absl::StrCat("optional_int32: ", kDebugStringSilentMarker, - "1 " + absl::StrCat(single_line_debug_format_prefix_, + "optional_int32: 1 " "optional_string: \"hello\" " "optional_nested_message { bb: 2 } " "optional_foreign_message { }")); @@ -337,8 +348,8 @@ TEST_F(TextFormatTest, StringEscape) { // Hardcode a correct value to test against. std::string correct_string = - absl::StrCat("optional_string: ", kDebugStringSilentMarker, - kEscapeTestStringEscaped, "\n"); + absl::StrCat(multi_line_debug_format_prefix_, + "optional_string: ", kEscapeTestStringEscaped, "\n"); // Compare. EXPECT_EQ(correct_string, debug_string); @@ -346,8 +357,9 @@ TEST_F(TextFormatTest, StringEscape) { // the protocol buffer contains no UTF-8 text. EXPECT_EQ(correct_string, utf8_debug_string); - std::string expected_short_debug_string = absl::StrCat( - "optional_string: ", kDebugStringSilentMarker, kEscapeTestStringEscaped); + std::string expected_short_debug_string = + absl::StrCat(single_line_debug_format_prefix_, + "optional_string: ", kEscapeTestStringEscaped); EXPECT_EQ(expected_short_debug_string, proto_.ShortDebugString()); } @@ -362,12 +374,12 @@ TEST_F(TextFormatTest, Utf8DebugString) { // Hardcode a correct value to test against. std::string correct_utf8_string = - absl::StrCat("optional_string: ", kDebugStringSilentMarker, - "\"\350\260\267\346\255\214\"\n" + absl::StrCat(multi_line_debug_format_prefix_, + "optional_string: \"\350\260\267\346\255\214\"\n" "optional_bytes: \"\\350\\260\\267\\346\\255\\214\"\n"); std::string correct_string = - absl::StrCat("optional_string: ", kDebugStringSilentMarker, - "\"\\350\\260\\267\\346\\255\\214\"\n" + absl::StrCat(multi_line_debug_format_prefix_, + "optional_string: \"\\350\\260\\267\\346\\255\\214\"\n" "optional_bytes: \"\\350\\260\\267\\346\\255\\214\"\n"); // Compare. @@ -404,8 +416,9 @@ TEST_F(TextFormatTest, PrintUnknownFields) { unknown_fields->AddVarint(8, 2); unknown_fields->AddVarint(8, 3); - EXPECT_EQ(absl::StrCat("5: ", kDebugStringSilentMarker, - "1\n" + std::string message_text; + TextFormat::PrintToString(message, &message_text); + EXPECT_EQ(absl::StrCat("5: 1\n" "5: 0x00000002\n" "5: 0x0000000000000003\n" "5: \"4\"\n" @@ -415,7 +428,7 @@ TEST_F(TextFormatTest, PrintUnknownFields) { "8: 1\n" "8: 2\n" "8: 3\n"), - message.DebugString()); + message_text); EXPECT_THAT(absl::StrCat(message), testing::MatchesRegex(absl::Substitute( "5: UNKNOWN_VARINT $0\n" @@ -1012,8 +1025,8 @@ TEST_F(TextFormatTest, PrintUnknownEnumFieldProto3) { proto.add_repeated_nested_enum( static_cast(-2147483648)); - EXPECT_EQ(absl::StrCat("repeated_nested_enum: ", kDebugStringSilentMarker, - "10\n" + EXPECT_EQ(absl::StrCat(multi_line_debug_format_prefix_, + "repeated_nested_enum: 10\n" "repeated_nested_enum: -10\n" "repeated_nested_enum: 2147483647\n" "repeated_nested_enum: -2147483648\n"), @@ -1464,8 +1477,8 @@ TEST_F(TextFormatTest, PrintExotic) { // have this problem, so we switched to that instead. EXPECT_EQ( - absl::StrCat("repeated_int64: ", kDebugStringSilentMarker, - "-9223372036854775808\n" + absl::StrCat(multi_line_debug_format_prefix_, + "repeated_int64: -9223372036854775808\n" "repeated_uint64: 18446744073709551615\n" "repeated_double: 123.456\n" "repeated_double: 1.23e+21\n" @@ -1524,8 +1537,8 @@ TEST_F(TextFormatTest, PrintFloatPrecision) { message.add_repeated_double(1.2345678987654e100); message.add_repeated_double(1.23456789876543e100); - EXPECT_EQ(absl::StrCat("repeated_float: ", kDebugStringSilentMarker, - "1\n" + EXPECT_EQ(absl::StrCat(multi_line_debug_format_prefix_, + "repeated_float: 1\n" "repeated_float: 1.2\n" "repeated_float: 1.23\n" "repeated_float: 1.234\n" From e9c9970eb4ac488d9fb94e8dea6945d3881c405d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Mon, 29 Apr 2024 16:25:34 -0700 Subject: [PATCH 057/179] In the Kotlin DSL, reference builder methods with property syntax. Instead of `_builder.getFoo()` and `_builder.setFoo(value)`, the generated code now uses `_builder.foo` and `_builder.foo = value`. When compiling against the Java Proto API, this makes no difference, since the Kotlin compiler treats Java methods `getFoo`/`setFoo` as defining a Kotlin property `foo`. But if a compatible proto API is implemented in Kotlin then there is no such equivalence. Such an implementation would have to define either both forms or just the one that the DSL uses. For a Kotlin API it is more natural to use properties. Similarly, the generated Java methods `getFoosList()`, `getFoosCount()`, `getFoosMap()`, and `getEnumValue()` are accessed via property names (`foosList`, `foosCount`, `foosMap`, `enumValue`). PiperOrigin-RevId: 629220383 --- .../kotlin/com/google/protobuf/Proto2Test.kt | 28 +++++++--- .../kotlin/com/google/protobuf/Proto3Test.kt | 14 ++++- .../google/protobuf/evil_names_proto2.proto | 9 +++ .../google/protobuf/evil_names_proto3.proto | 6 ++ src/google/protobuf/compiler/java/BUILD.bazel | 1 + .../protobuf/compiler/java/field_common.cc | 56 +++++++++++++++++++ .../compiler/java/immutable/enum_field.cc | 10 ++-- .../compiler/java/immutable/map_field.cc | 2 +- .../compiler/java/immutable/message_field.cc | 6 +- .../java/immutable/primitive_field.cc | 31 +++++++--- .../compiler/java/immutable/string_field.cc | 6 +- .../protobuf/compiler/java/lite/enum_field.cc | 10 ++-- .../protobuf/compiler/java/lite/map_field.cc | 2 +- .../compiler/java/lite/message_field.cc | 6 +- .../compiler/java/lite/primitive_field.cc | 38 ++++++++----- .../compiler/java/lite/string_field.cc | 6 +- 16 files changed, 174 insertions(+), 57 deletions(-) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt index c9fee8a78f..7c319bfaac 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt @@ -225,7 +225,7 @@ class Proto2Test { TestAllTypesKt.repeatedGroup { a = 1 }, TestAllTypesKt.repeatedGroup { a = 2 }, TestAllTypesKt.repeatedGroup { a = 3 }, - TestAllTypesKt.repeatedGroup { a = 4 } + TestAllTypesKt.repeatedGroup { a = 4 }, ) ) repeatedGroup[0] = TestAllTypesKt.repeatedGroup { a = 5 } @@ -235,7 +235,7 @@ class Proto2Test { TestAllTypesKt.repeatedGroup { a = 5 }, TestAllTypesKt.repeatedGroup { a = 2 }, TestAllTypesKt.repeatedGroup { a = 3 }, - TestAllTypesKt.repeatedGroup { a = 4 } + TestAllTypesKt.repeatedGroup { a = 4 }, ) ) @@ -249,7 +249,7 @@ class Proto2Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) repeatedNestedMessage[0] = nestedMessage { bb = 5 } @@ -259,7 +259,7 @@ class Proto2Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) @@ -548,7 +548,7 @@ class Proto2Test { repeatedGroupExtension { a = 1 }, repeatedGroupExtension { a = 2 }, repeatedGroupExtension { a = 3 }, - repeatedGroupExtension { a = 4 } + repeatedGroupExtension { a = 4 }, ) ) this[UnittestProto.repeatedGroupExtension][0] = repeatedGroupExtension { a = 5 } @@ -558,7 +558,7 @@ class Proto2Test { repeatedGroupExtension { a = 5 }, repeatedGroupExtension { a = 2 }, repeatedGroupExtension { a = 3 }, - repeatedGroupExtension { a = 4 } + repeatedGroupExtension { a = 4 }, ) ) @@ -575,7 +575,7 @@ class Proto2Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) this[UnittestProto.repeatedNestedMessageExtension][0] = nestedMessage { bb = 5 } @@ -585,7 +585,7 @@ class Proto2Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) @@ -757,7 +757,7 @@ class Proto2Test { 1 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO, 2 to Proto2MapEnum.PROTO2_MAP_ENUM_BAR, 3 to Proto2MapEnum.PROTO2_MAP_ENUM_BAZ, - 4 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO + 4 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO, ) ) } @@ -844,6 +844,11 @@ class Proto2Test { cachedSize_ = "foo" serializedSize_ = true by = "foo" + dEPRECATEDFoo = "foo" + DEPRECATEDBar = "foo" + iD = "foo" + aBNotification = "foo" + notDEPRECATEDFoo = "foo" } ) .isEqualTo( @@ -869,6 +874,11 @@ class Proto2Test { .setCachedSize_("foo") .setSerializedSize_(true) .setBy("foo") + .setDEPRECATEDFoo("foo") + .setDEPRECATEDBar("foo") + .setID("foo") + .setABNotification("foo") + .setNotDEPRECATEDFoo("foo") .build() ) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt index 0b270e6e75..72ab8e1f80 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt @@ -81,7 +81,7 @@ class Proto3Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) repeatedNestedMessage[0] = nestedMessage { bb = 5 } @@ -91,7 +91,7 @@ class Proto3Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) @@ -200,6 +200,11 @@ class Proto3Test { pairs["foo"] = 1 LeadingUnderscore = "foo" option = 1 + dEPRECATEDFoo = "foo" + iD = "foo" + aBNotification = "foo" + DEPRECATEDBar = "foo" + notDEPRECATEDFoo = "foo" } ) .isEqualTo( @@ -237,6 +242,11 @@ class Proto3Test { .putPairs("foo", 1) .setLeadingUnderscore("foo") .setOption(1) + .setDEPRECATEDFoo("foo") + .setID("foo") + .setABNotification("foo") + .setDEPRECATEDBar("foo") + .setNotDEPRECATEDFoo("foo") .build() ) diff --git a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto index ba0827422b..f771bf97bf 100644 --- a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto +++ b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto @@ -43,6 +43,15 @@ message EvilNamesProto2 { optional string cached_size = 23; optional bool serialized_size = 24; optional string by = 25; + + optional string DEPRECATED_foo = 26; + optional group DEPRECATED_NavigationImageRequested = 27 { + optional int32 DEPRECATED_FooBar = 28; + } + optional string __DEPRECATED_Bar = 29; + optional string ID = 30; + optional string a_b_notification = 31; + optional string not_DEPRECATED_foo = 32; } message List {} diff --git a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto index 2c9b5a2d68..238bebc06b 100644 --- a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto +++ b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto @@ -56,6 +56,12 @@ message EvilNamesProto3 { oneof _leading_underscore_oneof { int32 option = 34; } + + optional string DEPRECATED_foo = 35; + optional string ID = 36; + optional string a_b_notification = 37; + optional string __DEPRECATED_Bar = 38; + optional string not_DEPRECATED_foo = 39; } message HardKeywordsAllTypesProto3 { diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 39f862f6cc..915b370309 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -144,6 +144,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/src/google/protobuf/compiler/java/field_common.cc b/src/google/protobuf/compiler/java/field_common.cc index 4acafabe36..bb3cd6a748 100644 --- a/src/google/protobuf/compiler/java/field_common.cc +++ b/src/google/protobuf/compiler/java/field_common.cc @@ -1,8 +1,11 @@ #include "google/protobuf/compiler/java/field_common.h" +#include #include +#include "absl/strings/str_cat.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/names.h" #include "google/protobuf/descriptor.h" namespace google { @@ -10,6 +13,8 @@ namespace protobuf { namespace compiler { namespace java { +std::string GetKotlinPropertyName(const FieldDescriptor* descriptor); + void SetCommonFieldVariables( const FieldDescriptor* descriptor, const FieldGeneratorInfo* info, absl::flat_hash_map* variables) { @@ -30,6 +35,11 @@ void SetCommonFieldVariables( (*variables)["kt_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->name, "_") : info->name; + auto kt_property_name = GetKotlinPropertyName(descriptor); + (*variables)["kt_property_name"] = kt_property_name; + (*variables)["kt_safe_name"] = IsForbiddenKotlin(kt_property_name) + ? absl::StrCat("`", kt_property_name, "`") + : kt_property_name; (*variables)["kt_capitalized_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->capitalized_name, "_") : info->capitalized_name; @@ -51,6 +61,52 @@ void SetCommonFieldVariables( } } +// Locale-independent ASCII upper and lower case munging. +static bool IsUpper(char c) { + return static_cast(c - 'A') <= 'Z' - 'A'; +} + +static char ToLower(char c) { return IsUpper(c) ? c - 'A' + 'a' : c; } + +// Returns the name by which the generated Java getters and setters should be +// referenced from Kotlin as properties. In the simplest case, the original name +// is something like `foo_bar`, which gets translated into `getFooBar()` etc, +// and that in turn can be referenced from Kotlin as `fooBar`. +// +// The algorithm for translating proto names into Java getters and setters is +// straightforward. The first letter of each underscore-separated word gets +// uppercased and the underscores are deleted. There are no other changes, so in +// particular if the proto name has a string of capitals then those remain +// as-is. +// +// The algorithm that the Kotlin compiler uses to derive the property name is +// slightly more complicated. If the first character after `get` (etc) is a +// capital and the second isn't, then the property name is just that string with +// its first letter lowercased. So `getFoo` becomes `foo` and `getX` becomes +// `x`. But if there is more than one capital, then all but the last get +// lowercased. So `getHTMLPage` becomes `htmlPage`. If there are only capitals +// then they all get lowercased, so `getID` becomes `id`. +// TODO: move this to a Kotlin-specific location +std::string GetKotlinPropertyName(const FieldDescriptor* descriptor) { + // Find the first non-capital. If it is the second character, then we just + // need to lowercase the first one. Otherwise we need to lowercase everything + // up to but not including the last capital, except that if everything is + // capitals then everything must be lowercased. + std::string capitalized_name = CapitalizedFieldName(descriptor); + std::string kt_property_name = capitalized_name; + size_t first_non_capital; + for (first_non_capital = 0; first_non_capital < capitalized_name.length() && + IsUpper(capitalized_name[first_non_capital]); + first_non_capital++) { + } + size_t stop = first_non_capital; + if (stop > 1 && stop < capitalized_name.length()) stop--; + for (size_t i = 0; i < stop; i++) { + kt_property_name[i] = ToLower(kt_property_name[i]); + } + return kt_property_name; +} + void SetCommonOneofVariables( const FieldDescriptor* descriptor, const OneofGeneratorInfo* info, absl::flat_hash_map* variables) { diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/immutable/enum_field.cc index 2cb734f498..4a64460fb5 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/immutable/enum_field.cc @@ -273,10 +273,10 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); if (SupportUnknownEnumValue(descriptor_)) { @@ -284,10 +284,10 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" + " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" " }\n"); } @@ -1100,7 +1100,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/map_field.cc b/src/google/protobuf/compiler/java/immutable/map_field.cc index 02e85874c0..13e6319b79 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.cc +++ b/src/google/protobuf/compiler/java/immutable/map_field.cc @@ -981,7 +981,7 @@ void ImmutableMapFieldGenerator::GenerateKotlinDslMembers( " @kotlin.jvm.JvmSynthetic\n" " @JvmName(\"get$kt_capitalized_name$Map\")\n" " get() = com.google.protobuf.kotlin.DslMap(\n" - " $kt_dsl_builder$.${$get$capitalized_name$Map$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), diff --git a/src/google/protobuf/compiler/java/immutable/message_field.cc b/src/google/protobuf/compiler/java/immutable/message_field.cc index 1b76fc2f63..31eb0ea858 100644 --- a/src/google/protobuf/compiler/java/immutable/message_field.cc +++ b/src/google/protobuf/compiler/java/immutable/message_field.cc @@ -378,10 +378,10 @@ void ImmutableMessageFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -1376,7 +1376,7 @@ void RepeatedImmutableMessageFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/primitive_field.cc b/src/google/protobuf/compiler/java/immutable/primitive_field.cc index eb31a7b756..3dc402e5c2 100644 --- a/src/google/protobuf/compiler/java/immutable/primitive_field.cc +++ b/src/google/protobuf/compiler/java/immutable/primitive_field.cc @@ -291,14 +291,27 @@ void ImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" - " }\n"); + if (descriptor_->name() == "is_initialized") { + printer->Print( + variables_, + "// TODO: remove this hack; we should access properties\n" + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$get$kt_capitalized_name$$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" + " }\n"); + } else { + printer->Print(variables_, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); + } WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), @@ -848,7 +861,7 @@ void RepeatedImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/string_field.cc b/src/google/protobuf/compiler/java/immutable/string_field.cc index c9ac5f4e7c..df12b05f07 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.cc +++ b/src/google/protobuf/compiler/java/immutable/string_field.cc @@ -372,10 +372,10 @@ void ImmutableStringFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: kotlin.String\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -992,7 +992,7 @@ void RepeatedImmutableStringFieldGenerator::GenerateKotlinDslMembers( "\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); // List.add(String) diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index f2fb340b1b..b9cde7a483 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -308,10 +308,10 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); if (SupportUnknownEnumValue(descriptor_)) { @@ -319,10 +319,10 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" + " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" " }\n"); } @@ -914,7 +914,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 74fc93f347..491523df91 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -857,7 +857,7 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( " @kotlin.jvm.JvmSynthetic\n" " @JvmName(\"get$kt_capitalized_name$Map\")\n" " get() = com.google.protobuf.kotlin.DslMap(\n" - " $kt_dsl_builder$.${$get$capitalized_name$Map$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index 7c5cc3cf77..c46407bf51 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -283,10 +283,10 @@ void ImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -818,7 +818,7 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index 69924c9c83..c9b3a2947a 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -217,10 +217,9 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateMembers( "private static final $field_type$ $bytes_default$ = $default$;\n"); } if (!context_->options().opensource_runtime) { - printer->Print( - variables_, - "@com.google.protobuf.ProtoField(\n" - " isRequired=$required$)\n"); + printer->Print(variables_, + "@com.google.protobuf.ProtoField(\n" + " isRequired=$required$)\n"); if (HasHasbit(descriptor_)) { printer->Print(variables_, "@com.google.protobuf.ProtoPresenceCheckedField(\n" @@ -329,14 +328,27 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void ImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_, context_->options()); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" - " }\n"); + if (descriptor_->name() == "is_initialized") { + printer->Print( + variables_, + "// TODO: b/336400327 - remove this hack; we should access properties\n" + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.get${$$kt_capitalized_name$$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" + " }\n"); + } else { + printer->Print(variables_, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); + } WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), @@ -718,7 +730,7 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index 833f415585..a08bae50b6 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -336,10 +336,10 @@ void ImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: kotlin.String\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -825,7 +825,7 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( "@kotlin.OptIn" "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); // List.add(String) From 208cae7fd7fdfc7fef6eddded07c2c7431bf0d8f Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Mon, 29 Apr 2024 16:48:41 -0700 Subject: [PATCH 058/179] Internal Change PiperOrigin-RevId: 629226391 --- upb/reflection/field_def.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/upb/reflection/field_def.c b/upb/reflection/field_def.c index d5597f4b2e..6a70361b6b 100644 --- a/upb/reflection/field_def.c +++ b/upb/reflection/field_def.c @@ -714,6 +714,11 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } + } else { + if (syntax == kUpb_Syntax_Editions) { + _upb_DefBuilder_Errf(ctx, "Editions proto cannot lack type for field %s", + f->full_name); + } } if (!has_type && has_type_name) { From 6d33efeefe5232a4f598c3f8223c56743912a21c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 30 Apr 2024 00:01:52 +0000 Subject: [PATCH 059/179] Auto-generate files after cl/629226391 --- php/ext/google/protobuf/php-upb.c | 5 +++++ ruby/ext/google/protobuf_c/ruby-upb.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 508b7f46dc..bff21695b8 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -14203,6 +14203,11 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } + } else { + if (syntax == kUpb_Syntax_Editions) { + _upb_DefBuilder_Errf(ctx, "Editions proto cannot lack type for field %s", + f->full_name); + } } if (!has_type && has_type_name) { diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 9578e8e976..caa348f73f 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -13696,6 +13696,11 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } + } else { + if (syntax == kUpb_Syntax_Editions) { + _upb_DefBuilder_Errf(ctx, "Editions proto cannot lack type for field %s", + f->full_name); + } } if (!has_type && has_type_name) { From e181855abef398e61b4c65a37658d3a7527d0de0 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Tue, 30 Apr 2024 03:35:14 -0700 Subject: [PATCH 060/179] Implement flat setters for map and repeated fields. This doesn't yet use the new IntoProxied trait, but is functionaly equivalent with MutProxy::set() and always copies. PiperOrigin-RevId: 629358666 --- rust/test/shared/accessors_map_test.rs | 17 +++++ rust/test/shared/accessors_repeated_test.rs | 14 ++-- .../protobuf/compiler/rust/accessors/map.cc | 14 ++++ .../compiler/rust/accessors/repeated_field.cc | 73 ++++++++++++------- 4 files changed, 85 insertions(+), 33 deletions(-) diff --git a/rust/test/shared/accessors_map_test.rs b/rust/test/shared/accessors_map_test.rs index 0fc153f63b..50341b0c3a 100644 --- a/rust/test/shared/accessors_map_test.rs +++ b/rust/test/shared/accessors_map_test.rs @@ -157,6 +157,23 @@ fn test_bytes_and_string_copied() { assert_that!(msg.map_int32_bytes_mut().get(1).unwrap(), eq(b"world")); } +#[test] +fn test_map_setter() { + let mut msg = TestMap::new(); + msg.map_string_string_mut().insert("hello", "world"); + msg.map_string_string_mut().insert("fizz", "buzz"); + + let mut msg2 = TestMap::new(); + msg2.set_map_string_string(msg.map_string_string()); + assert_that!( + msg2.map_string_string().iter().collect::>(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); +} + macro_rules! generate_map_with_msg_values_tests { ( $(($k_field:ident, $k_nonzero:expr, $k_other:expr $(,)?)),* diff --git a/rust/test/shared/accessors_repeated_test.rs b/rust/test/shared/accessors_repeated_test.rs index 9aee4907ea..99ae45374c 100644 --- a/rust/test/shared/accessors_repeated_test.rs +++ b/rust/test/shared/accessors_repeated_test.rs @@ -55,16 +55,17 @@ macro_rules! generate_repeated_numeric_test { #[test] fn [< test_repeated_ $field _set >]() { let mut msg = TestAllTypes::new(); - let mut mutator = msg.[](); let mut msg2 = TestAllTypes::new(); let mut mutator2 = msg2.[](); for i in 0..5 { mutator2.push(i as $t); } - protobuf::MutProxy::set(&mut mutator, mutator2.as_view()); + msg.[](mutator2.as_view()); + + let view = msg.[](); assert_that!( - mutator.iter().collect::>(), + view.iter().collect::>(), eq(mutator2.iter().collect::>()) ); } @@ -177,15 +178,16 @@ fn test_repeated_enum_accessors() { #[test] fn test_repeated_bool_set() { let mut msg = TestAllTypes::new(); - let mut mutator = msg.repeated_bool_mut(); let mut msg2 = TestAllTypes::new(); let mut mutator2 = msg2.repeated_bool_mut(); for _ in 0..5 { mutator2.push(true); } - protobuf::MutProxy::set(&mut mutator, mutator2.as_view()); - assert_that!(mutator.iter().collect::>(), eq(mutator2.iter().collect::>())); + msg.set_repeated_bool(mutator2.as_view()); + + let view = msg.repeated_bool(); + assert_that!(view.iter().collect::>(), eq(mutator2.iter().collect::>())); } #[test] diff --git a/src/google/protobuf/compiler/rust/accessors/map.cc b/src/google/protobuf/compiler/rust/accessors/map.cc index 75aa5007ee..397d210af9 100644 --- a/src/google/protobuf/compiler/rust/accessors/map.cc +++ b/src/google/protobuf/compiler/rust/accessors/map.cc @@ -42,6 +42,7 @@ void Map::InMsgImpl(Context& ctx, const FieldDescriptor& field, std::string field_name = FieldNameWithCollisionAvoidance(field); ctx.Emit({{"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, // Never r# prefixed {"Key", RsTypePath(ctx, key_type)}, {"Value", RsTypePath(ctx, value_type)}, {"view_lifetime", ViewLifetime(accessor_case)}, @@ -99,10 +100,23 @@ void Map::InMsgImpl(Context& ctx, const FieldDescriptor& field, unsafe { $pb$::MapMut::from_inner($pbi$::Private, inner) } })rs"); } + }}, + {"setter", + [&] { + if (accessor_case == AccessorCase::VIEW) { + return; + } + ctx.Emit({}, R"rs( + pub fn set_$raw_field_name$(&mut self, src: $pb$::MapView<'_, $Key$, $Value$>) { + // TODO: Implement IntoProxied and avoid copying. + self.$field$_mut().copy_from(src); + } + )rs"); }}}, R"rs( $getter$ $getter_mut$ + $setter$ )rs"); } diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index e1e14e646f..11a85852de 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -23,16 +23,19 @@ namespace rust { void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, AccessorCase accessor_case) const { std::string field_name = FieldNameWithCollisionAvoidance(field); - ctx.Emit({{"field", RsSafeName(field_name)}, - {"RsType", RsTypePath(ctx, field)}, - {"view_lifetime", ViewLifetime(accessor_case)}, - {"view_self", ViewReceiver(accessor_case)}, - {"getter_thunk", ThunkName(ctx, field, "get")}, - {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, - {"getter", - [&] { - if (ctx.is_upb()) { - ctx.Emit({}, R"rs( + ctx.Emit( + { + {"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, // Never r# prefixed + {"RsType", RsTypePath(ctx, field)}, + {"view_lifetime", ViewLifetime(accessor_case)}, + {"view_self", ViewReceiver(accessor_case)}, + {"getter_thunk", ThunkName(ctx, field, "get")}, + {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, + {"getter", + [&] { + if (ctx.is_upb()) { + ctx.Emit({}, R"rs( pub fn $field$($view_self$) -> $pb$::RepeatedView<$view_lifetime$, $RsType$> { unsafe { $getter_thunk$( @@ -47,8 +50,8 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, ) } )rs"); - } else { - ctx.Emit({}, R"rs( + } else { + ctx.Emit({}, R"rs( pub fn $field$($view_self$) -> $pb$::RepeatedView<$view_lifetime$, $RsType$> { unsafe { $pb$::RepeatedView::from_raw( @@ -58,16 +61,16 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, } } )rs"); - } - }}, - {"clearer_thunk", ThunkName(ctx, field, "clear")}, - {"getter_mut", - [&] { - if (accessor_case == AccessorCase::VIEW) { - return; - } - if (ctx.is_upb()) { - ctx.Emit({}, R"rs( + } + }}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"getter_mut", + [&] { + if (accessor_case == AccessorCase::VIEW) { + return; + } + if (ctx.is_upb()) { + ctx.Emit({}, R"rs( pub fn $field$_mut(&mut self) -> $pb$::RepeatedMut<'_, $RsType$> { unsafe { $pb$::RepeatedMut::from_inner( @@ -85,8 +88,8 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, } } )rs"); - } else { - ctx.Emit({}, R"rs( + } else { + ctx.Emit({}, R"rs( pub fn $field$_mut(&mut self) -> $pb$::RepeatedMut<'_, $RsType$> { unsafe { $pb$::RepeatedMut::from_inner( @@ -99,11 +102,25 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, } } )rs"); - } - }}}, - R"rs( + } + }}, + {"setter", + [&] { + if (accessor_case == AccessorCase::VIEW) { + return; + } + ctx.Emit({}, R"rs( + pub fn set_$raw_field_name$(&mut self, src: $pb$::RepeatedView<'_, $RsType$>) { + // TODO: Implement IntoProxied and avoid copying. + self.$field$_mut().copy_from(src); + } + )rs"); + }}, + }, + R"rs( $getter$ $getter_mut$ + $setter$ )rs"); } @@ -180,6 +197,8 @@ void RepeatedField::InThunkCc(Context& ctx, {"clearer_thunk", ThunkName(ctx, field, "clear")}, {"getter_thunk", ThunkName(ctx, field, "get")}, {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, + {"repeated_copy_from_thunk", + ThunkName(ctx, field, "repeated_copy_from")}, {"impls", [&] { ctx.Emit( From 959903d1991a3989b03bdca9502472af30afe905 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Tue, 30 Apr 2024 04:01:45 -0700 Subject: [PATCH 061/179] Remove the now unused vtable types. In particular, * SettableValue * ProxiedWithPresence * FieldEntry PiperOrigin-RevId: 629363417 --- rust/BUILD | 1 - rust/cpp.rs | 23 - rust/internal.rs | 4 - rust/map.rs | 13 +- rust/optional.rs | 677 +------------------ rust/proxied.rs | 152 ----- rust/repeated.rs | 14 +- rust/shared.rs | 8 +- rust/string.rs | 9 +- rust/test/shared/BUILD | 42 +- rust/test/shared/matchers.rs | 59 -- rust/test/shared/utf8/BUILD | 4 - rust/upb.rs | 25 - rust/vtable.rs | 257 ------- src/google/protobuf/compiler/rust/message.cc | 58 -- 15 files changed, 8 insertions(+), 1338 deletions(-) delete mode 100644 rust/test/shared/matchers.rs delete mode 100644 rust/vtable.rs diff --git a/rust/BUILD b/rust/BUILD index 21fe525d4b..aff4d2bec1 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -54,7 +54,6 @@ PROTOBUF_SHARED = [ "repeated.rs", "shared.rs", "string.rs", - "vtable.rs", "map.rs", "proto_macro.rs", ] diff --git a/rust/cpp.rs b/rust/cpp.rs index 06b45f6f92..2ad2a4ab30 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -233,31 +233,8 @@ pub fn debug_string(_private: Private, msg: RawMessage, f: &mut fmt::Formatter<' write!(f, "{dbg_str}") } -pub type MessagePresentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutatorData<'msg, T>; -pub type MessageAbsentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutatorData<'msg, T>; -pub type BytesPresentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; -pub type BytesAbsentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; -pub type InnerBytesMut<'msg> = crate::vtable::RawVTableMutator<'msg, [u8]>; pub type RawMapIter = UntypedMapIterator; -#[derive(Debug)] -pub struct MessageVTable { - pub getter: unsafe extern "C" fn(msg: RawMessage) -> RawMessage, - pub mut_getter: unsafe extern "C" fn(msg: RawMessage) -> RawMessage, - pub clearer: unsafe extern "C" fn(msg: RawMessage), -} - -impl MessageVTable { - pub const fn new( - _private: Private, - getter: unsafe extern "C" fn(msg: RawMessage) -> RawMessage, - mut_getter: unsafe extern "C" fn(msg: RawMessage) -> RawMessage, - clearer: unsafe extern "C" fn(msg: RawMessage), - ) -> Self { - MessageVTable { getter, mut_getter, clearer } - } -} - /// The raw contents of every generated message. #[derive(Debug)] pub struct MessageInner { diff --git a/rust/internal.rs b/rust/internal.rs index d9fa408ff1..f8118c88d6 100644 --- a/rust/internal.rs +++ b/rust/internal.rs @@ -13,10 +13,6 @@ pub use paste::paste; pub use crate::r#enum::Enum; -pub use crate::vtable::{ - new_vtable_field_entry, ProxiedWithRawOptionalVTable, ProxiedWithRawVTable, RawVTableMutator, - RawVTableOptionalMutatorData, -}; pub use crate::ProtoStr; // TODO: Temporarily re-export these symbols which are now under diff --git a/rust/map.rs b/rust/map.rs index 367793668f..f63aa3b1c5 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -6,7 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd use crate::{ - Mut, MutProxied, MutProxy, Proxied, SettableValue, View, ViewProxy, + Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, __internal::Private, __runtime::{InnerMap, InnerMapMut, RawMap, RawMapIter}, }; @@ -101,17 +101,6 @@ impl + ?Sized> MutProxied for Map = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> SettableValue> - for MapView<'msg, K, V> -{ - fn set_on<'b>(self, _private: Private, mut mutator: Mut<'b, Map>) - where - Map: 'b, - { - mutator.copy_from(self); - } -} - impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg> for MapView<'msg, K, V> { diff --git a/rust/optional.rs b/rust/optional.rs index 741f12d4f5..1f5cb9ad46 100644 --- a/rust/optional.rs +++ b/rust/optional.rs @@ -10,9 +10,7 @@ #![allow(unused)] use crate::__internal::Private; -use crate::{ - Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, ViewProxy, -}; +use crate::{Mut, MutProxied, MutProxy, Proxied, View, ViewProxy}; use std::convert::{AsMut, AsRef}; use std::fmt::{self, Debug}; use std::panic; @@ -23,9 +21,6 @@ use std::ptr; /// This can be pattern matched with `match` or `if let` to determine if the /// field is set and access the field data. /// -/// [`FieldEntry`], a specific type alias for `Optional`, provides much of the -/// functionality for this type. -/// /// Two `Optional`s are equal if they match both presence and the field values. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Optional { @@ -82,673 +77,3 @@ impl From> for Option { x.into_option() } } - -/// A mutable view into the value of an optional field, which may be set or -/// unset. -pub type FieldEntry<'msg, T> = Optional, AbsentField<'msg, T>>; - -/// Methods for `_mut()` accessors of optional types. -/// -/// The most common methods are [`set`] and [`or_default`]. -impl<'msg, T: ProxiedWithPresence + ?Sized + 'msg> FieldEntry<'msg, T> { - // is_set() is provided by `impl Optional` - - /// Gets a mutator for this field. Sets to the default value if not set. - pub fn or_default(self) -> Mut<'msg, T> { - match self { - Optional::Set(x) => x.into_mut(), - Optional::Unset(x) => x.set_default().into_mut(), - } - } - - /// Gets a mutator for this field. Sets to the given `val` if not set. - /// - /// If the field is already set, `val` is ignored. - pub fn or_set(self, val: impl SettableValue) -> Mut<'msg, T> { - self.or_set_with(move || val) - } - - /// Gets a mutator for this field. Sets using the given `val` function if - /// not set. - /// - /// If the field is already set, `val` is not invoked. - pub fn or_set_with(self, val: impl FnOnce() -> S) -> Mut<'msg, T> - where - S: SettableValue, - { - match self { - Optional::Set(x) => x.into_mut(), - Optional::Unset(x) => x.set(val()).into_mut(), - } - } - - /// Sets the value of this field to `val`. - /// - /// Equivalent to `self.or_default().set(val)`, but does not consume `self`. - /// - /// `set` has the same parameters as in [`MutProxy`], so making a field - /// `optional` will switch to using this method. This makes transitioning - /// from implicit to explicit presence easier. - pub fn set(&mut self, val: impl SettableValue) { - transform_mut(self, |mut self_| match self_ { - Optional::Set(ref mut present) => { - present.set(val); - self_ - } - Optional::Unset(absent) => Optional::Set(absent.set(val)), - }) - } - - /// Clears the field; `is_set()` will return `false`. - pub fn clear(&mut self) { - transform_mut(self, |self_| match self_ { - Optional::Set(present) => Optional::Unset(present.clear()), - absent => absent, - }) - } - - /// Gets an immutable view of this field, using its default value if not - /// set. This is shorthand for `as_view`. - /// - /// This provides a shorter lifetime than `into_view` but can also be called - /// multiple times - if the result of `get` is not living long enough - /// for your use, use that instead. - /// - /// `get` has the same parameters as in [`MutProxy`], so making a field - /// `optional` will switch to using this method. This makes transitioning - /// from implicit to explicit presence easier. - pub fn get(&self) -> View<'_, T> { - self.as_view() - } - - /// Converts to an immutable view of this optional field, preserving the - /// field's presence. - pub fn into_optional_view(self) -> Optional> { - let is_set = self.is_set(); - Optional::new(self.into_view(), is_set) - } - - /// Returns a field mutator if the field is set. - /// - /// Returns `None` if the field is not set. This does not affect `is_set()`. - /// - /// This returns `Option` and _not_ `Optional` since returning a defaulted - /// `Mut` would require mutating the presence of the field - for that - /// behavior, use `or_default()`. - pub fn try_into_mut(self) -> Option> { - match self { - Optional::Set(x) => Some(x.into_mut()), - Optional::Unset(_) => None, - } - } -} - -impl<'msg, T: ProxiedWithPresence + ?Sized + 'msg> ViewProxy<'msg> for FieldEntry<'msg, T> { - type Proxied = T; - - fn as_view(&self) -> View<'_, T> { - match self { - Optional::Set(x) => x.as_view(), - Optional::Unset(x) => x.as_view(), - } - } - - fn into_view<'shorter>(self) -> View<'shorter, T> - where - 'msg: 'shorter, - { - match self { - Optional::Set(x) => x.into_view(), - Optional::Unset(x) => x.into_view(), - } - } -} - -// `MutProxy` not implemented for `FieldEntry` since the field may not be set, -// and `as_mut`/`into_mut` should not insert. - -/// A field mutator capable of clearing that is statically known to point to a -/// set field. -pub struct PresentField<'msg, T> -where - T: ProxiedWithPresence + ?Sized + 'msg, -{ - pub(crate) inner: T::PresentMutData<'msg>, -} - -impl<'msg, T: ProxiedWithPresence + ?Sized + 'msg> Debug for PresentField<'msg, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.inner.fmt(f) - } -} - -impl<'msg, T: ProxiedWithPresence + ?Sized + 'msg> PresentField<'msg, T> { - #[doc(hidden)] - pub fn from_inner(_private: Private, inner: T::PresentMutData<'msg>) -> Self { - Self { inner } - } - - /// Gets an immutable view of this present field. This is shorthand for - /// `as_view`. - /// - /// This provides a shorter lifetime than `into_view` but can also be called - /// multiple times - if the result of `get` is not living long enough - /// for your use, use that instead. - pub fn get(&self) -> View<'_, T> { - self.as_view() - } - - pub fn set(&mut self, val: impl SettableValue) { - val.set_on(Private, self.as_mut()) - } - - /// See [`FieldEntry::clear`]. - pub fn clear(mut self) -> AbsentField<'msg, T> { - AbsentField { inner: T::clear_present_field(self.inner) } - } - - // This cannot provide `reborrow` - `clear` consumes after setting the field - // because it would violate a condition of `PresentField` - the field being set. -} - -impl<'msg, T> ViewProxy<'msg> for PresentField<'msg, T> -where - T: ProxiedWithPresence + ?Sized + 'msg, -{ - type Proxied = T; - - fn as_view(&self) -> View<'_, T> { - self.inner.as_view() - } - - fn into_view<'shorter>(self) -> View<'shorter, T> - where - 'msg: 'shorter, - { - self.inner.into_view() - } -} - -impl<'msg, T> MutProxy<'msg> for PresentField<'msg, T> -where - T: ProxiedWithPresence + ?Sized + 'msg, -{ - fn as_mut(&mut self) -> Mut<'_, T> { - self.inner.as_mut() - } - - fn into_mut<'shorter>(self) -> Mut<'shorter, T> - where - 'msg: 'shorter, - { - self.inner.into_mut() - } -} - -/// A field mutator capable of setting that is statically known to point to a -/// non-set field. -pub struct AbsentField<'msg, T> -where - T: ProxiedWithPresence + ?Sized + 'msg, -{ - pub(crate) inner: T::AbsentMutData<'msg>, -} - -impl<'msg, T: ProxiedWithPresence + ?Sized + 'msg> Debug for AbsentField<'msg, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.inner.fmt(f) - } -} - -impl<'msg, T: ProxiedWithPresence + ?Sized> AbsentField<'msg, T> { - #[doc(hidden)] - pub fn from_inner(_private: Private, inner: T::AbsentMutData<'msg>) -> Self { - Self { inner } - } - - /// Gets the default value for this unset field. - /// - /// This is the same value that the primitive accessor would provide, though - /// with the shorter lifetime of `as_view`. - pub fn default_value(&self) -> View<'_, T> { - self.as_view() - } - - /// See [`FieldEntry::set`]. Note that this consumes and returns a - /// `PresentField`. - pub fn set(self, val: impl SettableValue) -> PresentField<'msg, T> { - PresentField { inner: val.set_on_absent(Private, self.inner) } - } - - /// Sets this absent field to its default value. - pub fn set_default(self) -> PresentField<'msg, T> { - PresentField { inner: T::set_absent_to_default(self.inner) } - } - - // This cannot provide `reborrow` - `set` consumes after setting the field - // because it would violate a condition of `AbsentField` - the field being - // unset. -} - -impl<'msg, T> ViewProxy<'msg> for AbsentField<'msg, T> -where - T: ProxiedWithPresence + ?Sized + 'msg, -{ - type Proxied = T; - - fn as_view(&self) -> View<'_, T> { - self.inner.as_view() - } - - fn into_view<'shorter>(self) -> View<'shorter, T> - where - 'msg: 'shorter, - { - self.inner.into_view() - } -} - -/// Transforms a mutable reference in-place, treating it as if it were owned. -/// -/// The program will abort if `transform` panics. -/// -/// This is the same operation as provided by [`take_mut::take`]. -/// -/// [`take_mut::take`]: https://docs.rs/take_mut/latest/take_mut/fn.take.html -fn transform_mut(mut_ref: &mut T, transform: impl FnOnce(T) -> T) { - #[cold] - #[inline(never)] - fn panicked_in_transform_mut() -> ! { - use std::io::Write as _; - let backtrace = std::backtrace::Backtrace::force_capture(); - let stderr = std::io::stderr(); - let mut stderr = stderr.lock(); - let _ = write!(&mut stderr, "BUG: A protobuf mutator panicked! Backtrace:\n{backtrace}\n"); - let _ = stderr.flush(); - std::process::abort() - } - - // https://play.rust-lang.org/?edition=2021&gist=f3014e1f209013f0a38352e211f4a240 - // provides a sample test to confirm this operation is sound in Miri. - // SAFETY: - // - `old_t` is not dropped without also replacing `*mut_ref`, preventing a - // double-free. - // - If `transform` panics, the process aborts since `*mut_ref` has no possible - // valid value. - // - After `ptr::write`, a valid `T` is located at `*mut_ref` - unsafe { - let p: *mut T = mut_ref; - let old_t = p.read(); - let new_t = panic::catch_unwind(panic::AssertUnwindSafe(move || transform(old_t))) - .unwrap_or_else(|_| panicked_in_transform_mut()); - p.write(new_t); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::borrow::Cow; - - /// A sample message with custom presence bits, meant to mirror a C++ - /// message. - #[derive(Default, Debug)] - struct MyMessage { - /// has a default of `0` - a: i32, - - /// has a default of `5` - b: i32, - - /// Packed presence bitfield for `a` and `b` - presence: u8, - } - - impl MyMessage { - fn a(&self) -> View<'_, VtableProxied> { - VtableProxiedView { val: get_a(self) } - } - - fn a_opt(&self) -> Optional> { - Optional::new(self.a(), has_a(self)) - } - - fn a_mut(&mut self) -> FieldEntry<'_, VtableProxied> { - static A_VTABLE: ProxyVtable = - ProxyVtable { get: get_a, set: set_a, clear: clear_a, has: has_a }; - make_field_entry(self, &A_VTABLE) - } - - fn b(&self) -> View<'_, VtableProxied> { - VtableProxiedView { val: get_b(self) } - } - - fn b_opt(&self) -> Optional> { - Optional::new(self.b(), has_b(self)) - } - - fn b_mut(&mut self) -> FieldEntry<'_, VtableProxied> { - static B_VTABLE: ProxyVtable = - ProxyVtable { get: get_b, set: set_b, clear: clear_b, has: has_b }; - make_field_entry(self, &B_VTABLE) - } - } - - fn make_field_entry<'msg>( - msg: &'msg mut MyMessage, - vtable: &'msg ProxyVtable, - ) -> FieldEntry<'msg, VtableProxied> { - if (vtable.has)(&*msg) { - Optional::Set(PresentField::from_inner(Private, VtableProxiedMut { msg, vtable })) - } else { - Optional::Unset(AbsentField::from_inner(Private, VtableProxiedMut { msg, vtable })) - } - } - - // Thunks used for the vtable. For a C++ message these would be defined in C++ - // and exported via a C API - const A_BIT: u8 = 0; - const B_BIT: u8 = 1; - - fn get_a(msg: &MyMessage) -> i32 { - if has_a(msg) { msg.a } else { 0 } - } - - fn get_b(msg: &MyMessage) -> i32 { - if has_b(msg) { msg.b } else { 5 } - } - - fn set_a(msg: &mut MyMessage, val: i32) { - msg.presence |= (1 << A_BIT); - msg.a = val; - } - - fn set_b(msg: &mut MyMessage, val: i32) { - msg.presence |= (1 << B_BIT); - msg.b = val; - } - - fn clear_a(msg: &mut MyMessage) { - msg.presence &= !(1 << A_BIT); - } - - fn clear_b(msg: &mut MyMessage) { - msg.presence &= !(1 << B_BIT); - } - - fn has_a(msg: &MyMessage) -> bool { - msg.presence & (1 << A_BIT) != 0 - } - - fn has_b(msg: &MyMessage) -> bool { - msg.presence & (1 << B_BIT) != 0 - } - - #[derive(Debug)] - struct ProxyVtable { - get: fn(&MyMessage) -> i32, - set: fn(&mut MyMessage, val: i32), - clear: fn(&mut MyMessage), - has: fn(&MyMessage) -> bool, - } - - /// A proxy for a `i32` that is accessed through methods on a vtable. - struct VtableProxied; - - impl Proxied for VtableProxied { - type View<'msg> = VtableProxiedView; - } - - impl MutProxied for VtableProxied { - type Mut<'msg> = VtableProxiedMut<'msg>; - } - - impl ProxiedWithPresence for VtableProxied { - // In this case, the `PresentMutData` and `AbsentMutData` are identical to the - // `Mut` in layout. Other types/runtimes could require otherwise, e.g. `Mut` - // could be defined to only have get/set functions in its vtable, and not - // has/clear. - type PresentMutData<'msg> = VtableProxiedMut<'msg>; - type AbsentMutData<'msg> = VtableProxiedMut<'msg>; - - fn clear_present_field<'msg>( - present_mutator: Self::PresentMutData<'msg>, - ) -> Self::AbsentMutData<'msg> { - (present_mutator.vtable.clear)(&mut *present_mutator.msg); - present_mutator - } - - fn set_absent_to_default<'msg>( - absent_mutator: Self::AbsentMutData<'msg>, - ) -> Self::PresentMutData<'msg> { - SettableValue::::set_on_absent( - absent_mutator.as_view().val(), - Private, - absent_mutator, - ) - } - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - struct VtableProxiedView { - val: i32, - } - - impl VtableProxiedView { - fn val(&self) -> i32 { - self.val - } - - fn read(msg: &MyMessage, vtable: &ProxyVtable) -> Self { - VtableProxiedView { val: (vtable.get)(msg) } - } - } - - impl<'msg> ViewProxy<'msg> for VtableProxiedView { - type Proxied = VtableProxied; - - fn as_view(&self) -> View<'msg, VtableProxied> { - *self - } - - fn into_view<'shorter>(self) -> View<'shorter, VtableProxied> - where - 'msg: 'shorter, - { - self - } - } - - #[derive(Debug)] - struct VtableProxiedMut<'msg> { - msg: &'msg mut MyMessage, - vtable: &'msg ProxyVtable, - } - - impl<'msg> ViewProxy<'msg> for VtableProxiedMut<'msg> { - type Proxied = VtableProxied; - - fn as_view(&self) -> View<'_, VtableProxied> { - VtableProxiedView::read(self.msg, self.vtable) - } - - fn into_view<'shorter>(self) -> View<'shorter, VtableProxied> - where - 'msg: 'shorter, - { - VtableProxiedView::read(self.msg, self.vtable) - } - } - - impl<'msg> MutProxy<'msg> for VtableProxiedMut<'msg> { - fn as_mut(&mut self) -> Mut<'_, VtableProxied> { - VtableProxiedMut { msg: self.msg, vtable: self.vtable } - } - - fn into_mut<'shorter>(self) -> Mut<'shorter, VtableProxied> - where - 'msg: 'shorter, - { - self - } - } - - impl SettableValue for View<'_, VtableProxied> { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, VtableProxied>) - where - VtableProxied: 'msg, - { - SettableValue::::set_on(self.val(), Private, mutator) - } - - fn set_on_absent<'msg>( - self, - _private: Private, - absent_mutator: ::AbsentMutData<'msg>, - ) -> ::PresentMutData<'msg> { - SettableValue::::set_on_absent(self.val(), Private, absent_mutator) - } - } - - impl SettableValue for i32 { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, VtableProxied>) - where - VtableProxied: 'msg, - { - (mutator.vtable.set)(mutator.msg, self) - } - - fn set_on_absent<'msg>( - self, - _private: Private, - absent_mutator: ::AbsentMutData<'msg>, - ) -> ::PresentMutData<'msg> { - (absent_mutator.vtable.set)(absent_mutator.msg, self); - absent_mutator - } - } - - #[test] - fn test_field_entry() { - use googletest::prelude::*; - let mut m1 = MyMessage::default(); - let mut m2 = MyMessage::default(); - - let mut m1_a = m1.a_mut(); - assert_that!(m1_a, matches_pattern!(Optional::Unset(_))); - - assert_that!(m1_a.as_view().val(), eq(0)); - - assert_that!(m2.b().val(), eq(5)); - - let mut m2_b = m2.b_mut(); - assert_that!(m2_b.is_unset(), eq(true)); - assert_that!(m2_b.as_view().val(), eq(5)); - - m2_b.set(10); - assert_that!(m2_b.is_set(), eq(true)); - assert_that!(m2_b, matches_pattern!(Optional::Set(_))); - assert_that!(m2_b.as_view().val(), eq(10)); - - assert_that!(m1_a.or_default().as_view().val(), eq(0)); - assert_that!(m1.a_opt(), eq(Optional::Set(VtableProxiedView { val: 0 }))); - m1.a_mut().clear(); - - assert_that!(m1.a().val(), eq(0)); - assert_that!(m1.b().val(), eq(5)); - assert_that!(m2.a().val(), eq(0)); - assert_that!(m2.b().val(), eq(10)); - } - - #[test] - fn test_or_set() { - use googletest::prelude::*; - let mut m1 = MyMessage::default(); - let mut m2 = MyMessage::default(); - - assert_that!(m1.a_mut().or_set(10).get().val(), eq(10)); - assert_that!(m1.a_opt(), eq(Optional::Set(VtableProxiedView { val: 10 }))); - assert_that!(m1.a_mut().or_set(20).get().val(), eq(10)); - assert_that!(m1.a_opt(), eq(Optional::Set(VtableProxiedView { val: 10 }))); - - assert_that!(m2.a_mut().or_set_with(|| m1.a().val() + m1.b().val()).get().val(), eq(15)); - assert_that!(m2.a_opt(), eq(Optional::Set(VtableProxiedView { val: 15 }))); - assert_that!(m2.a_mut().or_set_with(|| None::.unwrap()).get().val(), eq(15)); - assert_that!(m2.a_opt(), eq(Optional::Set(VtableProxiedView { val: 15 }))); - } - - #[test] - fn test_into_optional_view() { - use googletest::prelude::*; - let mut m1 = MyMessage::default(); - assert_that!( - m1.a_mut().into_optional_view(), - eq(Optional::Unset(VtableProxiedView { val: 0 })) - ); - m1.a_mut().set(10); - assert_that!( - m1.a_mut().into_optional_view(), - eq(Optional::Set(VtableProxiedView { val: 10 })) - ); - assert_that!( - m1.b_mut().into_optional_view(), - eq(Optional::Unset(VtableProxiedView { val: 5 })) - ); - } - - #[test] - fn test_try_into_mut() { - use googletest::prelude::*; - let mut m1 = MyMessage::default(); - assert_that!(m1.a_mut().try_into_mut().is_none(), eq(true)); - m1.a_mut().set(10); - let mut a_mut = m1.a_mut().try_into_mut().expect("field to be set"); - a_mut.set(20); - assert_that!(m1.a().val(), eq(20)); - } - - #[test] - fn test_present_field() { - use googletest::prelude::*; - let mut m = MyMessage::default(); - m.a_mut().set(10); - match m.a_mut() { - Optional::Set(mut present) => { - assert_that!(present.as_view().val(), eq(10)); - present.set(20); - assert_that!(present.as_view().val(), eq(20)); - present.into_mut().set(30); - } - Optional::Unset(_) => unreachable!(), - } - assert_that!(m.a_opt(), eq(Optional::Set(VtableProxiedView { val: 30 }))); - m.b_mut().set(20); - match m.b_mut() { - Optional::Set(present) => present.clear(), - Optional::Unset(_) => unreachable!(), - }; - assert_that!(m.b_opt(), eq(Optional::Unset(VtableProxiedView { val: 5 }))); - } - - #[test] - fn test_absent_field() { - use googletest::prelude::*; - let mut m = MyMessage::default(); - match m.a_mut() { - Optional::Set(_) => unreachable!(), - Optional::Unset(absent) => { - assert_that!(absent.as_view().val(), eq(0)); - absent.set(20); - } - } - assert_that!(m.a_opt(), eq(Optional::Set(VtableProxiedView { val: 20 }))); - match m.b_mut() { - Optional::Set(_) => unreachable!(), - Optional::Unset(absent) => { - assert_that!(absent.as_view().val(), eq(5)); - absent.set_default(); - } - } - assert_that!(m.b_opt(), eq(Optional::Set(VtableProxiedView { val: 5 }))); - } -} diff --git a/rust/proxied.rs b/rust/proxied.rs index beef53c15d..3d7124813e 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -44,9 +44,7 @@ //! implemented the concept of "proxy" types. Proxy types are a reference-like //! indirection between the user and the internal memory representation. -use crate::RepeatedMut; use crate::__internal::Private; -use crate::repeated::ProxiedInRepeated; use std::fmt::Debug; /// A type that can be accessed through a reference-like proxy. @@ -167,13 +165,6 @@ where self.as_view() } - /// Sets this field to the given `val`. - /// - /// Any borrowed data from `val` will be cloned. - fn set(&mut self, val: impl SettableValue) { - val.set_on(Private, self.as_mut()) - } - /// Converts a borrow into a `Mut` with the lifetime of that borrow. /// /// This function enables calling multiple methods consuming `self`, for @@ -216,85 +207,6 @@ where 'msg: 'shorter; } -// TODO: move this to `optional.rs` as it's only used for optionals -/// `Proxied` types that can be optionally set or unset. -/// -/// All scalar and message types implement `ProxiedWithPresence`, while repeated -/// types don't. -pub trait ProxiedWithPresence: MutProxied { - /// The data necessary to store a present field mutator proxying `Self`. - /// This is the contents of `PresentField<'msg, Self>`. - type PresentMutData<'msg>: MutProxy<'msg, Proxied = Self>; - - /// The data necessary to store an absent field mutator proxying `Self`. - /// This is the contents of `AbsentField<'msg, Self>`. - type AbsentMutData<'msg>: ViewProxy<'msg, Proxied = Self>; - - /// Clears a present field. - fn clear_present_field(present_mutator: Self::PresentMutData<'_>) -> Self::AbsentMutData<'_>; - - /// Sets an absent field to its default value. - /// - /// This can be more efficient than setting with a default value, e.g. - /// a default submessage could share resources with the parent message. - fn set_absent_to_default(absent_mutator: Self::AbsentMutData<'_>) -> Self::PresentMutData<'_>; -} - -/// Values that can be used to set a field of `T`. -pub trait SettableValue: Sized -where - T: MutProxied + ?Sized, -{ - /// Consumes `self` to set the given mutator to the value of `self`. - #[doc(hidden)] - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, T>) - where - T: 'msg; - - /// Consumes `self` and `absent_mutator` to set the given empty field to - /// the value of `self`. - #[doc(hidden)] - fn set_on_absent( - self, - _private: Private, - absent_mutator: T::AbsentMutData<'_>, - ) -> T::PresentMutData<'_> - where - T: ProxiedWithPresence, - { - let mut present = T::set_absent_to_default(absent_mutator); - self.set_on(Private, present.as_mut()); - present - } - - /// Consumes `self` and `present_mutator` to set the given present field - /// to the value of `self`. - #[doc(hidden)] - fn set_on_present(self, _private: Private, mut present_mutator: T::PresentMutData<'_>) - where - T: ProxiedWithPresence, - { - self.set_on(Private, present_mutator.as_mut()) - } - - /// Consumes `self` and `repeated_mutator` to set the value at the - /// given index to the value of `self`. - /// - /// # Safety - /// `index` must be less than `repeated_mutator.len()` - #[doc(hidden)] - unsafe fn set_on_repeated_unchecked( - self, - _private: Private, - mut _repeated_mutator: RepeatedMut, - _index: usize, - ) where - T: ProxiedInRepeated, - { - unimplemented!() - } -} - /// A value to `Proxied`-value conversion that consumes the input value. /// /// All setter functions accept types that implement `IntoProxied`. The purpose @@ -395,45 +307,6 @@ mod tests { } } - impl SettableValue for MyProxiedView<'_> { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, MyProxied>) - where - MyProxied: 'msg, - { - mutator.my_proxied_ref.val = self.my_proxied_ref.val.clone(); - } - } - - impl SettableValue for String { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, MyProxied>) - where - MyProxied: 'msg, - { - mutator.my_proxied_ref.val = self; - } - } - - impl SettableValue for &'_ str { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, MyProxied>) - where - MyProxied: 'msg, - { - mutator.my_proxied_ref.val.replace_range(.., self); - } - } - - impl SettableValue for Cow<'_, str> { - fn set_on<'msg>(self, _private: Private, mutator: Mut<'msg, MyProxied>) - where - MyProxied: 'msg, - { - match self { - Cow::Owned(x) => >::set_on(x, Private, mutator), - Cow::Borrowed(x) => <&str as SettableValue>::set_on(x, Private, mutator), - } - } - } - #[test] fn test_as_view() { let my_proxied = MyProxied { val: "Hello World".to_string() }; @@ -443,18 +316,6 @@ mod tests { assert_that!(my_view.val(), eq(&my_proxied.val)); } - #[test] - fn test_as_mut() { - let mut my_proxied = MyProxied { val: "Hello World".to_string() }; - - let mut my_mut = my_proxied.as_mut(); - my_mut.set("Hello indeed".to_string()); - - let val_after_set = my_mut.as_view().val().to_string(); - assert_that!(my_proxied.val, eq(val_after_set)); - assert_that!(my_proxied.val, eq("Hello indeed")); - } - fn reborrow_mut_into_view<'msg>(x: Mut<'msg, MyProxied>) -> View<'msg, MyProxied> { // x.as_view() fails to compile with: // `ERROR: attempt to return function-local borrowed content` @@ -578,17 +439,4 @@ mod tests { reborrow_generic_mut_into_mut::(my_mut, other_mut); } } - - #[test] - fn test_set() { - let mut my_proxied = MyProxied::default(); - my_proxied.as_mut().set("hello"); - assert_that!(my_proxied.as_view().val(), eq("hello")); - - my_proxied.as_mut().set(String::from("hello2")); - assert_that!(my_proxied.as_view().val(), eq("hello2")); - - my_proxied.as_mut().set(Cow::Borrowed("hello3")); - assert_that!(my_proxied.as_view().val(), eq("hello3")); - } } diff --git a/rust/repeated.rs b/rust/repeated.rs index 7253307fcd..edc59472dc 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -15,7 +15,7 @@ use std::iter::FusedIterator; use std::marker::PhantomData; use crate::{ - Mut, MutProxied, MutProxy, Proxied, SettableValue, View, ViewProxy, + Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, __internal::Private, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -375,18 +375,6 @@ where } } -impl<'msg, T> SettableValue> for RepeatedView<'msg, T> -where - T: ProxiedInRepeated + ?Sized + 'msg, -{ - fn set_on<'b>(self, _private: Private, mutator: Mut<'b, Repeated>) - where - Repeated: 'b, - { - T::repeated_copy_from(self, mutator) - } -} - impl<'msg, T> iter::Iterator for RepeatedIter<'msg, T> where T: ProxiedInRepeated + ?Sized + 'msg, diff --git a/rust/shared.rs b/rust/shared.rs index c4aaef7906..1e0c645291 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -24,12 +24,9 @@ use std::fmt; pub mod __public { pub use crate::r#enum::UnknownEnumValue; pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; - pub use crate::optional::{AbsentField, FieldEntry, Optional, PresentField}; + pub use crate::optional::Optional; pub use crate::proto; - pub use crate::proxied::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, ProxiedWithPresence, SettableValue, View, - ViewProxy, - }; + pub use crate::proxied::{IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy}; pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, }; @@ -63,7 +60,6 @@ mod proto_macro; mod proxied; mod repeated; mod string; -mod vtable; /// An error that happened during deserialization. #[derive(Debug, Clone)] diff --git a/rust/string.rs b/rust/string.rs index ed1cfdc3ba..9272c8d0f0 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -10,13 +10,8 @@ #![allow(unused)] use crate::__internal::Private; -use crate::__runtime::{ - BytesAbsentMutData, BytesPresentMutData, InnerBytesMut, PtrAndLen, RawMessage, -}; -use crate::{ - AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, Proxied, - ProxiedWithPresence, SettableValue, View, ViewProxy, -}; +use crate::__runtime::{PtrAndLen, RawMessage}; +use crate::{Mut, MutProxied, MutProxy, Optional, Proxied, View, ViewProxy}; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use std::convert::{AsMut, AsRef}; diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index 460542bf46..fe7eae45dc 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -14,35 +14,7 @@ # Once we have a couple of these tests we will investigate ways to remove boilerplate (for example # by introducing a build macro that registers 2 rust_test targets). -load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") - -rust_library( - name = "matchers_upb", - testonly = True, - srcs = ["matchers.rs"], - aliases = { - "//rust:protobuf_upb_export": "protobuf", - }, - visibility = ["//rust/test/shared:__subpackages__"], - deps = [ - "//rust:protobuf_upb_export", - "@crate_index//:googletest", - ], -) - -rust_library( - name = "matchers_cpp", - testonly = True, - srcs = ["matchers.rs"], - aliases = { - "//rust:protobuf_cpp_export": "protobuf", - }, - visibility = ["//rust/test/shared:__subpackages__"], - deps = [ - "//rust:protobuf_cpp_export", - "@crate_index//:googletest", - ], -) +load("@rules_rust//rust:defs.bzl", "rust_test") rust_test( name = "child_parent_upb_test", @@ -197,7 +169,6 @@ rust_test( srcs = ["accessors_test.rs"], aliases = { "//rust:protobuf_cpp_export": "protobuf", - "//rust/test/shared:matchers_cpp": "matchers", }, proc_macro_deps = [ "@crate_index//:paste", @@ -205,7 +176,6 @@ rust_test( deps = [ "//rust:protobuf_cpp_export", "//rust/test:unittest_cc_rust_proto", - "//rust/test/shared:matchers_cpp", "@crate_index//:googletest", ], ) @@ -215,7 +185,6 @@ rust_test( srcs = ["accessors_test.rs"], aliases = { "//rust:protobuf_upb_export": "protobuf", - "//rust/test/shared:matchers_upb": "matchers", }, proc_macro_deps = [ "@crate_index//:paste", @@ -223,7 +192,6 @@ rust_test( deps = [ "//rust:protobuf_upb_export", "//rust/test:unittest_upb_rust_proto", - "//rust/test/shared:matchers_upb", "@crate_index//:googletest", ], ) @@ -233,13 +201,11 @@ rust_test( srcs = ["accessors_proto3_test.rs"], aliases = { "//rust:protobuf_cpp_export": "protobuf", - "//rust/test/shared:matchers_cpp": "matchers", }, deps = [ "//rust:protobuf_cpp_export", "//rust/test:unittest_proto3_cc_rust_proto", "//rust/test:unittest_proto3_optional_cc_rust_proto", - "//rust/test/shared:matchers_cpp", "@crate_index//:googletest", ], ) @@ -249,13 +215,11 @@ rust_test( srcs = ["accessors_proto3_test.rs"], aliases = { "//rust:protobuf_upb_export": "protobuf", - "//rust/test/shared:matchers_upb": "matchers", }, deps = [ "//rust:protobuf_upb_export", "//rust/test:unittest_proto3_optional_upb_rust_proto", "//rust/test:unittest_proto3_upb_rust_proto", - "//rust/test/shared:matchers_upb", "@crate_index//:googletest", ], ) @@ -407,12 +371,10 @@ rust_test( srcs = ["proto_macro_test.rs"], aliases = { "//rust:protobuf_cpp": "protobuf", - "//rust/test/shared:matchers_cpp": "matchers", }, deps = [ "//rust:protobuf_cpp", "//rust/test:unittest_cc_rust_proto", - "//rust/test/shared:matchers_cpp", "@crate_index//:googletest", ], ) @@ -422,12 +384,10 @@ rust_test( srcs = ["proto_macro_test.rs"], aliases = { "//rust:protobuf_upb": "protobuf", - "//rust/test/shared:matchers_upb": "matchers", }, deps = [ "//rust:protobuf_upb", "//rust/test:unittest_upb_rust_proto", - "//rust/test/shared:matchers_upb", "@crate_index//:googletest", ], ) diff --git a/rust/test/shared/matchers.rs b/rust/test/shared/matchers.rs deleted file mode 100644 index 70bea18464..0000000000 --- a/rust/test/shared/matchers.rs +++ /dev/null @@ -1,59 +0,0 @@ -use googletest::description::Description; -use googletest::matcher::MatcherResult; -use googletest::prelude::*; -use protobuf::{AbsentField, Optional, PresentField, ProxiedWithPresence}; -use std::marker::PhantomData; - -/// =============================== -/// IS_UNSET -/// =============================== -pub fn is_unset<'a, T: std::fmt::Debug + ProxiedWithPresence + ?Sized + 'a>() --> impl Matcher, AbsentField<'a, T>>> { - UnsetMatcher:: { _phantom: PhantomData } -} - -struct UnsetMatcher<'a, T: ProxiedWithPresence + ?Sized> { - _phantom: PhantomData>, -} - -impl<'a, T: std::fmt::Debug + ProxiedWithPresence + ?Sized> Matcher for UnsetMatcher<'a, T> { - type ActualT = Optional, AbsentField<'a, T>>; - - fn matches(&self, actual: &Self::ActualT) -> MatcherResult { - actual.is_unset().into() - } - - fn describe(&self, matcher_result: MatcherResult) -> Description { - match matcher_result { - MatcherResult::Match => "is not set".into(), - MatcherResult::NoMatch => "is set".into(), - } - } -} - -/// =============================== -/// IS_SET -/// =============================== -pub fn is_set<'a, T: std::fmt::Debug + ProxiedWithPresence + ?Sized + 'a>() --> impl Matcher, AbsentField<'a, T>>> { - SetMatcher:: { _phantom: PhantomData } -} - -struct SetMatcher<'a, T: ProxiedWithPresence + ?Sized> { - _phantom: PhantomData>, -} - -impl<'a, T: std::fmt::Debug + ProxiedWithPresence + ?Sized> Matcher for SetMatcher<'a, T> { - type ActualT = Optional, AbsentField<'a, T>>; - - fn matches(&self, actual: &Self::ActualT) -> MatcherResult { - actual.is_set().into() - } - - fn describe(&self, matcher_result: MatcherResult) -> Description { - match matcher_result { - MatcherResult::Match => "is set".into(), - MatcherResult::NoMatch => "is not set".into(), - } - } -} diff --git a/rust/test/shared/utf8/BUILD b/rust/test/shared/utf8/BUILD index a0ffd15f51..7ff155158e 100644 --- a/rust/test/shared/utf8/BUILD +++ b/rust/test/shared/utf8/BUILD @@ -9,14 +9,12 @@ rust_test( srcs = ["utf8_test.rs"], aliases = { "//rust:protobuf_cpp": "protobuf", - "//rust/test/shared:matchers_cpp": "matchers", }, deps = [ ":feature_verify_cc_rust_proto", ":no_features_proto2_cc_rust_proto", ":no_features_proto3_cc_rust_proto", "//rust:protobuf_cpp", - "//rust/test/shared:matchers_cpp", "@crate_index//:googletest", ], ) @@ -26,14 +24,12 @@ rust_test( srcs = ["utf8_test.rs"], aliases = { "//rust:protobuf_upb": "protobuf", - "//rust/test/shared:matchers_upb": "matchers", }, deps = [ ":feature_verify_upb_rust_proto", ":no_features_proto2_upb_rust_proto", ":no_features_proto3_upb_rust_proto", "//rust:protobuf_upb", - "//rust/test/shared:matchers_upb", "@crate_index//:googletest", ], ) diff --git a/rust/upb.rs b/rust/upb.rs index 98e1c849c1..6072349df1 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -60,31 +60,6 @@ impl ScratchSpace { pub type SerializedData = upb::OwnedArenaBox<[u8]>; -// TODO: Investigate replacing this with direct access to UPB bits. -pub type MessagePresentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutatorData<'msg, T>; -pub type MessageAbsentMutData<'msg, T> = crate::vtable::RawVTableOptionalMutatorData<'msg, T>; -pub type BytesPresentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; -pub type BytesAbsentMutData<'msg> = crate::vtable::RawVTableOptionalMutatorData<'msg, [u8]>; -pub type InnerBytesMut<'msg> = crate::vtable::RawVTableMutator<'msg, [u8]>; - -#[derive(Debug)] -pub struct MessageVTable { - pub getter: unsafe extern "C" fn(msg: RawMessage) -> Option, - pub mut_getter: unsafe extern "C" fn(msg: RawMessage, arena: RawArena) -> RawMessage, - pub clearer: unsafe extern "C" fn(msg: RawMessage), -} - -impl MessageVTable { - pub const fn new( - _private: Private, - getter: unsafe extern "C" fn(msg: RawMessage) -> Option, - mut_getter: unsafe extern "C" fn(msg: RawMessage, arena: RawArena) -> RawMessage, - clearer: unsafe extern "C" fn(msg: RawMessage), - ) -> Self { - MessageVTable { getter, mut_getter, clearer } - } -} - /// The raw contents of every generated message. #[derive(Debug)] pub struct MessageInner { diff --git a/rust/vtable.rs b/rust/vtable.rs deleted file mode 100644 index 83d6c0592c..0000000000 --- a/rust/vtable.rs +++ /dev/null @@ -1,257 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -use crate::__internal::Private; -use crate::__runtime::MutatorMessageRef; -use crate::{ - AbsentField, FieldEntry, Mut, MutProxied, MutProxy, Optional, PresentField, - ProxiedWithPresence, View, ViewProxy, -}; -use std::fmt::{self, Debug}; -use std::marker::PhantomData; -use std::ptr::NonNull; - -/// A proxied type that can use a vtable to provide get/set access for a -/// present field. -/// -/// This vtable should consist of `unsafe fn`s that call thunks that operate on -/// `RawMessage`. The structure of this vtable is different per proxied type. -pub trait ProxiedWithRawVTable: MutProxied { - /// The vtable for get/set access, stored in static memory. - type VTable: Debug + 'static; - - fn make_view(_private: Private, mut_inner: RawVTableMutator<'_, Self>) -> View<'_, Self>; - fn make_mut(_private: Private, inner: RawVTableMutator<'_, Self>) -> Mut<'_, Self>; -} - -/// A proxied type that can use a vtable to provide get/set/clear access for -/// an optional field. -/// -/// This vtable should consist of `unsafe fn`s that call thunks that operate on -/// `RawMessage`. The structure of this vtable is different per-proxied type. -pub trait ProxiedWithRawOptionalVTable: ProxiedWithRawVTable + ProxiedWithPresence { - /// The vtable for get/set/clear, must contain `Self::VTable`. - type OptionalVTable: Debug + 'static; - - /// Cast from a static reference of `OptionalVTable` to `VTable`. - /// This should mean `OptionalVTable` contains a `VTable`. - fn upcast_vtable( - _private: Private, - optional_vtable: &'static Self::OptionalVTable, - ) -> &'static Self::VTable; -} - -/// Constructs a new field entry from a raw message, a vtable for manipulation, -/// and an eager check for whether the value is present or not. -/// -/// # Safety -/// - `msg_ref` must be valid to provide as an argument for `vtable`'s methods -/// for `'msg`. -/// - If given `msg_ref` as an argument, any values returned by `vtable` methods -/// must be valid for `'msg`. -/// - Operations on the vtable must be thread-compatible. -#[doc(hidden)] -pub unsafe fn new_vtable_field_entry<'msg, T: ProxiedWithRawOptionalVTable + ?Sized>( - _private: Private, - msg_ref: MutatorMessageRef<'msg>, - optional_vtable: &'static T::OptionalVTable, - is_set: bool, -) -> FieldEntry<'msg, T> -where - T: ProxiedWithPresence< - PresentMutData<'msg> = RawVTableOptionalMutatorData<'msg, T>, - AbsentMutData<'msg> = RawVTableOptionalMutatorData<'msg, T>, - >, -{ - // SAFETY: safe as promised by the caller of the function - let data = unsafe { RawVTableOptionalMutatorData::new(Private, msg_ref, optional_vtable) }; - if is_set { - Optional::Set(PresentField::from_inner(Private, data)) - } else { - Optional::Unset(AbsentField::from_inner(Private, data)) - } -} - -/// The internal implementation type for a vtable-based `protobuf::Mut`. -/// -/// This stores the two components necessary to mutate the field: -/// borrowed message data and a vtable reference. -/// -/// The borrowed message data varies per runtime: C++ needs a message pointer, -/// while UPB needs a message pointer and an `&Arena`. -/// -/// Implementations of `ProxiedWithRawVTable` implement get/set -/// on top of `RawVTableMutator`, and the top-level mutator (e.g. -/// `BytesMut`) calls these methods. -/// -/// [`RawVTableOptionalMutatorData`] is similar, but also includes the -/// capability to has/clear. -pub struct RawVTableMutator<'msg, T: ?Sized> { - msg_ref: MutatorMessageRef<'msg>, - /// Stores `&'static ::Vtable` - /// as a type-erased pointer to avoid a bound on the struct. - vtable: NonNull<()>, - _phantom: PhantomData<&'msg T>, -} - -// These use manual impls instead of derives to avoid unnecessary bounds on `T`. -// This problem is referred to as "perfect derive". -// https://smallcultfollowing.com/babysteps/blog/2022/04/12/implied-bounds-and-perfect-derive/ -impl<'msg, T: ?Sized> Clone for RawVTableMutator<'msg, T> { - fn clone(&self) -> Self { - *self - } -} -impl<'msg, T: ?Sized> Copy for RawVTableMutator<'msg, T> {} - -impl<'msg, T: ProxiedWithRawVTable + ?Sized> Debug for RawVTableMutator<'msg, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawVTableMutator") - .field("msg_ref", &self.msg_ref) - .field("vtable", self.vtable()) - .finish() - } -} - -impl<'msg, T: ProxiedWithRawVTable + ?Sized> RawVTableMutator<'msg, T> { - /// # Safety - /// - `msg_ref` must be valid to provide as an argument for `vtable`'s - /// methods for `'msg`. - /// - If given `msg_ref` as an argument, any values returned by `vtable` - /// methods must be valid for `'msg`. - #[doc(hidden)] - pub unsafe fn new( - _private: Private, - msg_ref: MutatorMessageRef<'msg>, - vtable: &'static T::VTable, - ) -> Self { - RawVTableMutator { msg_ref, vtable: NonNull::from(vtable).cast(), _phantom: PhantomData } - } - - pub fn vtable(self) -> &'static T::VTable { - // SAFETY: This was cast from `&'static T::VTable`. - unsafe { self.vtable.cast().as_ref() } - } - - pub fn msg_ref(self) -> MutatorMessageRef<'msg> { - self.msg_ref - } -} - -/// [`RawVTableMutator`], but also includes has/clear. -/// -/// This is used as the `PresentData` and `AbsentData` for `impl -/// ProxiedWithPresence for T`. In that implementation, `clear_present_field` -/// and `set_absent_to_default` will use methods implemented on -/// `RawVTableOptionalMutatorData` to do the setting and clearing. -/// -/// This has the same representation for "present" and "absent" data; -/// differences like default values are obviated by the vtable. -pub struct RawVTableOptionalMutatorData<'msg, T: ?Sized> { - msg_ref: MutatorMessageRef<'msg>, - /// Stores `&'static ::Vtable` - /// as a type-erased pointer to avoid a bound on the struct. - optional_vtable: NonNull<()>, - _phantom: PhantomData<&'msg T>, -} - -// SAFETY: all `T` that can perform mutations don't mutate through a shared -// reference. -unsafe impl<'msg, T: ?Sized> Sync for RawVTableOptionalMutatorData<'msg, T> {} - -// These use manual impls instead of derives to avoid unnecessary bounds on `T`. -// This problem is referred to as "perfect derive". -// https://smallcultfollowing.com/babysteps/blog/2022/04/12/implied-bounds-and-perfect-derive/ -impl<'msg, T: ?Sized> Clone for RawVTableOptionalMutatorData<'msg, T> { - fn clone(&self) -> Self { - *self - } -} -impl<'msg, T: ?Sized> Copy for RawVTableOptionalMutatorData<'msg, T> {} - -impl<'msg, T: ProxiedWithRawOptionalVTable + ?Sized> Debug - for RawVTableOptionalMutatorData<'msg, T> -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RawVTableOptionalMutatorData") - .field("msg_ref", &self.msg_ref) - .field("vtable", self.optional_vtable()) - .finish() - } -} - -impl<'msg, T: ProxiedWithRawOptionalVTable + ?Sized> RawVTableOptionalMutatorData<'msg, T> { - /// # Safety - /// - `msg_ref` must be valid to provide as an argument for `vtable`'s - /// methods for `'msg`. - /// - If given `msg_ref` as an argument, any values returned by `vtable` - /// methods must be valid for `'msg`. - #[doc(hidden)] - pub unsafe fn new( - _private: Private, - msg_ref: MutatorMessageRef<'msg>, - vtable: &'static T::OptionalVTable, - ) -> Self { - Self { msg_ref, optional_vtable: NonNull::from(vtable).cast(), _phantom: PhantomData } - } - - pub fn msg_ref(self) -> MutatorMessageRef<'msg> { - self.msg_ref - } - - pub fn optional_vtable(self) -> &'static T::OptionalVTable { - // SAFETY: This was cast from `&'static T::OptionalVTable` in `new`. - unsafe { self.optional_vtable.cast().as_ref() } - } - - fn into_raw_mut(self) -> RawVTableMutator<'msg, T> { - // SAFETY: the safety requirements have been met by the caller of `new`. - unsafe { - RawVTableMutator::new( - Private, - self.msg_ref, - T::upcast_vtable(Private, self.optional_vtable()), - ) - } - } -} - -impl<'msg, T: ProxiedWithRawOptionalVTable + ?Sized + 'msg> ViewProxy<'msg> - for RawVTableOptionalMutatorData<'msg, T> -{ - type Proxied = T; - - fn as_view(&self) -> View<'_, T> { - T::make_view(Private, self.into_raw_mut()) - } - - fn into_view<'shorter>(self) -> View<'shorter, T> - where - 'msg: 'shorter, - { - T::make_view(Private, self.into_raw_mut()) - } -} - -// Note: though this raw value implements `MutProxy`, the `as_mut` is only valid -// when the field is known to be present. `FieldEntry` enforces this in its -// design: `AbsentField { inner: RawVTableOptionalMutatorData }` does not -// implement `MutProxy`. -impl<'msg, T: ProxiedWithRawOptionalVTable + ?Sized + 'msg> MutProxy<'msg> - for RawVTableOptionalMutatorData<'msg, T> -{ - fn as_mut(&mut self) -> Mut<'_, T> { - T::make_mut(Private, self.into_raw_mut()) - } - - fn into_mut<'shorter>(self) -> Mut<'shorter, T> - where - 'msg: 'shorter, - { - T::make_mut(Private, self.into_raw_mut()) - } -} diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 67c569f4f5..ec0ab962f1 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -924,64 +924,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } - impl $pbi$::ProxiedWithRawVTable for $Msg$ { - type VTable = $pbr$::MessageVTable; - - fn make_view(_private: $pbi$::Private, - mut_inner: $pbi$::RawVTableMutator<'_, Self>) - -> $pb$::View<'_, Self> { - let msg = unsafe { - (mut_inner.vtable().getter)(mut_inner.msg_ref().msg()) - }; - $Msg$View::new($pbi$::Private, msg$unwrap_upb$) - } - - fn make_mut(_private: $pbi$::Private, - inner: $pbi$::RawVTableMutator<'_, Self>) - -> $pb$::Mut<'_, Self> { - let raw_submsg = unsafe { - (inner.vtable().mut_getter)(inner.msg_ref().msg()$upb_arena$) - }; - $Msg$Mut::from_parent($pbi$::Private, inner.msg_ref(), raw_submsg) - } - } - - impl $pbi$::ProxiedWithRawOptionalVTable for $Msg$ { - type OptionalVTable = $pbr$::MessageVTable; - - fn upcast_vtable(_private: $pbi$::Private, - optional_vtable: &'static Self::OptionalVTable) - -> &'static Self::VTable { - &optional_vtable - } - } - - impl $pb$::ProxiedWithPresence for $Msg$ { - type PresentMutData<'a> = $pbr$::MessagePresentMutData<'a, $Msg$>; - type AbsentMutData<'a> = $pbr$::MessageAbsentMutData<'a, $Msg$>; - - fn clear_present_field(present_mutator: Self::PresentMutData<'_>) - -> Self::AbsentMutData<'_> { - // SAFETY: The raw ptr msg_ref is valid - unsafe { - (present_mutator.optional_vtable().clearer)(present_mutator.msg_ref().msg()); - - $pbi$::RawVTableOptionalMutatorData::new($pbi$::Private, - present_mutator.msg_ref(), - present_mutator.optional_vtable()) - } - } - - fn set_absent_to_default(absent_mutator: Self::AbsentMutData<'_>) - -> Self::PresentMutData<'_> { - unsafe { - $pbi$::RawVTableOptionalMutatorData::new($pbi$::Private, - absent_mutator.msg_ref(), - absent_mutator.optional_vtable()) - } - } - } - $into_proxied_impl$ $repeated_impl$ From c24e7529eb7a709285a5e61d8d05a347576de7c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Tue, 30 Apr 2024 07:24:51 -0700 Subject: [PATCH 062/179] Automated rollback of commit e9c9970eb4ac488d9fb94e8dea6945d3881c405d. PiperOrigin-RevId: 629405086 --- .../kotlin/com/google/protobuf/Proto2Test.kt | 28 +++------- .../kotlin/com/google/protobuf/Proto3Test.kt | 14 +---- .../google/protobuf/evil_names_proto2.proto | 9 --- .../google/protobuf/evil_names_proto3.proto | 6 -- src/google/protobuf/compiler/java/BUILD.bazel | 1 - .../protobuf/compiler/java/field_common.cc | 56 ------------------- .../compiler/java/immutable/enum_field.cc | 10 ++-- .../compiler/java/immutable/map_field.cc | 2 +- .../compiler/java/immutable/message_field.cc | 6 +- .../java/immutable/primitive_field.cc | 31 +++------- .../compiler/java/immutable/string_field.cc | 6 +- .../protobuf/compiler/java/lite/enum_field.cc | 10 ++-- .../protobuf/compiler/java/lite/map_field.cc | 2 +- .../compiler/java/lite/message_field.cc | 6 +- .../compiler/java/lite/primitive_field.cc | 38 +++++-------- .../compiler/java/lite/string_field.cc | 6 +- 16 files changed, 57 insertions(+), 174 deletions(-) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt index 7c319bfaac..c9fee8a78f 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt @@ -225,7 +225,7 @@ class Proto2Test { TestAllTypesKt.repeatedGroup { a = 1 }, TestAllTypesKt.repeatedGroup { a = 2 }, TestAllTypesKt.repeatedGroup { a = 3 }, - TestAllTypesKt.repeatedGroup { a = 4 }, + TestAllTypesKt.repeatedGroup { a = 4 } ) ) repeatedGroup[0] = TestAllTypesKt.repeatedGroup { a = 5 } @@ -235,7 +235,7 @@ class Proto2Test { TestAllTypesKt.repeatedGroup { a = 5 }, TestAllTypesKt.repeatedGroup { a = 2 }, TestAllTypesKt.repeatedGroup { a = 3 }, - TestAllTypesKt.repeatedGroup { a = 4 }, + TestAllTypesKt.repeatedGroup { a = 4 } ) ) @@ -249,7 +249,7 @@ class Proto2Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 }, + nestedMessage { bb = 4 } ) ) repeatedNestedMessage[0] = nestedMessage { bb = 5 } @@ -259,7 +259,7 @@ class Proto2Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 }, + nestedMessage { bb = 4 } ) ) @@ -548,7 +548,7 @@ class Proto2Test { repeatedGroupExtension { a = 1 }, repeatedGroupExtension { a = 2 }, repeatedGroupExtension { a = 3 }, - repeatedGroupExtension { a = 4 }, + repeatedGroupExtension { a = 4 } ) ) this[UnittestProto.repeatedGroupExtension][0] = repeatedGroupExtension { a = 5 } @@ -558,7 +558,7 @@ class Proto2Test { repeatedGroupExtension { a = 5 }, repeatedGroupExtension { a = 2 }, repeatedGroupExtension { a = 3 }, - repeatedGroupExtension { a = 4 }, + repeatedGroupExtension { a = 4 } ) ) @@ -575,7 +575,7 @@ class Proto2Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 }, + nestedMessage { bb = 4 } ) ) this[UnittestProto.repeatedNestedMessageExtension][0] = nestedMessage { bb = 5 } @@ -585,7 +585,7 @@ class Proto2Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 }, + nestedMessage { bb = 4 } ) ) @@ -757,7 +757,7 @@ class Proto2Test { 1 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO, 2 to Proto2MapEnum.PROTO2_MAP_ENUM_BAR, 3 to Proto2MapEnum.PROTO2_MAP_ENUM_BAZ, - 4 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO, + 4 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO ) ) } @@ -844,11 +844,6 @@ class Proto2Test { cachedSize_ = "foo" serializedSize_ = true by = "foo" - dEPRECATEDFoo = "foo" - DEPRECATEDBar = "foo" - iD = "foo" - aBNotification = "foo" - notDEPRECATEDFoo = "foo" } ) .isEqualTo( @@ -874,11 +869,6 @@ class Proto2Test { .setCachedSize_("foo") .setSerializedSize_(true) .setBy("foo") - .setDEPRECATEDFoo("foo") - .setDEPRECATEDBar("foo") - .setID("foo") - .setABNotification("foo") - .setNotDEPRECATEDFoo("foo") .build() ) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt index 72ab8e1f80..0b270e6e75 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt @@ -81,7 +81,7 @@ class Proto3Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 }, + nestedMessage { bb = 4 } ) ) repeatedNestedMessage[0] = nestedMessage { bb = 5 } @@ -91,7 +91,7 @@ class Proto3Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 }, + nestedMessage { bb = 4 } ) ) @@ -200,11 +200,6 @@ class Proto3Test { pairs["foo"] = 1 LeadingUnderscore = "foo" option = 1 - dEPRECATEDFoo = "foo" - iD = "foo" - aBNotification = "foo" - DEPRECATEDBar = "foo" - notDEPRECATEDFoo = "foo" } ) .isEqualTo( @@ -242,11 +237,6 @@ class Proto3Test { .putPairs("foo", 1) .setLeadingUnderscore("foo") .setOption(1) - .setDEPRECATEDFoo("foo") - .setID("foo") - .setABNotification("foo") - .setDEPRECATEDBar("foo") - .setNotDEPRECATEDFoo("foo") .build() ) diff --git a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto index f771bf97bf..ba0827422b 100644 --- a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto +++ b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto @@ -43,15 +43,6 @@ message EvilNamesProto2 { optional string cached_size = 23; optional bool serialized_size = 24; optional string by = 25; - - optional string DEPRECATED_foo = 26; - optional group DEPRECATED_NavigationImageRequested = 27 { - optional int32 DEPRECATED_FooBar = 28; - } - optional string __DEPRECATED_Bar = 29; - optional string ID = 30; - optional string a_b_notification = 31; - optional string not_DEPRECATED_foo = 32; } message List {} diff --git a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto index 238bebc06b..2c9b5a2d68 100644 --- a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto +++ b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto @@ -56,12 +56,6 @@ message EvilNamesProto3 { oneof _leading_underscore_oneof { int32 option = 34; } - - optional string DEPRECATED_foo = 35; - optional string ID = 36; - optional string a_b_notification = 37; - optional string __DEPRECATED_Bar = 38; - optional string not_DEPRECATED_foo = 39; } message HardKeywordsAllTypesProto3 { diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 915b370309..39f862f6cc 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -144,7 +144,6 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/src/google/protobuf/compiler/java/field_common.cc b/src/google/protobuf/compiler/java/field_common.cc index bb3cd6a748..4acafabe36 100644 --- a/src/google/protobuf/compiler/java/field_common.cc +++ b/src/google/protobuf/compiler/java/field_common.cc @@ -1,11 +1,8 @@ #include "google/protobuf/compiler/java/field_common.h" -#include #include -#include "absl/strings/str_cat.h" #include "google/protobuf/compiler/java/helpers.h" -#include "google/protobuf/compiler/java/names.h" #include "google/protobuf/descriptor.h" namespace google { @@ -13,8 +10,6 @@ namespace protobuf { namespace compiler { namespace java { -std::string GetKotlinPropertyName(const FieldDescriptor* descriptor); - void SetCommonFieldVariables( const FieldDescriptor* descriptor, const FieldGeneratorInfo* info, absl::flat_hash_map* variables) { @@ -35,11 +30,6 @@ void SetCommonFieldVariables( (*variables)["kt_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->name, "_") : info->name; - auto kt_property_name = GetKotlinPropertyName(descriptor); - (*variables)["kt_property_name"] = kt_property_name; - (*variables)["kt_safe_name"] = IsForbiddenKotlin(kt_property_name) - ? absl::StrCat("`", kt_property_name, "`") - : kt_property_name; (*variables)["kt_capitalized_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->capitalized_name, "_") : info->capitalized_name; @@ -61,52 +51,6 @@ void SetCommonFieldVariables( } } -// Locale-independent ASCII upper and lower case munging. -static bool IsUpper(char c) { - return static_cast(c - 'A') <= 'Z' - 'A'; -} - -static char ToLower(char c) { return IsUpper(c) ? c - 'A' + 'a' : c; } - -// Returns the name by which the generated Java getters and setters should be -// referenced from Kotlin as properties. In the simplest case, the original name -// is something like `foo_bar`, which gets translated into `getFooBar()` etc, -// and that in turn can be referenced from Kotlin as `fooBar`. -// -// The algorithm for translating proto names into Java getters and setters is -// straightforward. The first letter of each underscore-separated word gets -// uppercased and the underscores are deleted. There are no other changes, so in -// particular if the proto name has a string of capitals then those remain -// as-is. -// -// The algorithm that the Kotlin compiler uses to derive the property name is -// slightly more complicated. If the first character after `get` (etc) is a -// capital and the second isn't, then the property name is just that string with -// its first letter lowercased. So `getFoo` becomes `foo` and `getX` becomes -// `x`. But if there is more than one capital, then all but the last get -// lowercased. So `getHTMLPage` becomes `htmlPage`. If there are only capitals -// then they all get lowercased, so `getID` becomes `id`. -// TODO: move this to a Kotlin-specific location -std::string GetKotlinPropertyName(const FieldDescriptor* descriptor) { - // Find the first non-capital. If it is the second character, then we just - // need to lowercase the first one. Otherwise we need to lowercase everything - // up to but not including the last capital, except that if everything is - // capitals then everything must be lowercased. - std::string capitalized_name = CapitalizedFieldName(descriptor); - std::string kt_property_name = capitalized_name; - size_t first_non_capital; - for (first_non_capital = 0; first_non_capital < capitalized_name.length() && - IsUpper(capitalized_name[first_non_capital]); - first_non_capital++) { - } - size_t stop = first_non_capital; - if (stop > 1 && stop < capitalized_name.length()) stop--; - for (size_t i = 0; i < stop; i++) { - kt_property_name[i] = ToLower(kt_property_name[i]); - } - return kt_property_name; -} - void SetCommonOneofVariables( const FieldDescriptor* descriptor, const OneofGeneratorInfo* info, absl::flat_hash_map* variables) { diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/immutable/enum_field.cc index 4a64460fb5..2cb734f498 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/immutable/enum_field.cc @@ -273,10 +273,10 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); if (SupportUnknownEnumValue(descriptor_)) { @@ -284,10 +284,10 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" " }\n"); } @@ -1100,7 +1100,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/map_field.cc b/src/google/protobuf/compiler/java/immutable/map_field.cc index 13e6319b79..02e85874c0 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.cc +++ b/src/google/protobuf/compiler/java/immutable/map_field.cc @@ -981,7 +981,7 @@ void ImmutableMapFieldGenerator::GenerateKotlinDslMembers( " @kotlin.jvm.JvmSynthetic\n" " @JvmName(\"get$kt_capitalized_name$Map\")\n" " get() = com.google.protobuf.kotlin.DslMap(\n" - " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$Map$}$()\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), diff --git a/src/google/protobuf/compiler/java/immutable/message_field.cc b/src/google/protobuf/compiler/java/immutable/message_field.cc index 31eb0ea858..1b76fc2f63 100644 --- a/src/google/protobuf/compiler/java/immutable/message_field.cc +++ b/src/google/protobuf/compiler/java/immutable/message_field.cc @@ -378,10 +378,10 @@ void ImmutableMessageFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -1376,7 +1376,7 @@ void RepeatedImmutableMessageFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/primitive_field.cc b/src/google/protobuf/compiler/java/immutable/primitive_field.cc index 3dc402e5c2..eb31a7b756 100644 --- a/src/google/protobuf/compiler/java/immutable/primitive_field.cc +++ b/src/google/protobuf/compiler/java/immutable/primitive_field.cc @@ -291,27 +291,14 @@ void ImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - if (descriptor_->name() == "is_initialized") { - printer->Print( - variables_, - "// TODO: remove this hack; we should access properties\n" - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$kt_capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" - " }\n"); - } else { - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); - } + printer->Print(variables_, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), @@ -861,7 +848,7 @@ void RepeatedImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/string_field.cc b/src/google/protobuf/compiler/java/immutable/string_field.cc index df12b05f07..c9ac5f4e7c 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.cc +++ b/src/google/protobuf/compiler/java/immutable/string_field.cc @@ -372,10 +372,10 @@ void ImmutableStringFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: kotlin.String\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -992,7 +992,7 @@ void RepeatedImmutableStringFieldGenerator::GenerateKotlinDslMembers( "\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); // List.add(String) diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index b9cde7a483..f2fb340b1b 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -308,10 +308,10 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); if (SupportUnknownEnumValue(descriptor_)) { @@ -319,10 +319,10 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" " }\n"); } @@ -914,7 +914,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 491523df91..74fc93f347 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -857,7 +857,7 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( " @kotlin.jvm.JvmSynthetic\n" " @JvmName(\"get$kt_capitalized_name$Map\")\n" " get() = com.google.protobuf.kotlin.DslMap(\n" - " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$Map$}$()\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index c46407bf51..7c5cc3cf77 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -283,10 +283,10 @@ void ImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -818,7 +818,7 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index c9b3a2947a..69924c9c83 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -217,9 +217,10 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateMembers( "private static final $field_type$ $bytes_default$ = $default$;\n"); } if (!context_->options().opensource_runtime) { - printer->Print(variables_, - "@com.google.protobuf.ProtoField(\n" - " isRequired=$required$)\n"); + printer->Print( + variables_, + "@com.google.protobuf.ProtoField(\n" + " isRequired=$required$)\n"); if (HasHasbit(descriptor_)) { printer->Print(variables_, "@com.google.protobuf.ProtoPresenceCheckedField(\n" @@ -328,27 +329,14 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void ImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_, context_->options()); - if (descriptor_->name() == "is_initialized") { - printer->Print( - variables_, - "// TODO: b/336400327 - remove this hack; we should access properties\n" - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.get${$$kt_capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" - " }\n"); - } else { - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); - } + printer->Print(variables_, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), @@ -730,7 +718,7 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index a08bae50b6..833f415585 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -336,10 +336,10 @@ void ImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: kotlin.String\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -825,7 +825,7 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( "@kotlin.OptIn" "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" " )\n"); // List.add(String) From 2d56d69a40a4a13e1132be0ced546d0eed6ad2ad Mon Sep 17 00:00:00 2001 From: Ashley <73482956+ascopes@users.noreply.github.com> Date: Tue, 30 Apr 2024 07:52:14 -0700 Subject: [PATCH 063/179] Add ascopes/protobuf-maven-plugin to third_party.md (#16656) This PR adds my project at https://github.com/ascopes/protobuf-maven-plugin to the thirdparty README. CLA signed in previous contributions to other Google projects on GitHub. Closes #16656 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/16656 from ascopes:patch-1 6f8b43aa0dd4da5a7bbf93460acd4094499aed2d PiperOrigin-RevId: 629411565 --- docs/third_party.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/third_party.md b/docs/third_party.md index fce383aa48..6076f9aa64 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -187,6 +187,7 @@ Inactive: [![Maven Central](https://img.shields.io/maven-central/v/org.xolstice.maven.plugins/protobuf-maven-plugin.svg)](https://repo1.maven.org/maven2/org/xolstice/maven/plugins/protobuf-maven-plugin/) * https://code.google.com/p/maven-protoc-plugin/ * https://github.com/os72/protoc-jar-maven-plugin + * https://ascopes.github.io/protobuf-maven-plugin * [Protobuf Plugin for Gradle](https://github.com/google/protobuf-gradle-plugin) * [Sbt plugin for Protocol Buffers](https://github.com/Atry/sbt-cppp) From 52c12a6f92d743fcf0b212e84dc60dff91152c2d Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 30 Apr 2024 08:39:39 -0700 Subject: [PATCH 064/179] Automated rollback of commit 208cae7fd7fdfc7fef6eddded07c2c7431bf0d8f. PiperOrigin-RevId: 629424223 --- upb/reflection/field_def.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/upb/reflection/field_def.c b/upb/reflection/field_def.c index 6a70361b6b..d5597f4b2e 100644 --- a/upb/reflection/field_def.c +++ b/upb/reflection/field_def.c @@ -714,11 +714,6 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } - } else { - if (syntax == kUpb_Syntax_Editions) { - _upb_DefBuilder_Errf(ctx, "Editions proto cannot lack type for field %s", - f->full_name); - } } if (!has_type && has_type_name) { From 54e2553abe6cefae9158f7e3f590b60a858bfd71 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 30 Apr 2024 15:55:56 +0000 Subject: [PATCH 065/179] Auto-generate files after cl/629424223 --- php/ext/google/protobuf/php-upb.c | 5 ----- ruby/ext/google/protobuf_c/ruby-upb.c | 5 ----- 2 files changed, 10 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index bff21695b8..508b7f46dc 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -14203,11 +14203,6 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } - } else { - if (syntax == kUpb_Syntax_Editions) { - _upb_DefBuilder_Errf(ctx, "Editions proto cannot lack type for field %s", - f->full_name); - } } if (!has_type && has_type_name) { diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index caa348f73f..9578e8e976 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -13696,11 +13696,6 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } - } else { - if (syntax == kUpb_Syntax_Editions) { - _upb_DefBuilder_Errf(ctx, "Editions proto cannot lack type for field %s", - f->full_name); - } } if (!has_type && has_type_name) { From 2c246484d3809c5aa0c637992ea710b7071d98b4 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 30 Apr 2024 09:14:59 -0700 Subject: [PATCH 066/179] Add editions-specific required fields to the required fields allow-list. PiperOrigin-RevId: 629433997 --- editions/BUILD | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/editions/BUILD b/editions/BUILD index 2cfd511e61..c78f0af10a 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -260,7 +260,7 @@ upb_proto_reflection_library( deps = ["test_messages_proto3_editions_proto"], ) -# Export these for conformance tests for ruby codegen. +# Export these for conformance tests for ruby and C# codegen. exports_files( [ "golden/test_messages_proto2_editions.proto", @@ -268,6 +268,7 @@ exports_files( ], visibility = [ "//ruby:__pkg__", + "//src/google/protobuf/csharp:__pkg__", ], ) From bf1d7664fbe5a84dd15cf857416f40cb5188995b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 30 Apr 2024 09:43:00 -0700 Subject: [PATCH 067/179] Refactor ThreadSafeArena Reset/destructor to avoid overhead of tracking space allocated in the destructor. PiperOrigin-RevId: 629442152 --- src/google/protobuf/arena.cc | 27 +++++++++---------------- src/google/protobuf/thread_safe_arena.h | 2 +- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index e4fa0cea45..4137615964 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -85,9 +85,8 @@ SizedPtr AllocateMemory(const AllocationPolicy* policy_ptr, size_t last_size, class GetDeallocator { public: - GetDeallocator(const AllocationPolicy* policy, size_t* space_allocated) - : dealloc_(policy ? policy->block_dealloc : nullptr), - space_allocated_(space_allocated) {} + explicit GetDeallocator(const AllocationPolicy* policy) + : dealloc_(policy ? policy->block_dealloc : nullptr) {} void operator()(SizedPtr mem) const { if (dealloc_) { @@ -95,14 +94,10 @@ class GetDeallocator { } else { internal::SizedDelete(mem.p, mem.n); } - AddSpaceAllocated(mem.n); } - void AddSpaceAllocated(size_t n) const { *space_allocated_ += n; } - private: void (*dealloc_)(void*, size_t); - size_t* space_allocated_; }; } // namespace @@ -179,7 +174,7 @@ SerialArena* SerialArena::New(SizedPtr mem, ThreadSafeArena& parent) { template SizedPtr SerialArena::Free(Deallocator deallocator) { - deallocator.AddSpaceAllocated(FreeStringBlocks()); + FreeStringBlocks(); ArenaBlock* b = head(); SizedPtr mem = {b, b->size}; @@ -692,19 +687,17 @@ ThreadSafeArena::~ThreadSafeArena() { // refer to memory in other blocks. CleanupList(); - size_t space_allocated = 0; - auto mem = Free(&space_allocated); + auto mem = Free(); if (alloc_policy_.is_user_owned_initial_block()) { // Unpoison the initial block, now that it's going back to the user. PROTOBUF_UNPOISON_MEMORY_REGION(mem.p, mem.n); - space_allocated += mem.n; } else if (mem.n > 0) { - GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); + GetDeallocator(alloc_policy_.get())(mem); } } -SizedPtr ThreadSafeArena::Free(size_t* space_allocated) { - auto deallocator = GetDeallocator(alloc_policy_.get(), space_allocated); +SizedPtr ThreadSafeArena::Free() { + auto deallocator = GetDeallocator(alloc_policy_.get()); WalkSerialArenaChunk([&](SerialArenaChunk* chunk) { absl::Span> span = chunk->arenas(); @@ -730,15 +723,15 @@ SizedPtr ThreadSafeArena::Free(size_t* space_allocated) { } uint64_t ThreadSafeArena::Reset() { + const size_t space_allocated = SpaceAllocated(); + // Have to do this in a first pass, because some of the destructors might // refer to memory in other blocks. CleanupList(); // Discard all blocks except the first one. Whether it is user-provided or // allocated, always reuse the first block for the first arena. - size_t space_allocated = 0; - auto mem = Free(&space_allocated); - space_allocated += mem.n; + auto mem = Free(); // Reset the first arena with the first block. This avoids redundant // free / allocation and re-allocating for AllocationPolicy. Adjust offset if diff --git a/src/google/protobuf/thread_safe_arena.h b/src/google/protobuf/thread_safe_arena.h index 93dc6a7cfa..2332f937a5 100644 --- a/src/google/protobuf/thread_safe_arena.h +++ b/src/google/protobuf/thread_safe_arena.h @@ -207,7 +207,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { // Releases all memory except the first block which it returns. The first // block might be owned by the user and thus need some extra checks before // deleting. - SizedPtr Free(size_t* space_allocated); + SizedPtr Free(); // ThreadCache is accessed very frequently, so we align it such that it's // located within a single cache line. From e1559c8efdcfc2d66edfd10b02022ecd737534b9 Mon Sep 17 00:00:00 2001 From: Daniel Kenji Toyama Date: Tue, 30 Apr 2024 14:04:33 -0700 Subject: [PATCH 068/179] Fix typo `s/a the/the/`. PiperOrigin-RevId: 629522997 --- src/google/protobuf/text_format.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 6b3c01bfbb..dd41e5600f 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -1374,8 +1374,8 @@ class TextFormat::Parser::ParserImpl { return result; } - // Attempts to consume the supplied value. Returns false if a the - // token found does not match the value specified. + // Attempts to consume the supplied value. Returns false if the token found + // does not match the value specified. bool TryConsume(const std::string& value) { if (tokenizer_.current().text == value) { tokenizer_.Next(); From fb054c8e4141b0fe50c33d237a6650d98a412de8 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Tue, 30 Apr 2024 18:05:14 -0700 Subject: [PATCH 069/179] Java Proto lite: avoid boxing Integers accessing enum lists Provide a specialization for ListAdapter that avoids boxing ints into Integers. Use this for repeated enum accessors. In practice, the ints for most enums will fit into the JDK's "boxed int cache", avoiding extra allocations. For large enum numbers, they won't fit, and would allocate. Removing the boxing should also slightly speed up code. PiperOrigin-RevId: 629583247 --- .../java/com/google/protobuf/Internal.java | 30 +++++++++++++++++++ .../protobuf/compiler/java/lite/enum_field.cc | 25 ++++++++-------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/Internal.java b/java/core/src/main/java/com/google/protobuf/Internal.java index 3024aa9dfb..07bec5ef29 100644 --- a/java/core/src/main/java/com/google/protobuf/Internal.java +++ b/java/core/src/main/java/com/google/protobuf/Internal.java @@ -371,6 +371,36 @@ public final class Internal { return ((MessageLite) destination).toBuilder().mergeFrom((MessageLite) source).buildPartial(); } + /** + * Provides an immutable view of {@code List} around an {@code IntList}. + * + *

Protobuf internal. Used in protobuf generated code only. + */ + public static class IntListAdapter extends AbstractList { + /** Convert individual elements of the List from int to T. */ + public interface IntConverter { + T convert(int from); + } + + private final IntList fromList; + private final IntConverter converter; + + public IntListAdapter(IntList fromList, IntConverter converter) { + this.fromList = fromList; + this.converter = converter; + } + + @Override + public T get(int index) { + return converter.convert(fromList.getInt(index)); + } + + @Override + public int size() { + return fromList.size(); + } + } + /** * Provides an immutable view of {@code List} around a {@code List}. * diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index f2fb340b1b..3e86e1942f 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -597,12 +597,12 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateMembers( variables_, "private com.google.protobuf.Internal.IntList $name$_;\n" "private static final " - "com.google.protobuf.Internal.ListAdapter.Converter<\n" - " java.lang.Integer, $type$> $name$_converter_ =\n" - " new com.google.protobuf.Internal.ListAdapter.Converter<\n" - " java.lang.Integer, $type$>() {\n" + "com.google.protobuf.Internal.IntListAdapter.IntConverter<\n" + " $type$> $name$_converter_ =\n" + " new com.google.protobuf.Internal.IntListAdapter.IntConverter<\n" + " $type$>() {\n" " @java.lang.Override\n" - " public $type$ convert(java.lang.Integer from) {\n" + " public $type$ convert(int from) {\n" " $type$ result = $type$.forNumber(from);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -610,14 +610,13 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateMembers( PrintExtraFieldInfo(variables_, printer); WriteFieldAccessorDocComment(printer, descriptor_, LIST_GETTER, context_->options()); - printer->Print( - variables_, - "@java.lang.Override\n" - "$deprecation$public java.util.List<$type$> " - "${$get$capitalized_name$List$}$() {\n" - " return new com.google.protobuf.Internal.ListAdapter<\n" - " java.lang.Integer, $type$>($name$_, $name$_converter_);\n" - "}\n"); + printer->Print(variables_, + "@java.lang.Override\n" + "$deprecation$public java.util.List<$type$> " + "${$get$capitalized_name$List$}$() {\n" + " return new com.google.protobuf.Internal.IntListAdapter<\n" + " $type$>($name$_, $name$_converter_);\n" + "}\n"); printer->Annotate("{", "}", descriptor_); WriteFieldAccessorDocComment(printer, descriptor_, LIST_COUNT, context_->options()); From 12a828c7005e1225eaf32afb7f9236e03dcbcc44 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 May 2024 05:53:04 -0700 Subject: [PATCH 070/179] Add UPB_ASSUME(!IsExtension) on GetMutableMessage. This function is UB to call on an extension field. PiperOrigin-RevId: 629701038 --- upb/message/accessors.h | 1 + 1 file changed, 1 insertion(+) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index 09ea66ae5b..f812e2da47 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -380,6 +380,7 @@ UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( const upb_MiniTableField* field, upb_Arena* arena) { UPB_ASSERT(arena); UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); + UPB_ASSUME(!upb_MiniTableField_IsExtension(field)); upb_Message* sub_message = *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); if (!sub_message) { From 15c07711d0165b9012b397ffe411bd7bcac65d89 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 May 2024 13:08:05 +0000 Subject: [PATCH 071/179] Auto-generate files after cl/629701038 --- php/ext/google/protobuf/php-upb.h | 1 + ruby/ext/google/protobuf_c/ruby-upb.h | 1 + 2 files changed, 2 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index b97bd1db2b..e8f88a62b3 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -3537,6 +3537,7 @@ UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( const upb_MiniTableField* field, upb_Arena* arena) { UPB_ASSERT(arena); UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); + UPB_ASSUME(!upb_MiniTableField_IsExtension(field)); upb_Message* sub_message = *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); if (!sub_message) { diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 7307f5e252..eebfde85aa 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -3539,6 +3539,7 @@ UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( const upb_MiniTableField* field, upb_Arena* arena) { UPB_ASSERT(arena); UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); + UPB_ASSUME(!upb_MiniTableField_IsExtension(field)); upb_Message* sub_message = *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); if (!sub_message) { From 4cbb7b8540eec77dbe6360d6b70c822f01bed815 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Wed, 1 May 2024 07:20:50 -0700 Subject: [PATCH 072/179] Duplicate the integer tests using octal and hex. PiperOrigin-RevId: 629715987 --- conformance/text_format_conformance_suite.cc | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index d94f3f2f96..295b10b046 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -321,6 +321,44 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { "optional_int64: -9223372036854775808"); RunValidTextFormatTest("Uint64FieldMaxValue", REQUIRED, "optional_uint64: 18446744073709551615"); + // Integer fields - Hex + RunValidTextFormatTestWithExpected("Int32FieldMaxValueHex", REQUIRED, + "optional_int32: 0x7FFFFFFF", + "optional_int32: 2147483647"); + RunValidTextFormatTestWithExpected("Int32FieldMinValueHex", REQUIRED, + "optional_int32: -0x80000000", + "optional_int32: -2147483648"); + RunValidTextFormatTestWithExpected("Uint32FieldMaxValueHex", REQUIRED, + "optional_uint32: 0xFFFFFFFF", + "optional_uint32: 4294967295"); + RunValidTextFormatTestWithExpected("Int64FieldMaxValueHex", REQUIRED, + "optional_int64: 0x7FFFFFFFFFFFFFFF", + "optional_int64: 9223372036854775807"); + RunValidTextFormatTestWithExpected("Int64FieldMinValueHex", REQUIRED, + "optional_int64: -0x8000000000000000", + "optional_int64: -9223372036854775808"); + RunValidTextFormatTestWithExpected("Uint64FieldMaxValueHex", REQUIRED, + "optional_uint64: 0xFFFFFFFFFFFFFFFF", + "optional_uint64: 18446744073709551615"); + // Integer fields - Octal + RunValidTextFormatTestWithExpected("Int32FieldMaxValueOctal", REQUIRED, + "optional_int32: 017777777777", + "optional_int32: 2147483647"); + RunValidTextFormatTestWithExpected("Int32FieldMinValueOctal", REQUIRED, + "optional_int32: -020000000000", + "optional_int32: -2147483648"); + RunValidTextFormatTestWithExpected("Uint32FieldMaxValueOctal", REQUIRED, + "optional_uint32: 037777777777", + "optional_uint32: 4294967295"); + RunValidTextFormatTestWithExpected("Int64FieldMaxValueOctal", REQUIRED, + "optional_int64: 0777777777777777777777", + "optional_int64: 9223372036854775807"); + RunValidTextFormatTestWithExpected("Int64FieldMinValueOctal", REQUIRED, + "optional_int64: -01000000000000000000000", + "optional_int64: -9223372036854775808"); + RunValidTextFormatTestWithExpected("Uint64FieldMaxValueOctal", REQUIRED, + "optional_uint64: 01777777777777777777777", + "optional_uint64: 18446744073709551615"); // Parsers reject out-of-bound integer values. ExpectParseFailure("Int32FieldTooLarge", REQUIRED, @@ -335,6 +373,32 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { "optional_int64: -9223372036854775809"); ExpectParseFailure("Uint64FieldTooLarge", REQUIRED, "optional_uint64: 18446744073709551616"); + // Parsers reject out-of-bound integer values - Hex + ExpectParseFailure("Int32FieldTooLargeHex", REQUIRED, + "optional_int32: 0x80000000"); + ExpectParseFailure("Int32FieldTooSmallHex", REQUIRED, + "optional_int32: -0x80000001"); + ExpectParseFailure("Uint32FieldTooLargeHex", REQUIRED, + "optional_uint32: 0x100000000"); + ExpectParseFailure("Int64FieldTooLargeHex", REQUIRED, + "optional_int64: 0x8000000000000000"); + ExpectParseFailure("Int64FieldTooSmallHex", REQUIRED, + "optional_int64: -0x8000000000000001"); + ExpectParseFailure("Uint64FieldTooLargeHex", REQUIRED, + "optional_uint64: 0x10000000000000000"); + // Parsers reject out-of-bound integer values - Octal + ExpectParseFailure("Int32FieldTooLargeOctal", REQUIRED, + "optional_int32: 020000000000"); + ExpectParseFailure("Int32FieldTooSmallOctal", REQUIRED, + "optional_int32: -020000000001"); + ExpectParseFailure("Uint32FieldTooLargeOctal", REQUIRED, + "optional_uint32: 040000000000"); + ExpectParseFailure("Int64FieldTooLargeOctal", REQUIRED, + "optional_int64: 01000000000000000000000"); + ExpectParseFailure("Int64FieldTooSmallOctal", REQUIRED, + "optional_int64: -01000000000000000000001"); + ExpectParseFailure("Uint64FieldTooLargeOctal", REQUIRED, + "optional_uint64: 02000000000000000000000"); // Floating point fields RunValidTextFormatTest("FloatField", REQUIRED, "optional_float: 3.192837"); From 26cf1cb94caeacfe0e4d0852307d8ca6d3a31ee3 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Wed, 1 May 2024 07:42:01 -0700 Subject: [PATCH 073/179] Add more floating point cases to the validations. Using https://protobuf.dev/reference/protobuf/textformat-spec as a guide; add valid suffixes, more cases for special values, confirm some invalid things do fail. Also resort the files since some weren't and sorting seems to be required for the update helper. PiperOrigin-RevId: 629720194 --- conformance/text_format_conformance_suite.cc | 101 ++++++++++++++---- conformance/text_format_failure_list_java.txt | 21 ++-- .../text_format_failure_list_python.txt | 76 +++++++++++-- .../text_format_failure_list_python_cpp.txt | 68 +++++++++++- .../text_format_failure_list_python_upb.txt | 68 +++++++++++- 5 files changed, 289 insertions(+), 45 deletions(-) diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index 295b10b046..94f63dbb60 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -401,28 +401,85 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { "optional_uint64: 02000000000000000000000"); // Floating point fields - RunValidTextFormatTest("FloatField", REQUIRED, "optional_float: 3.192837"); - RunValidTextFormatTest("FloatFieldWithVeryPreciseNumber", REQUIRED, - "optional_float: 3.123456789123456789"); - RunValidTextFormatTest("FloatFieldMaxValue", REQUIRED, - "optional_float: 3.4028235e+38"); - RunValidTextFormatTest("FloatFieldMinValue", REQUIRED, - "optional_float: 1.17549e-38"); - RunValidTextFormatTest("FloatFieldNaNValue", REQUIRED, "optional_float: NaN"); - RunValidTextFormatTest("FloatFieldPosInfValue", REQUIRED, - "optional_float: inf"); - RunValidTextFormatTest("FloatFieldNegInfValue", REQUIRED, - "optional_float: -inf"); - RunValidTextFormatTest("FloatFieldWithInt32Max", REQUIRED, - "optional_float: 4294967296"); - RunValidTextFormatTest("FloatFieldLargerThanInt64", REQUIRED, - "optional_float: 9223372036854775808"); - RunValidTextFormatTest("FloatFieldTooLarge", REQUIRED, - "optional_float: 3.4028235e+39"); - RunValidTextFormatTest("FloatFieldTooSmall", REQUIRED, - "optional_float: 1.17549e-39"); - RunValidTextFormatTest("FloatFieldLargerThanUint64", REQUIRED, - "optional_float: 18446744073709551616"); + for (const auto& suffix : std::vector{"", "f", "F"}) { + const std::string name_suffix = + suffix.empty() ? "" : absl::StrCat("_", suffix); + + RunValidTextFormatTest(absl::StrCat("FloatField", name_suffix), REQUIRED, + absl::StrCat("optional_float: 3.192837", suffix)); + RunValidTextFormatTestWithExpected( + absl::StrCat("FloatFieldZero", name_suffix), REQUIRED, + absl::StrCat("optional_float: 0", suffix), + "" /* implicit presence, so zero means unset*/); + RunValidTextFormatTest(absl::StrCat("FloatFieldNegative", name_suffix), + REQUIRED, + absl::StrCat("optional_float: -3.192837", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldWithVeryPreciseNumber", name_suffix), REQUIRED, + absl::StrCat("optional_float: 3.123456789123456789", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldMaxValue", name_suffix), REQUIRED, + absl::StrCat("optional_float: 3.4028235e+38", suffix)); + RunValidTextFormatTest(absl::StrCat("FloatFieldMinValue", name_suffix), + REQUIRED, + absl::StrCat("optional_float: 1.17549e-38", suffix)); + RunValidTextFormatTest(absl::StrCat("FloatFieldWithInt32Max", name_suffix), + REQUIRED, + absl::StrCat("optional_float: 4294967296", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldLargerThanInt64", name_suffix), REQUIRED, + absl::StrCat("optional_float: 9223372036854775808", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldTooLarge", name_suffix), REQUIRED, + absl::StrCat("optional_float: 3.4028235e+39", suffix)); + RunValidTextFormatTest(absl::StrCat("FloatFieldTooSmall", name_suffix), + REQUIRED, + absl::StrCat("optional_float: 1.17549e-39", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldLargerThanUint64", name_suffix), REQUIRED, + absl::StrCat("optional_float: 18446744073709551616", suffix)); + // https://protobuf.dev/reference/protobuf/textformat-spec/#literals says + // "-0" is a valid float literal. + // TODO: Figure out if this should count as not setting + // presence or if -0 should be reflected back. + // RunValidTextFormatTestWithExpected( + // absl::StrCat("FloatFieldNegativeZero", name_suffix), REQUIRED, + // absl::StrCat("optional_float: -0", suffix), + // "" /* implicit presence, so zero means unset*/); + // https://protobuf.dev/reference/protobuf/textformat-spec/#literals says + // ".123", "-.123", ".123e2" are a valid float literal. + RunValidTextFormatTest(absl::StrCat("FloatFieldNoLeadingZero", name_suffix), + REQUIRED, + absl::StrCat("optional_float: .123", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldNegativeNoLeadingZero", name_suffix), REQUIRED, + absl::StrCat("optional_float: -.123", suffix)); + RunValidTextFormatTest( + absl::StrCat("FloatFieldNoLeadingZeroWithExponent", name_suffix), + REQUIRED, absl::StrCat("optional_float: .123e2", suffix)); + } + // https://protobuf.dev/reference/protobuf/textformat-spec/#value say case + // doesn't matter for special values, test a few + for (const auto& value : std::vector{"nan", "NaN", "nAn"}) { + RunValidTextFormatTest(absl::StrCat("FloatFieldValue_", value), REQUIRED, + absl::StrCat("optional_float: ", value)); + } + for (const auto& value : std::vector{ + "inf", "infinity", "INF", "INFINITY", "iNF", "inFINITY"}) { + RunValidTextFormatTest(absl::StrCat("FloatFieldValue_Pos", value), REQUIRED, + absl::StrCat("optional_float: ", value)); + RunValidTextFormatTest(absl::StrCat("FloatFieldValue_Neg", value), REQUIRED, + absl::StrCat("optional_float: -", value)); + } + // https://protobuf.dev/reference/protobuf/textformat-spec/#numeric and + // https://protobuf.dev/reference/protobuf/textformat-spec/#value says + // hex or octal float literals are invalid. + ExpectParseFailure("FloatFieldNoHex", REQUIRED, "optional_float: 0x1"); + ExpectParseFailure("FloatFieldNoNegativeHex", REQUIRED, + "optional_float: -0x1"); + ExpectParseFailure("FloatFieldNoOctal", REQUIRED, "optional_float: 012"); + ExpectParseFailure("FloatFieldNoNegativeOctal", REQUIRED, + "optional_float: -012"); // String literals x {Strings, Bytes} for (const auto& field_type : std::vector{"String", "Bytes"}) { diff --git a/conformance/text_format_failure_list_java.txt b/conformance/text_format_failure_list_java.txt index 8dea2862cd..a035453944 100644 --- a/conformance/text_format_failure_list_java.txt +++ b/conformance/text_format_failure_list_java.txt @@ -1,17 +1,20 @@ -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput -Required.Proto3.TextFormatInput.AnyField.ProtobufOutput -Required.Proto3.TextFormatInput.AnyField.TextFormatOutput Recommended.Editions_Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput Recommended.Editions_Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput Recommended.Editions_Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput Recommended.Editions_Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput Required.Editions_Proto3.TextFormatInput.AnyField.ProtobufOutput Required.Editions_Proto3.TextFormatInput.AnyField.TextFormatOutput - +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal +Required.Proto3.TextFormatInput.AnyField.ProtobufOutput +Required.Proto3.TextFormatInput.AnyField.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Proto3.TextFormatInput.FloatFieldNoOctal Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal \ No newline at end of file diff --git a/conformance/text_format_failure_list_python.txt b/conformance/text_format_failure_list_python.txt index 6754aa4c4b..903160103f 100644 --- a/conformance/text_format_failure_list_python.txt +++ b/conformance/text_format_failure_list_python.txt @@ -1,15 +1,79 @@ # This is the list of text format conformance tests that are known to fail right # now. # TODO: These should be fixed. -Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Proto3.TextFormatInput.FloatFieldNoOctal +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput diff --git a/conformance/text_format_failure_list_python_cpp.txt b/conformance/text_format_failure_list_python_cpp.txt index 037ca00e13..870f686fcb 100644 --- a/conformance/text_format_failure_list_python_cpp.txt +++ b/conformance/text_format_failure_list_python_cpp.txt @@ -1,8 +1,68 @@ -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Proto3.TextFormatInput.FloatFieldNoOctal +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput diff --git a/conformance/text_format_failure_list_python_upb.txt b/conformance/text_format_failure_list_python_upb.txt index 377998b448..a4a6fc69c0 100644 --- a/conformance/text_format_failure_list_python_upb.txt +++ b/conformance/text_format_failure_list_python_upb.txt @@ -1,11 +1,71 @@ # This is the list of text format conformance tests that are known to fail right # now. # TODO: These should be fixed. -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal +Required.Proto3.TextFormatInput.FloatFieldNoOctal +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput From a45e0d83abe1b9620c91c8044535236774af3556 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 May 2024 07:57:22 -0700 Subject: [PATCH 074/179] Change `FieldDescriptor::type()` to not have the call_once behavior. The field is always set while building the descriptor, even for lazy descriptors. The `type()` function is called extensively in reflection based code and removing the call_once has significant performance impact. PiperOrigin-RevId: 629723749 --- src/google/protobuf/descriptor.cc | 34 ++++++++++++---------- src/google/protobuf/descriptor.h | 5 +--- src/google/protobuf/descriptor_unittest.cc | 16 +++++----- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 50d5a0a775..e16e11db4d 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -2808,7 +2808,7 @@ bool DescriptorPool::TryFindExtensionInFallbackDatabase( // =================================================================== bool FieldDescriptor::is_map_message_type() const { - return type_descriptor_.message_type->options().map_entry(); + return message_type()->options().map_entry(); } std::string FieldDescriptor::DefaultValueAsString( @@ -6503,12 +6503,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, result->has_json_name_ = proto.has_json_name(); - // Some compilers do not allow static_cast directly between two enum types, - // so we must cast to int first. - result->type_ = static_cast( - absl::implicit_cast(proto.type())); - result->label_ = static_cast( - absl::implicit_cast(proto.label())); + result->type_ = proto.type(); + result->label_ = proto.label(); result->is_repeated_ = result->label_ == FieldDescriptor::LABEL_REPEATED; if (result->label() == FieldDescriptor::LABEL_REQUIRED) { @@ -7388,6 +7384,10 @@ void DescriptorBuilder::CrossLinkField(FieldDescriptor* field, if (type.IsNull()) { if (is_lazy) { + ABSL_CHECK(field->type_ == FieldDescriptor::TYPE_MESSAGE || + field->type_ == FieldDescriptor::TYPE_GROUP || + field->type_ == FieldDescriptor::TYPE_ENUM) + << proto; // Save the symbol names for later for lookup, and allocate the once // object needed for the accessors. const std::string& name = proto.type_name(); @@ -9584,19 +9584,23 @@ void FieldDescriptor::TypeOnceInit(const FieldDescriptor* to_init) { // all share the same absl::call_once init path to do lazy // import building and cross linking of a field of a message. const Descriptor* FieldDescriptor::message_type() const { - if (type_once_) { - absl::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); + if (type_ == TYPE_MESSAGE || type_ == TYPE_GROUP) { + if (type_once_) { + absl::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); + } + return type_descriptor_.message_type; } - return type_ == TYPE_MESSAGE || type_ == TYPE_GROUP - ? type_descriptor_.message_type - : nullptr; + return nullptr; } const EnumDescriptor* FieldDescriptor::enum_type() const { - if (type_once_) { - absl::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); + if (type_ == TYPE_ENUM) { + if (type_once_) { + absl::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); + } + return type_descriptor_.enum_type; } - return type_ == TYPE_ENUM ? type_descriptor_.enum_type : nullptr; + return nullptr; } const EnumValueDescriptor* FieldDescriptor::default_value_enum() const { diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 443454543d..48d79022e2 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -1090,7 +1090,7 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase, uint8_t label_ : 2; // Actually a `Type`, but stored as uint8_t to save space. - mutable uint8_t type_; + uint8_t type_; // Logically: // all_names_ = [name, full_name, lower, camel, json] @@ -2676,9 +2676,6 @@ inline FieldDescriptor::Label FieldDescriptor::label() const { } inline FieldDescriptor::Type FieldDescriptor::type() const { - if (type_once_) { - absl::call_once(*type_once_, &FieldDescriptor::TypeOnceInit, this); - } return static_cast(type_); } diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index ad37a443bc..adb07750ae 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -13035,7 +13035,7 @@ TEST_F(LazilyBuildDependenciesTest, Type) { const FileDescriptor* file = pool_.FindFileByName("foo.proto"); - // Verify calling type() on a field that is a message type will + // Verify calling type() on a field that is a message type will _not_ // build the type defined in another file. EXPECT_FALSE(pool_.InternalIsFileLoaded("message1.proto")); const Descriptor* desc = file->FindMessageTypeByName("Lazy"); @@ -13043,31 +13043,31 @@ TEST_F(LazilyBuildDependenciesTest, Type) { const FieldDescriptor* field = desc->FindFieldByName("message1"); EXPECT_TRUE(field != nullptr); EXPECT_EQ(field->type(), FieldDescriptor::TYPE_MESSAGE); - EXPECT_TRUE(pool_.InternalIsFileLoaded("message1.proto")); + EXPECT_FALSE(pool_.InternalIsFileLoaded("message1.proto")); - // Verify calling cpp_type() on a field that is a message type will + // Verify calling cpp_type() on a field that is a message type will _not_ // build the type defined in another file. EXPECT_FALSE(pool_.InternalIsFileLoaded("message2.proto")); field = desc->FindFieldByName("message2"); EXPECT_TRUE(field != nullptr); EXPECT_EQ(field->cpp_type(), FieldDescriptor::CPPTYPE_MESSAGE); - EXPECT_TRUE(pool_.InternalIsFileLoaded("message2.proto")); + EXPECT_FALSE(pool_.InternalIsFileLoaded("message2.proto")); - // Verify calling type() on a field that is an enum type will + // Verify calling type() on a field that is an enum type will _not_ // build the type defined in another file. EXPECT_FALSE(pool_.InternalIsFileLoaded("enum1.proto")); field = desc->FindFieldByName("enum1"); EXPECT_TRUE(field != nullptr); EXPECT_EQ(field->type(), FieldDescriptor::TYPE_ENUM); - EXPECT_TRUE(pool_.InternalIsFileLoaded("enum1.proto")); + EXPECT_FALSE(pool_.InternalIsFileLoaded("enum1.proto")); - // Verify calling cpp_type() on a field that is an enum type will + // Verify calling cpp_type() on a field that is an enum type will _not_ // build the type defined in another file. EXPECT_FALSE(pool_.InternalIsFileLoaded("enum2.proto")); field = desc->FindFieldByName("enum2"); EXPECT_TRUE(field != nullptr); EXPECT_EQ(field->cpp_type(), FieldDescriptor::CPPTYPE_ENUM); - EXPECT_TRUE(pool_.InternalIsFileLoaded("enum2.proto")); + EXPECT_FALSE(pool_.InternalIsFileLoaded("enum2.proto")); } TEST_F(LazilyBuildDependenciesTest, Extension) { From 7c37b2d2a19e0599949588b992e4bbdaba9b29c0 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Wed, 1 May 2024 08:11:33 -0700 Subject: [PATCH 075/179] Add text format conformance tests for field separators. PiperOrigin-RevId: 629727312 --- conformance/text_format_conformance_suite.cc | 90 ++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index 94f63dbb60..3eca94304c 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -9,6 +9,7 @@ #include #include +#include #include #include "absl/log/absl_log.h" @@ -559,6 +560,95 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { absl::StrCat(field_name, ": '\\xc0'")); } + // Separators + for (const auto& test_case : std::vector>{ + {"string", "\"abc\""}, + {"bytes", "\"abc\""}, + {"int32", "123"}, + {"bool", "true"}, + {"double", "1.23"}, + {"fixed32", "0x123"}, + }) { + // Optional Field Separators + for (const auto& field_type : + std::vector{"Single", "Repeated"}) { + std::string field_name, field_value; + if (field_type == "Single") { + field_name = absl::StrCat("optional_", test_case.first); + field_value = test_case.second; + } else { + field_name = absl::StrCat("repeated_", test_case.first); + field_value = absl::StrCat("[", test_case.second, "]"); + } + + RunValidTextFormatTest(absl::StrCat("FieldSeparatorCommaTopLevel", + field_type, "_", test_case.first), + REQUIRED, + absl::StrCat(field_name, ": ", field_value, ",")); + RunValidTextFormatTest(absl::StrCat("FieldSeparatorSemiTopLevelSingle", + field_type, "_", test_case.first), + REQUIRED, + absl::StrCat(field_name, ": ", field_value, ";")); + + ExpectParseFailure( + absl::StrCat("FieldSeparatorCommaTopLevelDuplicatesFails", field_type, + "_", test_case.first), + REQUIRED, absl::StrCat(field_name, ": ", field_value, ",,")); + ExpectParseFailure( + absl::StrCat("FieldSeparatorSemiTopLevelDuplicateFails", field_type, + "_", test_case.first), + REQUIRED, absl::StrCat(field_name, ": ", field_value, ";;")); + } + + // Required List Separators + RunValidTextFormatTest( + absl::StrCat("ListSeparator_", test_case.first), REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, ",", + test_case.second, "]")); + ExpectParseFailure( + absl::StrCat("ListSeparatorSemiFails_", test_case.first), REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, ";", + test_case.second, "]")); + // For string and bytes, if we skip the separator, the parser will treat + // the two values as a single value. + if (test_case.first == "string" || test_case.first == "bytes") { + RunValidTextFormatTest( + absl::StrCat("ListSeparatorMissingIsOneValue_", test_case.first), + REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, + " ", test_case.second, "]")); + } else { + ExpectParseFailure( + absl::StrCat("ListSeparatorMissingFails_", test_case.first), REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, + " ", test_case.second, "]")); + } + ExpectParseFailure( + absl::StrCat("ListSeparatorDuplicateFails_", test_case.first), REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, + ",,", test_case.second, "]")); + ExpectParseFailure( + absl::StrCat("ListSeparatorSingleTrailingFails_", test_case.first), + REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, + ",]")); + ExpectParseFailure( + absl::StrCat("ListSeparatorTwoValuesTrailingFails_", test_case.first), + REQUIRED, + absl::StrCat("repeated_", test_case.first, ": [", test_case.second, ",", + test_case.second, ",]")); + } + // The test message don't really have all types nested, so just check one + // data type for the nested field separator support + RunValidTextFormatTest("FieldSeparatorCommaNested", REQUIRED, + "optional_nested_message: { a: 123, }"); + RunValidTextFormatTest("FieldSeparatorSemiNested", REQUIRED, + "optional_nested_message: { a: 123; }"); + ExpectParseFailure("FieldSeparatorCommaNestedDuplicates", REQUIRED, + "optional_nested_message: { a: 123,, }"); + ExpectParseFailure("FieldSeparatorSemiNestedDuplicates", REQUIRED, + "optional_nested_message: { a: 123;; }"); + // Unknown Fields UnknownToTestAllTypes message; // Unable to print unknown Fixed32/Fixed64 fields as if they are known. From 9c8e0e6b925aa58c58ba0e71a6f596224afa7c74 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Wed, 1 May 2024 09:27:15 -0700 Subject: [PATCH 076/179] generated_message_tctable_impl typo fix: registeer -> register PiperOrigin-RevId: 629745603 --- src/google/protobuf/generated_message_tctable_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 4cd6e9582b..1a214be1bb 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -1002,7 +1002,7 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ParseLoop( while (!ctx->Done(&ptr)) { #if defined(__GNUC__) // Note: this asm prevents the compiler (clang, specifically) from - // believing (thanks to CSE) that it needs to dedicate a registeer both + // believing (thanks to CSE) that it needs to dedicate a register both // to "table" and "&table->fast_entries". // TODO: remove this asm asm("" : "+r"(table)); From c4bf83b534e9736fe31e1f8ad6f640e316836ba5 Mon Sep 17 00:00:00 2001 From: Kyle Montemayor Date: Wed, 1 May 2024 10:22:51 -0700 Subject: [PATCH 077/179] Automated rollback of commit f799af8bf5c1ff24fa4b6aedfeaef99ef09dd16e. PiperOrigin-RevId: 629760811 --- .../protobuf/internal/enum_type_wrapper.py | 11 ++++++++++ .../google/protobuf/internal/message_test.py | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/python/google/protobuf/internal/enum_type_wrapper.py b/python/google/protobuf/internal/enum_type_wrapper.py index cc65fc0bc8..1fa6ab9f58 100644 --- a/python/google/protobuf/internal/enum_type_wrapper.py +++ b/python/google/protobuf/internal/enum_type_wrapper.py @@ -12,6 +12,8 @@ on proto classes. For usage, see: reflection_test.py """ +import sys + __author__ = 'rabsatt@google.com (Kevin Rabsatt)' @@ -99,3 +101,12 @@ class EnumTypeWrapper(object): pass # fall out to break exception chaining raise AttributeError('Enum {} has no value defined for name {!r}'.format( self._enum_type.name, name)) + + def __or__(self, other): + """Returns the union type of self and other.""" + if sys.version_info >= (3, 10): + return type(self) | other + else: + raise NotImplementedError( + 'You may not use | on EnumTypes (or classes) below python 3.10' + ) diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 68f1999b16..7b4478e245 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -22,6 +22,7 @@ import operator import pickle import pydoc import sys +import types import unittest from unittest import mock import warnings @@ -30,6 +31,7 @@ cmp = lambda x, y: (x > y) - (x < y) from google.protobuf.internal import api_implementation # pylint: disable=g-import-not-at-top from google.protobuf.internal import encoder +from google.protobuf.internal import enum_type_wrapper from google.protobuf.internal import more_extensions_pb2 from google.protobuf.internal import more_messages_pb2 from google.protobuf.internal import packed_field_test_pb2 @@ -1314,6 +1316,26 @@ class MessageTest(unittest.TestCase): self.assertNotEqual(m, ComparesWithFoo()) self.assertNotEqual(ComparesWithFoo(), m) + def testTypeUnion(self, message_module): + # Below python 3.10 you cannot create union types with the | operator, so we + # skip testing for unions with old versions. + if sys.version_info < (3, 10): + return + enum_type = enum_type_wrapper.EnumTypeWrapper( + message_module.TestAllTypes.NestedEnum.DESCRIPTOR + ) + union_type = enum_type | int + self.assertIsInstance(union_type, types.UnionType) + + def get_union() -> union_type: + return enum_type + + union = get_union() + self.assertIsInstance(union, enum_type_wrapper.EnumTypeWrapper) + self.assertEqual( + union.DESCRIPTOR, message_module.TestAllTypes.NestedEnum.DESCRIPTOR + ) + # Class to test proto2-only features (required, extensions, etc.) @testing_refleaks.TestCase From 7a09c4569bec008504bca3eb0201f0e75863a1fc Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Wed, 1 May 2024 11:30:23 -0700 Subject: [PATCH 078/179] Add upb_Message_ClearOneof upb users currently need to manually fetch a oneof field in order to clear it. In this CL, we add a convenience method to do that in one fell swoop. PiperOrigin-RevId: 629782904 --- upb/message/accessors.h | 8 +++----- upb/message/accessors_test.cc | 23 +++++++++++++++++++++++ upb/message/internal/accessors.h | 21 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index f812e2da47..723d10d871 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -68,11 +68,9 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} +UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f); // NOTE: The default_val is only used for fields that support presence. // For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a diff --git a/upb/message/accessors_test.cc b/upb/message/accessors_test.cc index 5b0dd3638c..524e5fb655 100644 --- a/upb/message/accessors_test.cc +++ b/upb/message/accessors_test.cc @@ -477,5 +477,28 @@ TEST(GeneratedCode, EnumClosedCheck) { EXPECT_TRUE(upb_MiniTableField_IsClosedEnum(closedEnumField)); upb_Arena_Free(arena); } +TEST(GeneratedCode, OneofClear) { + upb_Arena* arena = upb_Arena_New(); + + protobuf_test_messages_proto2_TestAllTypesProto2* msg = + protobuf_test_messages_proto2_TestAllTypesProto2_new(arena); + + const upb_MiniTable* table = + &protobuf_0test_0messages__proto2__TestAllTypesProto2_msg_init; + + // oneof_uint32 + const upb_MiniTableField* oneofField = + upb_MiniTable_FindFieldByNumber(table, 111); + EXPECT_TRUE(upb_MiniTableField_IsInOneof(oneofField)); + protobuf_test_messages_proto2_TestAllTypesProto2_set_oneof_uint32(msg, 522); + EXPECT_TRUE( + protobuf_test_messages_proto2_TestAllTypesProto2_has_oneof_uint32(msg)); + + upb_Message_ClearOneof((upb_Message*)msg, table, oneofField); + EXPECT_FALSE( + protobuf_test_messages_proto2_TestAllTypesProto2_has_oneof_uint32(msg)); + + upb_Arena_Free(arena); +} } // namespace diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index bfae12e601..1c5a397487 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -335,6 +335,27 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const struct upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} + +UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); + if (field_number == 0) { + // No field in the oneof is set. + return; + } + + const upb_MiniTableField* field = + upb_MiniTable_FindFieldByNumber(m, field_number); + upb_Message_ClearBaseField(msg, field); +} + UPB_INLINE void _upb_Message_AssertMapIsUntagged( const struct upb_Message* msg, const upb_MiniTableField* field) { UPB_UNUSED(msg); From 8a11178606ce85d9e561729328795471fe8a6de5 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Wed, 1 May 2024 11:41:02 -0700 Subject: [PATCH 079/179] Add system_python to protobuf_deps.bzl. These are needed to get python headers for Python C++ and Python UPB from local system python PiperOrigin-RevId: 629786458 --- WORKSPACE | 7 ------- protobuf_deps.bzl | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 63785c3f4b..0e620deda3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -164,13 +164,6 @@ http_archive( patch_cmds = ["find google -type f -name BUILD.bazel -delete"], ) -load("//python/dist:system_python.bzl", "system_python") - -system_python( - name = "system_python", - minimum_python_version = "3.7", -) - load("@system_python//:pip.bzl", "pip_parse") pip_parse( diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 19c51746e0..9fee18a2a6 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -2,6 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//python/dist:python_downloads.bzl", "python_nuget_package", "python_source_archive") +load("//python/dist:system_python.bzl", "system_python") PROTOBUF_MAVEN_ARTIFACTS = [ "com.google.caliper:caliper:1.0-beta-3", @@ -104,6 +105,12 @@ def protobuf_deps(): url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", ) + if not native.existing_rule("system_python"): + system_python( + name = "system_python", + minimum_python_version = "3.7", + ) + if not native.existing_rule("rules_jvm_external"): _github_archive( name = "rules_jvm_external", From 990380bc553a43ffe264f3ee7ee7048a983d8909 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 May 2024 18:44:18 +0000 Subject: [PATCH 080/179] Auto-generate files after cl/629782904 --- php/ext/google/protobuf/php-upb.h | 29 ++++++++++++++++++++++----- ruby/ext/google/protobuf_c/ruby-upb.h | 29 ++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index e8f88a62b3..c426679dc0 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -2914,6 +2914,27 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const struct upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} + +UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); + if (field_number == 0) { + // No field in the oneof is set. + return; + } + + const upb_MiniTableField* field = + upb_MiniTable_FindFieldByNumber(m, field_number); + upb_Message_ClearBaseField(msg, field); +} + UPB_INLINE void _upb_Message_AssertMapIsUntagged( const struct upb_Message* msg, const upb_MiniTableField* field) { UPB_UNUSED(msg); @@ -3225,11 +3246,9 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} +UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f); // NOTE: The default_val is only used for fields that support presence. // For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index eebfde85aa..028e076eb4 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -2916,6 +2916,27 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const struct upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} + +UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); + if (field_number == 0) { + // No field in the oneof is set. + return; + } + + const upb_MiniTableField* field = + upb_MiniTable_FindFieldByNumber(m, field_number); + upb_Message_ClearBaseField(msg, field); +} + UPB_INLINE void _upb_Message_AssertMapIsUntagged( const struct upb_Message* msg, const upb_MiniTableField* field) { UPB_UNUSED(msg); @@ -3227,11 +3248,9 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} +UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f); // NOTE: The default_val is only used for fields that support presence. // For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a From 4f23cafa0209c0f0048002b7c0c18ff2b928a327 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Wed, 1 May 2024 12:40:51 -0700 Subject: [PATCH 081/179] Cancel in-progress workflow runs for pull requests and dispatches (#16601) This will reduce unnecessary resource consumption by cancelling running workflows when new commits are pushed to a PR or workflow dispatch. It does not cancel pushes to main or branches. Closes #16601 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/16601 from sethvargo:sethvargo/actions-concurrency 82fd070dbad88920501f0f4452f93b687adc3120 PiperOrigin-RevId: 629805311 --- .github/workflows/test_runner.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index d67f45593b..70cfc27a4a 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml @@ -52,6 +52,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ contains(fromJSON('["pull_request", "pull_request_target", "workflow_dispatch"]'), github.event_name) }} + jobs: check-tag: name: Check for Safety From 39a1c6a46e6921577cab24440fb16e7a1e903f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Wed, 1 May 2024 12:47:43 -0700 Subject: [PATCH 082/179] In the Kotlin DSL, reference builder methods with property syntax. Instead of _builder.getFoo() and _builder.setFoo(value), the generated code now uses _builder.foo and _builder.foo = value. When compiling against the Java Proto API, this makes no difference, since the Kotlin compiler treats Java methods getFoo/setFoo as defining a Kotlin property foo. But if a compatible proto API is implemented in Kotlin then there is no such equivalence. Such an implementation would have to define either both forms or just the one that the DSL uses. For a Kotlin API it is more natural to use properties. Similarly, the generated Java methods getFoosList(), getFoosCount(), getFoosMap(), and getEnumValue() are accessed via property names (foosList, foosCount, foosMap, enumValue). (This is the second application of this change. The first had to be rolled back because it didn't handle conflict-resolution names correctly.) PiperOrigin-RevId: 629807104 --- .../kotlin/com/google/protobuf/Proto2Test.kt | 28 +++++++--- .../kotlin/com/google/protobuf/Proto3Test.kt | 14 ++++- .../google/protobuf/evil_names_proto2.proto | 9 +++ .../google/protobuf/evil_names_proto3.proto | 6 ++ src/google/protobuf/compiler/java/BUILD.bazel | 1 + .../protobuf/compiler/java/field_common.cc | 55 +++++++++++++++++++ .../compiler/java/immutable/enum_field.cc | 10 ++-- .../compiler/java/immutable/map_field.cc | 2 +- .../compiler/java/immutable/message_field.cc | 6 +- .../java/immutable/primitive_field.cc | 31 ++++++++--- .../compiler/java/immutable/string_field.cc | 6 +- .../protobuf/compiler/java/lite/enum_field.cc | 10 ++-- .../protobuf/compiler/java/lite/map_field.cc | 2 +- .../compiler/java/lite/message_field.cc | 6 +- .../compiler/java/lite/primitive_field.cc | 38 ++++++++----- .../compiler/java/lite/string_field.cc | 6 +- 16 files changed, 173 insertions(+), 57 deletions(-) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt index c9fee8a78f..7c319bfaac 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt @@ -225,7 +225,7 @@ class Proto2Test { TestAllTypesKt.repeatedGroup { a = 1 }, TestAllTypesKt.repeatedGroup { a = 2 }, TestAllTypesKt.repeatedGroup { a = 3 }, - TestAllTypesKt.repeatedGroup { a = 4 } + TestAllTypesKt.repeatedGroup { a = 4 }, ) ) repeatedGroup[0] = TestAllTypesKt.repeatedGroup { a = 5 } @@ -235,7 +235,7 @@ class Proto2Test { TestAllTypesKt.repeatedGroup { a = 5 }, TestAllTypesKt.repeatedGroup { a = 2 }, TestAllTypesKt.repeatedGroup { a = 3 }, - TestAllTypesKt.repeatedGroup { a = 4 } + TestAllTypesKt.repeatedGroup { a = 4 }, ) ) @@ -249,7 +249,7 @@ class Proto2Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) repeatedNestedMessage[0] = nestedMessage { bb = 5 } @@ -259,7 +259,7 @@ class Proto2Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) @@ -548,7 +548,7 @@ class Proto2Test { repeatedGroupExtension { a = 1 }, repeatedGroupExtension { a = 2 }, repeatedGroupExtension { a = 3 }, - repeatedGroupExtension { a = 4 } + repeatedGroupExtension { a = 4 }, ) ) this[UnittestProto.repeatedGroupExtension][0] = repeatedGroupExtension { a = 5 } @@ -558,7 +558,7 @@ class Proto2Test { repeatedGroupExtension { a = 5 }, repeatedGroupExtension { a = 2 }, repeatedGroupExtension { a = 3 }, - repeatedGroupExtension { a = 4 } + repeatedGroupExtension { a = 4 }, ) ) @@ -575,7 +575,7 @@ class Proto2Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) this[UnittestProto.repeatedNestedMessageExtension][0] = nestedMessage { bb = 5 } @@ -585,7 +585,7 @@ class Proto2Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) @@ -757,7 +757,7 @@ class Proto2Test { 1 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO, 2 to Proto2MapEnum.PROTO2_MAP_ENUM_BAR, 3 to Proto2MapEnum.PROTO2_MAP_ENUM_BAZ, - 4 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO + 4 to Proto2MapEnum.PROTO2_MAP_ENUM_FOO, ) ) } @@ -844,6 +844,11 @@ class Proto2Test { cachedSize_ = "foo" serializedSize_ = true by = "foo" + dEPRECATEDFoo = "foo" + DEPRECATEDBar = "foo" + iD = "foo" + aBNotification = "foo" + notDEPRECATEDFoo = "foo" } ) .isEqualTo( @@ -869,6 +874,11 @@ class Proto2Test { .setCachedSize_("foo") .setSerializedSize_(true) .setBy("foo") + .setDEPRECATEDFoo("foo") + .setDEPRECATEDBar("foo") + .setID("foo") + .setABNotification("foo") + .setNotDEPRECATEDFoo("foo") .build() ) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt index 0b270e6e75..72ab8e1f80 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt @@ -81,7 +81,7 @@ class Proto3Test { nestedMessage { bb = 1 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) repeatedNestedMessage[0] = nestedMessage { bb = 5 } @@ -91,7 +91,7 @@ class Proto3Test { nestedMessage { bb = 5 }, nestedMessage { bb = 2 }, nestedMessage { bb = 3 }, - nestedMessage { bb = 4 } + nestedMessage { bb = 4 }, ) ) @@ -200,6 +200,11 @@ class Proto3Test { pairs["foo"] = 1 LeadingUnderscore = "foo" option = 1 + dEPRECATEDFoo = "foo" + iD = "foo" + aBNotification = "foo" + DEPRECATEDBar = "foo" + notDEPRECATEDFoo = "foo" } ) .isEqualTo( @@ -237,6 +242,11 @@ class Proto3Test { .putPairs("foo", 1) .setLeadingUnderscore("foo") .setOption(1) + .setDEPRECATEDFoo("foo") + .setID("foo") + .setABNotification("foo") + .setDEPRECATEDBar("foo") + .setNotDEPRECATEDFoo("foo") .build() ) diff --git a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto index ba0827422b..f771bf97bf 100644 --- a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto +++ b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto @@ -43,6 +43,15 @@ message EvilNamesProto2 { optional string cached_size = 23; optional bool serialized_size = 24; optional string by = 25; + + optional string DEPRECATED_foo = 26; + optional group DEPRECATED_NavigationImageRequested = 27 { + optional int32 DEPRECATED_FooBar = 28; + } + optional string __DEPRECATED_Bar = 29; + optional string ID = 30; + optional string a_b_notification = 31; + optional string not_DEPRECATED_foo = 32; } message List {} diff --git a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto index 2c9b5a2d68..238bebc06b 100644 --- a/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto +++ b/java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto @@ -56,6 +56,12 @@ message EvilNamesProto3 { oneof _leading_underscore_oneof { int32 option = 34; } + + optional string DEPRECATED_foo = 35; + optional string ID = 36; + optional string a_b_notification = 37; + optional string __DEPRECATED_Bar = 38; + optional string not_DEPRECATED_foo = 39; } message HardKeywordsAllTypesProto3 { diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 39f862f6cc..915b370309 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -144,6 +144,7 @@ cc_library( "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:string_view", ], ) diff --git a/src/google/protobuf/compiler/java/field_common.cc b/src/google/protobuf/compiler/java/field_common.cc index 4acafabe36..cf5cdcbf0a 100644 --- a/src/google/protobuf/compiler/java/field_common.cc +++ b/src/google/protobuf/compiler/java/field_common.cc @@ -1,8 +1,11 @@ #include "google/protobuf/compiler/java/field_common.h" +#include #include +#include "absl/strings/str_cat.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/names.h" #include "google/protobuf/descriptor.h" namespace google { @@ -10,6 +13,8 @@ namespace protobuf { namespace compiler { namespace java { +std::string GetKotlinPropertyName(std::string capitalized_name); + void SetCommonFieldVariables( const FieldDescriptor* descriptor, const FieldGeneratorInfo* info, absl::flat_hash_map* variables) { @@ -30,6 +35,11 @@ void SetCommonFieldVariables( (*variables)["kt_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->name, "_") : info->name; + auto kt_property_name = GetKotlinPropertyName(info->capitalized_name); + (*variables)["kt_property_name"] = kt_property_name; + (*variables)["kt_safe_name"] = IsForbiddenKotlin(kt_property_name) + ? absl::StrCat("`", kt_property_name, "`") + : kt_property_name; (*variables)["kt_capitalized_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->capitalized_name, "_") : info->capitalized_name; @@ -51,6 +61,51 @@ void SetCommonFieldVariables( } } +// Locale-independent ASCII upper and lower case munging. +static bool IsUpper(char c) { + return static_cast(c - 'A') <= 'Z' - 'A'; +} + +static char ToLower(char c) { return IsUpper(c) ? c - 'A' + 'a' : c; } + +// Returns the name by which the generated Java getters and setters should be +// referenced from Kotlin as properties. In the simplest case, the original name +// is something like `foo_bar`, which gets translated into `getFooBar()` etc, +// and that in turn can be referenced from Kotlin as `fooBar`. +// +// The algorithm for translating proto names into Java getters and setters is +// straightforward. The first letter of each underscore-separated word gets +// uppercased and the underscores are deleted. There are no other changes, so in +// particular if the proto name has a string of capitals then those remain +// as-is. +// +// The algorithm that the Kotlin compiler uses to derive the property name is +// slightly more complicated. If the first character after `get` (etc) is a +// capital and the second isn't, then the property name is just that string with +// its first letter lowercased. So `getFoo` becomes `foo` and `getX` becomes +// `x`. But if there is more than one capital, then all but the last get +// lowercased. So `getHTMLPage` becomes `htmlPage`. If there are only capitals +// then they all get lowercased, so `getID` becomes `id`. +// TODO: move this to a Kotlin-specific location +std::string GetKotlinPropertyName(std::string capitalized_name) { + // Find the first non-capital. If it is the second character, then we just + // need to lowercase the first one. Otherwise we need to lowercase everything + // up to but not including the last capital, except that if everything is + // capitals then everything must be lowercased. + std::string kt_property_name = capitalized_name; + size_t first_non_capital; + for (first_non_capital = 0; first_non_capital < capitalized_name.length() && + IsUpper(capitalized_name[first_non_capital]); + first_non_capital++) { + } + size_t stop = first_non_capital; + if (stop > 1 && stop < capitalized_name.length()) stop--; + for (size_t i = 0; i < stop; i++) { + kt_property_name[i] = ToLower(kt_property_name[i]); + } + return kt_property_name; +} + void SetCommonOneofVariables( const FieldDescriptor* descriptor, const OneofGeneratorInfo* info, absl::flat_hash_map* variables) { diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/immutable/enum_field.cc index 2cb734f498..4a64460fb5 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/immutable/enum_field.cc @@ -273,10 +273,10 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); if (SupportUnknownEnumValue(descriptor_)) { @@ -284,10 +284,10 @@ void ImmutableEnumFieldGenerator::GenerateKotlinDslMembers( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" + " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" " }\n"); } @@ -1100,7 +1100,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/map_field.cc b/src/google/protobuf/compiler/java/immutable/map_field.cc index 02e85874c0..13e6319b79 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.cc +++ b/src/google/protobuf/compiler/java/immutable/map_field.cc @@ -981,7 +981,7 @@ void ImmutableMapFieldGenerator::GenerateKotlinDslMembers( " @kotlin.jvm.JvmSynthetic\n" " @JvmName(\"get$kt_capitalized_name$Map\")\n" " get() = com.google.protobuf.kotlin.DslMap(\n" - " $kt_dsl_builder$.${$get$capitalized_name$Map$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), diff --git a/src/google/protobuf/compiler/java/immutable/message_field.cc b/src/google/protobuf/compiler/java/immutable/message_field.cc index 1b76fc2f63..31eb0ea858 100644 --- a/src/google/protobuf/compiler/java/immutable/message_field.cc +++ b/src/google/protobuf/compiler/java/immutable/message_field.cc @@ -378,10 +378,10 @@ void ImmutableMessageFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -1376,7 +1376,7 @@ void RepeatedImmutableMessageFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/primitive_field.cc b/src/google/protobuf/compiler/java/immutable/primitive_field.cc index eb31a7b756..3dc402e5c2 100644 --- a/src/google/protobuf/compiler/java/immutable/primitive_field.cc +++ b/src/google/protobuf/compiler/java/immutable/primitive_field.cc @@ -291,14 +291,27 @@ void ImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" - " }\n"); + if (descriptor_->name() == "is_initialized") { + printer->Print( + variables_, + "// TODO: remove this hack; we should access properties\n" + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$get$kt_capitalized_name$$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" + " }\n"); + } else { + printer->Print(variables_, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); + } WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), @@ -848,7 +861,7 @@ void RepeatedImmutablePrimitiveFieldGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/immutable/string_field.cc b/src/google/protobuf/compiler/java/immutable/string_field.cc index c9ac5f4e7c..df12b05f07 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.cc +++ b/src/google/protobuf/compiler/java/immutable/string_field.cc @@ -372,10 +372,10 @@ void ImmutableStringFieldGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: kotlin.String\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -992,7 +992,7 @@ void RepeatedImmutableStringFieldGenerator::GenerateKotlinDslMembers( "\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); // List.add(String) diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index 3e86e1942f..c26797739e 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -308,10 +308,10 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); if (SupportUnknownEnumValue(descriptor_)) { @@ -319,10 +319,10 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( variables_, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$Value$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$Value$}$(value)\n" + " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" " }\n"); } @@ -913,7 +913,7 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 74fc93f347..491523df91 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -857,7 +857,7 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( " @kotlin.jvm.JvmSynthetic\n" " @JvmName(\"get$kt_capitalized_name$Map\")\n" " get() = com.google.protobuf.kotlin.DslMap(\n" - " $kt_dsl_builder$.${$get$capitalized_name$Map$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index 7c5cc3cf77..c46407bf51 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -283,10 +283,10 @@ void ImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: $kt_type$\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -818,7 +818,7 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index 69924c9c83..c9b3a2947a 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -217,10 +217,9 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateMembers( "private static final $field_type$ $bytes_default$ = $default$;\n"); } if (!context_->options().opensource_runtime) { - printer->Print( - variables_, - "@com.google.protobuf.ProtoField(\n" - " isRequired=$required$)\n"); + printer->Print(variables_, + "@com.google.protobuf.ProtoField(\n" + " isRequired=$required$)\n"); if (HasHasbit(descriptor_)) { printer->Print(variables_, "@com.google.protobuf.ProtoPresenceCheckedField(\n" @@ -329,14 +328,27 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void ImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_, context_->options()); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" - " }\n"); + if (descriptor_->name() == "is_initialized") { + printer->Print( + variables_, + "// TODO: b/336400327 - remove this hack; we should access properties\n" + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.get${$$kt_capitalized_name$$}$()\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" + " }\n"); + } else { + printer->Print(variables_, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); + } WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), @@ -718,7 +730,7 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " @kotlin.jvm.JvmSynthetic\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index 833f415585..a08bae50b6 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -336,10 +336,10 @@ void ImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( printer->Print(variables_, "$kt_deprecation$public var $kt_name$: kotlin.String\n" " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$get$capitalized_name$$}$()\n" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" " set(value) {\n" - " $kt_dsl_builder$.${$set$capitalized_name$$}$(value)\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, @@ -825,7 +825,7 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( "@kotlin.OptIn" "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$get$capitalized_name$List$}$()\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" " )\n"); // List.add(String) From b4f4f8ef650c5934f6ed3bdaf996973c953aeef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Wed, 1 May 2024 12:51:22 -0700 Subject: [PATCH 083/179] Automated rollback of commit 7a09c4569bec008504bca3eb0201f0e75863a1fc. PiperOrigin-RevId: 629808114 --- upb/message/accessors.h | 8 +++++--- upb/message/accessors_test.cc | 23 ----------------------- upb/message/internal/accessors.h | 21 --------------------- 3 files changed, 5 insertions(+), 47 deletions(-) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index 723d10d871..f812e2da47 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -68,9 +68,11 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, - const upb_MiniTable* m, - const upb_MiniTableField* f); +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} // NOTE: The default_val is only used for fields that support presence. // For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a diff --git a/upb/message/accessors_test.cc b/upb/message/accessors_test.cc index 524e5fb655..5b0dd3638c 100644 --- a/upb/message/accessors_test.cc +++ b/upb/message/accessors_test.cc @@ -477,28 +477,5 @@ TEST(GeneratedCode, EnumClosedCheck) { EXPECT_TRUE(upb_MiniTableField_IsClosedEnum(closedEnumField)); upb_Arena_Free(arena); } -TEST(GeneratedCode, OneofClear) { - upb_Arena* arena = upb_Arena_New(); - - protobuf_test_messages_proto2_TestAllTypesProto2* msg = - protobuf_test_messages_proto2_TestAllTypesProto2_new(arena); - - const upb_MiniTable* table = - &protobuf_0test_0messages__proto2__TestAllTypesProto2_msg_init; - - // oneof_uint32 - const upb_MiniTableField* oneofField = - upb_MiniTable_FindFieldByNumber(table, 111); - EXPECT_TRUE(upb_MiniTableField_IsInOneof(oneofField)); - protobuf_test_messages_proto2_TestAllTypesProto2_set_oneof_uint32(msg, 522); - EXPECT_TRUE( - protobuf_test_messages_proto2_TestAllTypesProto2_has_oneof_uint32(msg)); - - upb_Message_ClearOneof((upb_Message*)msg, table, oneofField); - EXPECT_FALSE( - protobuf_test_messages_proto2_TestAllTypesProto2_has_oneof_uint32(msg)); - - upb_Arena_Free(arena); -} } // namespace diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index 1c5a397487..bfae12e601 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -335,27 +335,6 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const struct upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} - -UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, - const upb_MiniTable* m, - const upb_MiniTableField* f) { - UPB_ASSERT(!upb_Message_IsFrozen(msg)); - uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); - if (field_number == 0) { - // No field in the oneof is set. - return; - } - - const upb_MiniTableField* field = - upb_MiniTable_FindFieldByNumber(m, field_number); - upb_Message_ClearBaseField(msg, field); -} - UPB_INLINE void _upb_Message_AssertMapIsUntagged( const struct upb_Message* msg, const upb_MiniTableField* field) { UPB_UNUSED(msg); From 05afdc2f0afbc7f1412ef3384f898f45e419d21a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 1 May 2024 20:04:44 +0000 Subject: [PATCH 084/179] Auto-generate files after cl/629808114 --- php/ext/google/protobuf/php-upb.h | 29 +++++---------------------- ruby/ext/google/protobuf_c/ruby-upb.h | 29 +++++---------------------- 2 files changed, 10 insertions(+), 48 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index c426679dc0..e8f88a62b3 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -2914,27 +2914,6 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const struct upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} - -UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, - const upb_MiniTable* m, - const upb_MiniTableField* f) { - UPB_ASSERT(!upb_Message_IsFrozen(msg)); - uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); - if (field_number == 0) { - // No field in the oneof is set. - return; - } - - const upb_MiniTableField* field = - upb_MiniTable_FindFieldByNumber(m, field_number); - upb_Message_ClearBaseField(msg, field); -} - UPB_INLINE void _upb_Message_AssertMapIsUntagged( const struct upb_Message* msg, const upb_MiniTableField* field) { UPB_UNUSED(msg); @@ -3246,9 +3225,11 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, - const upb_MiniTable* m, - const upb_MiniTableField* f); +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} // NOTE: The default_val is only used for fields that support presence. // For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 028e076eb4..eebfde85aa 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -2916,27 +2916,6 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const struct upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} - -UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, - const upb_MiniTable* m, - const upb_MiniTableField* f) { - UPB_ASSERT(!upb_Message_IsFrozen(msg)); - uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); - if (field_number == 0) { - // No field in the oneof is set. - return; - } - - const upb_MiniTableField* field = - upb_MiniTable_FindFieldByNumber(m, field_number); - upb_Message_ClearBaseField(msg, field); -} - UPB_INLINE void _upb_Message_AssertMapIsUntagged( const struct upb_Message* msg, const upb_MiniTableField* field) { UPB_UNUSED(msg); @@ -3248,9 +3227,11 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, - const upb_MiniTable* m, - const upb_MiniTableField* f); +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} // NOTE: The default_val is only used for fields that support presence. // For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a From c9eeb1c8a73f64fbdd7189c2aeaff6272f423e5d Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 2 May 2024 09:38:49 -0700 Subject: [PATCH 085/179] Internal changes. PiperOrigin-RevId: 630090538 --- upb/reflection/stage0/google/protobuf/descriptor.upb.h | 3 ++- upb_generator/common.cc | 6 +++++- upb_generator/stage0/google/protobuf/compiler/plugin.upb.h | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/upb/reflection/stage0/google/protobuf/descriptor.upb.h b/upb/reflection/stage0/google/protobuf/descriptor.upb.h index 33287809bf..9bc4bbbb14 100644 --- a/upb/reflection/stage0/google/protobuf/descriptor.upb.h +++ b/upb/reflection/stage0/google/protobuf/descriptor.upb.h @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ diff --git a/upb_generator/common.cc b/upb_generator/common.cc index 3bda1a3cab..f7f408128e 100644 --- a/upb_generator/common.cc +++ b/upb_generator/common.cc @@ -51,7 +51,11 @@ void EmitFileWarning(absl::string_view name, Output& output) { " * $0\n" " *\n" " * Do not edit -- your changes will be discarded when the file is\n" - " * regenerated. */\n\n", + " * regenerated.\n" + " * NO CHECKED-IN " + // Intentional line break. + "PROTOBUF GENCODE */\n" + "\n", name); } diff --git a/upb_generator/stage0/google/protobuf/compiler/plugin.upb.h b/upb_generator/stage0/google/protobuf/compiler/plugin.upb.h index 1ccffa7b7e..50c2d24823 100644 --- a/upb_generator/stage0/google/protobuf/compiler/plugin.upb.h +++ b/upb_generator/stage0/google/protobuf/compiler/plugin.upb.h @@ -3,7 +3,8 @@ * google/protobuf/compiler/plugin.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_COMPILER_PLUGIN_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_COMPILER_PLUGIN_PROTO_UPB_H_ From d3b2fc52a4aec2dc9711c8235cbcdb71d4c10295 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 2 May 2024 17:02:37 +0000 Subject: [PATCH 086/179] Auto-generate files after cl/630090538 --- php/ext/google/protobuf/php-upb.c | 6 ++++-- php/ext/google/protobuf/php-upb.h | 9 ++++++--- ruby/ext/google/protobuf_c/ruby-upb.c | 3 ++- ruby/ext/google/protobuf_c/ruby-upb.h | 6 ++++-- upb/cmake/google/protobuf/descriptor.upb.h | 3 ++- upb/cmake/google/protobuf/descriptor.upb_minitable.c | 3 ++- upb/cmake/google/protobuf/descriptor.upb_minitable.h | 3 ++- upb/reflection/cmake/google/protobuf/descriptor.upb.h | 3 ++- .../cmake/google/protobuf/descriptor.upb_minitable.c | 3 ++- .../cmake/google/protobuf/descriptor.upb_minitable.h | 3 ++- .../cmake/google/protobuf/compiler/plugin.upb.h | 3 ++- .../google/protobuf/compiler/plugin.upb_minitable.c | 3 ++- .../google/protobuf/compiler/plugin.upb_minitable.h | 3 ++- 13 files changed, 34 insertions(+), 17 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 508b7f46dc..2d78a633f4 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -404,7 +404,8 @@ void upb_Status_VAppendErrorFormat(upb_Status* status, const char* fmt, * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #include @@ -1762,7 +1763,8 @@ const upb_MiniTableFile google_protobuf_descriptor_proto_upb_file_layout = { * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ static const char descriptor[12155] = {'\n', ' ', 'g', 'o', 'o', 'g', 'l', 'e', '/', 'p', 'r', 'o', 't', 'o', 'b', 'u', 'f', '/', 'd', 'e', 's', 'c', 'r', 'i', 'p', diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index e8f88a62b3..c9fc8ec49e 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -4405,7 +4405,8 @@ TAGBYTES(r) * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ @@ -4509,7 +4510,8 @@ extern const upb_MiniTableFile google_protobuf_descriptor_proto_upb_file_layout; * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ @@ -11826,7 +11828,8 @@ const UPB_DESC(FeatureSet) * * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPBDEFS_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPBDEFS_H_ diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 9578e8e976..1483124033 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -404,7 +404,8 @@ void upb_Status_VAppendErrorFormat(upb_Status* status, const char* fmt, * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #include diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index eebfde85aa..277b52be07 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -4407,7 +4407,8 @@ TAGBYTES(r) * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ @@ -4911,7 +4912,8 @@ UPB_FORCEINLINE bool upb_EpsCopyInputStream_TryParseDelimitedFast( * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ diff --git a/upb/cmake/google/protobuf/descriptor.upb.h b/upb/cmake/google/protobuf/descriptor.upb.h index fdedcbbab7..bdb3b5f6e0 100644 --- a/upb/cmake/google/protobuf/descriptor.upb.h +++ b/upb/cmake/google/protobuf/descriptor.upb.h @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ diff --git a/upb/cmake/google/protobuf/descriptor.upb_minitable.c b/upb/cmake/google/protobuf/descriptor.upb_minitable.c index eba76e82eb..80d12682e5 100644 --- a/upb/cmake/google/protobuf/descriptor.upb_minitable.c +++ b/upb/cmake/google/protobuf/descriptor.upb_minitable.c @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #include #include "upb/generated_code_support.h" diff --git a/upb/cmake/google/protobuf/descriptor.upb_minitable.h b/upb/cmake/google/protobuf/descriptor.upb_minitable.h index f48a64df65..4fc7176c59 100644 --- a/upb/cmake/google/protobuf/descriptor.upb_minitable.h +++ b/upb/cmake/google/protobuf/descriptor.upb_minitable.h @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ diff --git a/upb/reflection/cmake/google/protobuf/descriptor.upb.h b/upb/reflection/cmake/google/protobuf/descriptor.upb.h index fdedcbbab7..bdb3b5f6e0 100644 --- a/upb/reflection/cmake/google/protobuf/descriptor.upb.h +++ b/upb/reflection/cmake/google/protobuf/descriptor.upb.h @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_H_ diff --git a/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c b/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c index eba76e82eb..80d12682e5 100644 --- a/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c +++ b/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #include #include "upb/generated_code_support.h" diff --git a/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.h b/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.h index f48a64df65..4fc7176c59 100644 --- a/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.h +++ b/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.h @@ -3,7 +3,8 @@ * google/protobuf/descriptor.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ #define GOOGLE_PROTOBUF_DESCRIPTOR_PROTO_UPB_MINITABLE_H_ diff --git a/upb_generator/cmake/google/protobuf/compiler/plugin.upb.h b/upb_generator/cmake/google/protobuf/compiler/plugin.upb.h index 364bc2e2e5..ee7ad4f82d 100644 --- a/upb_generator/cmake/google/protobuf/compiler/plugin.upb.h +++ b/upb_generator/cmake/google/protobuf/compiler/plugin.upb.h @@ -3,7 +3,8 @@ * google/protobuf/compiler/plugin.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_COMPILER_PLUGIN_PROTO_UPB_H_ #define GOOGLE_PROTOBUF_COMPILER_PLUGIN_PROTO_UPB_H_ diff --git a/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c b/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c index 3f24240c57..9fc1582433 100644 --- a/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c +++ b/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c @@ -3,7 +3,8 @@ * google/protobuf/compiler/plugin.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #include #include "upb/generated_code_support.h" diff --git a/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.h b/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.h index f66ce47047..b9d777cb46 100644 --- a/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.h +++ b/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.h @@ -3,7 +3,8 @@ * google/protobuf/compiler/plugin.proto * * Do not edit -- your changes will be discarded when the file is - * regenerated. */ + * regenerated. + * NO CHECKED-IN PROTOBUF GENCODE */ #ifndef GOOGLE_PROTOBUF_COMPILER_PLUGIN_PROTO_UPB_MINITABLE_H_ #define GOOGLE_PROTOBUF_COMPILER_PLUGIN_PROTO_UPB_MINITABLE_H_ From 2257232676e60f1d06f8c9545c88d1a73a3a8f54 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 May 2024 10:09:15 -0700 Subject: [PATCH 087/179] Split bootstrapped java_features.proto to keep it from leaking out. This can cause ODR violations in downstream users who link against both the bootstrapped proto and transitive C++ gencode of java_features.proto. Once protoc is split up, we can turn the bootstrapped proto into a real cc_proto_library target and avoid this problem altogether. PiperOrigin-RevId: 630099889 --- src/google/protobuf/compiler/java/BUILD.bazel | 21 +++- src/google/protobuf/compiler/java/helpers.cc | 80 ------------ src/google/protobuf/compiler/java/helpers.h | 36 ------ .../compiler/java/immutable/BUILD.bazel | 1 + .../compiler/java/immutable/enum_field.cc | 1 + .../compiler/java/immutable/map_field.cc | 1 + .../compiler/java/immutable/string_field.cc | 1 + .../compiler/java/internal_helpers.cc | 114 ++++++++++++++++++ .../protobuf/compiler/java/internal_helpers.h | 69 +++++++++++ .../protobuf/compiler/java/lite/BUILD.bazel | 2 + .../protobuf/compiler/java/lite/enum_field.cc | 2 +- .../protobuf/compiler/java/lite/map_field.cc | 1 + .../compiler/java/lite/message_field.cc | 1 + .../compiler/java/lite/primitive_field.cc | 1 + .../compiler/java/lite/string_field.cc | 1 + 15 files changed, 213 insertions(+), 119 deletions(-) create mode 100644 src/google/protobuf/compiler/java/internal_helpers.cc create mode 100644 src/google/protobuf/compiler/java/internal_helpers.h diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 915b370309..ad25fecb27 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -35,7 +35,6 @@ cc_library( hdrs = [ "context.h", "doc_comment.h", - "generator.h", "helpers.h", "name_resolver.h", "names.h", @@ -48,7 +47,6 @@ cc_library( "//src/google/protobuf/compiler/java:__subpackages__", ], deps = [ - ":java_features_bootstrap", "//src/google/protobuf", "//src/google/protobuf:port", "//src/google/protobuf/compiler:code_generator", @@ -64,6 +62,25 @@ cc_library( ], ) +cc_library( + name = "internal_helpers", + srcs = ["internal_helpers.cc"], + hdrs = [ + "generator.h", + "internal_helpers.h", + ], + strip_include_prefix = "/src", + visibility = ["//src/google/protobuf/compiler/java:__subpackages__"], + deps = [ + ":helpers", + ":java_features_bootstrap", + "//src/google/protobuf", + "//src/google/protobuf:port", + "//src/google/protobuf/compiler:code_generator", + "@com_google_absl//absl/log:absl_log", + ], +) + cc_library( name = "java_features_bootstrap", srcs = ["java_features.pb.cc"], diff --git a/src/google/protobuf/compiler/java/helpers.cc b/src/google/protobuf/compiler/java/helpers.cc index 826d9f8344..8deb2dcec3 100644 --- a/src/google/protobuf/compiler/java/helpers.cc +++ b/src/google/protobuf/compiler/java/helpers.cc @@ -878,86 +878,6 @@ void WriteUInt32ToUtf16CharSequence(uint32_t number, output->push_back(static_cast(number)); } -int GetExperimentalJavaFieldTypeForSingular(const FieldDescriptor* field) { - // j/c/g/protobuf/FieldType.java lists field types in a slightly different - // order from FieldDescriptor::Type so we can't do a simple cast. - // - // TODO: Make j/c/g/protobuf/FieldType.java follow the same order. - int result = field->type(); - if (result == FieldDescriptor::TYPE_GROUP) { - return 17; - } else if (result < FieldDescriptor::TYPE_GROUP) { - return result - 1; - } else { - return result - 2; - } -} - -int GetExperimentalJavaFieldTypeForRepeated(const FieldDescriptor* field) { - if (field->type() == FieldDescriptor::TYPE_GROUP) { - return 49; - } else { - return GetExperimentalJavaFieldTypeForSingular(field) + 18; - } -} - -int GetExperimentalJavaFieldTypeForPacked(const FieldDescriptor* field) { - int result = field->type(); - if (result < FieldDescriptor::TYPE_STRING) { - return result + 34; - } else if (result > FieldDescriptor::TYPE_BYTES) { - return result + 30; - } else { - ABSL_LOG(FATAL) << field->full_name() << " can't be packed."; - return 0; - } -} - -int GetExperimentalJavaFieldType(const FieldDescriptor* field) { - static const int kMapFieldType = 50; - static const int kOneofFieldTypeOffset = 51; - - static const int kRequiredBit = 0x100; - static const int kUtf8CheckBit = 0x200; - static const int kCheckInitialized = 0x400; - static const int kLegacyEnumIsClosedBit = 0x800; - static const int kHasHasBit = 0x1000; - int extra_bits = field->is_required() ? kRequiredBit : 0; - if (field->type() == FieldDescriptor::TYPE_STRING && CheckUtf8(field)) { - extra_bits |= kUtf8CheckBit; - } - if (field->is_required() || (GetJavaType(field) == JAVATYPE_MESSAGE && - HasRequiredFields(field->message_type()))) { - extra_bits |= kCheckInitialized; - } - if (HasHasbit(field)) { - extra_bits |= kHasHasBit; - } - if (GetJavaType(field) == JAVATYPE_ENUM && !SupportUnknownEnumValue(field)) { - extra_bits |= kLegacyEnumIsClosedBit; - } - - if (field->is_map()) { - if (!SupportUnknownEnumValue(MapValueField(field))) { - const FieldDescriptor* value = field->message_type()->map_value(); - if (GetJavaType(value) == JAVATYPE_ENUM) { - extra_bits |= kLegacyEnumIsClosedBit; - } - } - return kMapFieldType | extra_bits; - } else if (field->is_packed()) { - return GetExperimentalJavaFieldTypeForPacked(field) | extra_bits; - } else if (field->is_repeated()) { - return GetExperimentalJavaFieldTypeForRepeated(field) | extra_bits; - } else if (IsRealOneof(field)) { - return (GetExperimentalJavaFieldTypeForSingular(field) + - kOneofFieldTypeOffset) | - extra_bits; - } else { - return GetExperimentalJavaFieldTypeForSingular(field) | extra_bits; - } -} - // Escape a UTF-16 character to be embedded in a Java string. void EscapeUtf16ToString(uint16_t code, std::string* output) { if (code == '\t') { diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index ed27eae1d5..52b4017cf5 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -16,8 +16,6 @@ #include #include "absl/strings/string_view.h" -#include "google/protobuf/compiler/java/generator.h" -#include "google/protobuf/compiler/java/java_features.pb.h" #include "google/protobuf/compiler/java/names.h" #include "google/protobuf/compiler/java/options.h" #include "google/protobuf/descriptor.h" @@ -344,18 +342,6 @@ inline bool HasHasbit(const FieldDescriptor* descriptor) { return internal::cpp::HasHasbit(descriptor); } -// Whether unknown enum values are kept (i.e., not stored in UnknownFieldSet -// but in the message and can be queried using additional getters that return -// ints. -inline bool SupportUnknownEnumValue(const FieldDescriptor* field) { - if (JavaGenerator::GetResolvedSourceFeatures(*field) - .GetExtension(pb::java) - .legacy_closed_enum()) { - return false; - } - return field->enum_type() != nullptr && !field->enum_type()->is_closed(); -} - // Check whether a message has repeated fields. bool HasRepeatedFields(const Descriptor* descriptor); @@ -375,18 +361,6 @@ inline bool IsWrappersProtoFile(const FileDescriptor* descriptor) { return descriptor->name() == "google/protobuf/wrappers.proto"; } -inline bool CheckUtf8(const FieldDescriptor* descriptor) { - if (JavaGenerator::GetResolvedSourceFeatures(*descriptor) - .GetExtension(pb::java) - .utf8_validation() == pb::JavaFeatures::VERIFY) { - return true; - } - return JavaGenerator::GetResolvedSourceFeatures(*descriptor) - .utf8_validation() == FeatureSet::VERIFY || - // For legacy syntax. This is not allowed under Editions. - descriptor->file()->options().java_string_check_utf8(); -} - void WriteUInt32ToUtf16CharSequence(uint32_t number, std::vector* output); @@ -398,16 +372,6 @@ inline void WriteIntToUtf16CharSequence(int value, // Escape a UTF-16 character so it can be embedded in a Java string literal. void EscapeUtf16ToString(uint16_t code, std::string* output); -// Only the lowest two bytes of the return value are used. The lowest byte -// is the integer value of a j/c/g/protobuf/FieldType enum. For the other -// byte: -// bit 0: whether the field is required. -// bit 1: whether the field requires UTF-8 validation. -// bit 2: whether the field needs isInitialized check. -// bit 3: whether the field is a map field with proto2 enum value. -// bits 4-7: unused -int GetExperimentalJavaFieldType(const FieldDescriptor* field); - // To get the total number of entries need to be built for experimental runtime // and the first field number that are not in the table part std::pair GetTableDrivenNumberOfEntriesAndLookUpStartFieldNumber( diff --git a/src/google/protobuf/compiler/java/immutable/BUILD.bazel b/src/google/protobuf/compiler/java/immutable/BUILD.bazel index a65a03acbd..baa679ef42 100644 --- a/src/google/protobuf/compiler/java/immutable/BUILD.bazel +++ b/src/google/protobuf/compiler/java/immutable/BUILD.bazel @@ -35,6 +35,7 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/io:printer", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/immutable/enum_field.cc index 4a64460fb5..4d0f65539d 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/immutable/enum_field.cc @@ -22,6 +22,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/immutable/map_field.cc b/src/google/protobuf/compiler/java/immutable/map_field.cc index 13e6319b79..1281862958 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.cc +++ b/src/google/protobuf/compiler/java/immutable/map_field.cc @@ -15,6 +15,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/immutable/string_field.cc b/src/google/protobuf/compiler/java/immutable/string_field.cc index df12b05f07..1321f51675 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.cc +++ b/src/google/protobuf/compiler/java/immutable/string_field.cc @@ -22,6 +22,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/internal_helpers.cc b/src/google/protobuf/compiler/java/internal_helpers.cc new file mode 100644 index 0000000000..06dae866c4 --- /dev/null +++ b/src/google/protobuf/compiler/java/internal_helpers.cc @@ -0,0 +1,114 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#include "google/protobuf/compiler/java/internal_helpers.h" + +#include "absl/log/absl_log.h" +#include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/name_resolver.h" +#include "google/protobuf/descriptor.pb.h" + +// Must be last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { +namespace { + +int GetExperimentalJavaFieldTypeForSingular(const FieldDescriptor* field) { + // j/c/g/protobuf/FieldType.java lists field types in a slightly different + // order from FieldDescriptor::Type so we can't do a simple cast. + // + // TODO: Make j/c/g/protobuf/FieldType.java follow the same order. + int result = field->type(); + if (result == FieldDescriptor::TYPE_GROUP) { + return 17; + } else if (result < FieldDescriptor::TYPE_GROUP) { + return result - 1; + } else { + return result - 2; + } +} + +int GetExperimentalJavaFieldTypeForRepeated(const FieldDescriptor* field) { + if (field->type() == FieldDescriptor::TYPE_GROUP) { + return 49; + } else { + return GetExperimentalJavaFieldTypeForSingular(field) + 18; + } +} + +int GetExperimentalJavaFieldTypeForPacked(const FieldDescriptor* field) { + int result = field->type(); + if (result < FieldDescriptor::TYPE_STRING) { + return result + 34; + } else if (result > FieldDescriptor::TYPE_BYTES) { + return result + 30; + } else { + ABSL_LOG(FATAL) << field->full_name() << " can't be packed."; + return 0; + } +} +} // namespace + +int GetExperimentalJavaFieldType(const FieldDescriptor* field) { + static const int kMapFieldType = 50; + static const int kOneofFieldTypeOffset = 51; + + static const int kRequiredBit = 0x100; + static const int kUtf8CheckBit = 0x200; + static const int kCheckInitialized = 0x400; + static const int kLegacyEnumIsClosedBit = 0x800; + static const int kHasHasBit = 0x1000; + int extra_bits = field->is_required() ? kRequiredBit : 0; + if (field->type() == FieldDescriptor::TYPE_STRING && CheckUtf8(field)) { + extra_bits |= kUtf8CheckBit; + } + if (field->is_required() || (GetJavaType(field) == JAVATYPE_MESSAGE && + HasRequiredFields(field->message_type()))) { + extra_bits |= kCheckInitialized; + } + if (HasHasbit(field)) { + extra_bits |= kHasHasBit; + } + if (GetJavaType(field) == JAVATYPE_ENUM && !SupportUnknownEnumValue(field)) { + extra_bits |= kLegacyEnumIsClosedBit; + } + + if (field->is_map()) { + if (!SupportUnknownEnumValue(MapValueField(field))) { + const FieldDescriptor* value = field->message_type()->map_value(); + if (GetJavaType(value) == JAVATYPE_ENUM) { + extra_bits |= kLegacyEnumIsClosedBit; + } + } + return kMapFieldType | extra_bits; + } else if (field->is_packed()) { + return GetExperimentalJavaFieldTypeForPacked(field) | extra_bits; + } else if (field->is_repeated()) { + return GetExperimentalJavaFieldTypeForRepeated(field) | extra_bits; + } else if (IsRealOneof(field)) { + return (GetExperimentalJavaFieldTypeForSingular(field) + + kOneofFieldTypeOffset) | + extra_bits; + } else { + return GetExperimentalJavaFieldTypeForSingular(field) | extra_bits; + } +} + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" diff --git a/src/google/protobuf/compiler/java/internal_helpers.h b/src/google/protobuf/compiler/java/internal_helpers.h new file mode 100644 index 0000000000..e2534b95bf --- /dev/null +++ b/src/google/protobuf/compiler/java/internal_helpers.h @@ -0,0 +1,69 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ + +#include "google/protobuf/compiler/java/generator.h" +#include "google/protobuf/compiler/java/java_features.pb.h" +#include "google/protobuf/compiler/java/names.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" + +// Must be last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { + +// Whether unknown enum values are kept (i.e., not stored in UnknownFieldSet +// but in the message and can be queried using additional getters that return +// ints. +inline bool SupportUnknownEnumValue(const FieldDescriptor* field) { + if (JavaGenerator::GetResolvedSourceFeatures(*field) + .GetExtension(pb::java) + .legacy_closed_enum()) { + return false; + } + return field->enum_type() != nullptr && !field->enum_type()->is_closed(); +} + +inline bool CheckUtf8(const FieldDescriptor* descriptor) { + if (JavaGenerator::GetResolvedSourceFeatures(*descriptor) + .GetExtension(pb::java) + .utf8_validation() == pb::JavaFeatures::VERIFY) { + return true; + } + return JavaGenerator::GetResolvedSourceFeatures(*descriptor) + .utf8_validation() == FeatureSet::VERIFY || + // For legacy syntax. This is not allowed under Editions. + descriptor->file()->options().java_string_check_utf8(); +} + +// Only the lowest two bytes of the return value are used. The lowest byte +// is the integer value of a j/c/g/protobuf/FieldType enum. For the other +// byte: +// bit 0: whether the field is required. +// bit 1: whether the field requires UTF-8 validation. +// bit 2: whether the field needs isInitialized check. +// bit 3: whether the field is a map field with proto2 enum value. +// bits 4-7: unused +int GetExperimentalJavaFieldType(const FieldDescriptor* field); + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" +#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ diff --git a/src/google/protobuf/compiler/java/lite/BUILD.bazel b/src/google/protobuf/compiler/java/lite/BUILD.bazel index 7904d4b42e..a26f8f9da6 100644 --- a/src/google/protobuf/compiler/java/lite/BUILD.bazel +++ b/src/google/protobuf/compiler/java/lite/BUILD.bazel @@ -30,6 +30,7 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/io:printer", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", @@ -66,6 +67,7 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/compiler/java/immutable:service", "//src/google/protobuf/io", "//src/google/protobuf/io:printer", diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index c26797739e..bffec06347 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -17,11 +17,11 @@ #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" #include "google/protobuf/compiler/java/context.h" #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 491523df91..7ab58c50ba 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -14,6 +14,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index c46407bf51..84a10b9888 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -19,6 +19,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index c9b3a2947a..838df002ce 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -21,6 +21,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index a08bae50b6..5e926422bc 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -22,6 +22,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/wire_format.h" From 9340eec4224a8ea050f3b75060e961996c1aba5b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 2 May 2024 17:25:22 +0000 Subject: [PATCH 088/179] Auto-generate files after cl/630099889 --- src/file_lists.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 345d44bf5e..318d0763e8 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -346,6 +346,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/primitive_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/service.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/string_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/internal_helpers.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/java_features.pb.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/kotlin_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/enum.cc @@ -474,6 +475,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/primitive_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/service.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/string_field.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/internal_helpers.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/java_features.pb.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/kotlin_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/enum.h From 0dbd99a41db54a1adcfdc0946e9b8b724739a6e5 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 May 2024 15:07:47 -0700 Subject: [PATCH 089/179] Clarify map behaviors in editions. Map fields should remain length-prefixed for now, even if DELIMITED is inherited. Field presence will remain unchanged, but unit-tests are added to make sure proto2/proto3 behaviors stay consistent. Closes #16549 PiperOrigin-RevId: 630191163 --- src/google/protobuf/descriptor.cc | 23 ++- src/google/protobuf/descriptor_unittest.cc | 159 ++++++++++++++++----- 2 files changed, 143 insertions(+), 39 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index e16e11db4d..96df6f02bd 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -4362,7 +4362,8 @@ class DescriptorBuilder { DescriptorPool::ErrorCollector::ErrorLocation error_location, bool force_merge = false); - void PostProcessFieldFeatures(FieldDescriptor& field); + void PostProcessFieldFeatures(FieldDescriptor& field, + const FieldDescriptorProto& proto); // Allocates an array of two strings, the first one is a copy of // `proto_name`, and the second one is the full name. Full proto name is @@ -5542,7 +5543,8 @@ void DescriptorBuilder::ResolveFeatures(const FileDescriptorProto& proto, /*force_merge=*/true); } -void DescriptorBuilder::PostProcessFieldFeatures(FieldDescriptor& field) { +void DescriptorBuilder::PostProcessFieldFeatures( + FieldDescriptor& field, const FieldDescriptorProto& proto) { // TODO This can be replace by a runtime check in `is_required` // once the `label` getter is hidden. if (field.features().field_presence() == FeatureSet::LEGACY_REQUIRED && @@ -5552,8 +5554,15 @@ void DescriptorBuilder::PostProcessFieldFeatures(FieldDescriptor& field) { // TODO This can be replace by a runtime check of `is_delimited` // once the `TYPE_GROUP` value is removed. if (field.type_ == FieldDescriptor::TYPE_MESSAGE && + !field.containing_type()->options().map_entry() && field.features().message_encoding() == FeatureSet::DELIMITED) { - field.type_ = FieldDescriptor::TYPE_GROUP; + Symbol type = + LookupSymbol(proto.type_name(), field.full_name(), + DescriptorPool::PLACEHOLDER_MESSAGE, LOOKUP_TYPES, false); + if (type.descriptor() == nullptr || + !type.descriptor()->options().map_entry()) { + field.type_ = FieldDescriptor::TYPE_GROUP; + } } } @@ -6099,9 +6108,11 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( }); // Post-process cleanup for field features. - internal::VisitDescriptors(*result, [&](const FieldDescriptor& field) { - PostProcessFieldFeatures(const_cast(field)); - }); + internal::VisitDescriptors( + *result, proto, + [&](const FieldDescriptor& field, const FieldDescriptorProto& proto) { + PostProcessFieldFeatures(const_cast(field), proto); + }); // Interpret any remaining uninterpreted options gathered into // options_to_interpret_ during descriptor building. Cross-linking has made diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index adb07750ae..f2797b9f43 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -4059,6 +4059,22 @@ class ValidationErrorTest : public testing::Test { return ABSL_DIE_IF_NULL(pool_.BuildFile(file_proto)); } + const FileDescriptor* ParseAndBuildFile(absl::string_view file_name, + absl::string_view file_text) { + io::ArrayInputStream input_stream(file_text.data(), file_text.size()); + SimpleErrorCollector error_collector; + io::Tokenizer tokenizer(&input_stream, &error_collector); + compiler::Parser parser; + parser.RecordErrorsTo(&error_collector); + FileDescriptorProto proto; + ABSL_CHECK(parser.Parse(&tokenizer, &proto)) + << error_collector.last_error() << "\n" + << file_text; + ABSL_CHECK_EQ("", error_collector.last_error()); + proto.set_name(file_name); + return pool_.BuildFile(proto); + } + // Add file_proto to the DescriptorPool. Expect errors to be produced which // match the given error text. @@ -8684,7 +8700,9 @@ TEST_F(FeaturesTest, OneofFieldFeaturesOverride) { } TEST_F(FeaturesTest, MapFieldFeaturesOverride) { - constexpr absl::string_view kProtoFile = R"schema( + BuildDescriptorMessagesInTestPool(); + BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); + const FileDescriptor* file = ParseAndBuildFile("foo.proto", R"schema( edition = "2023"; import "google/protobuf/unittest_features.proto"; @@ -8701,22 +8719,7 @@ TEST_F(FeaturesTest, MapFieldFeaturesOverride) { features.(pb.test).multiple_feature = VALUE3 ]; } - )schema"; - io::ArrayInputStream input_stream(kProtoFile.data(), kProtoFile.size()); - SimpleErrorCollector error_collector; - io::Tokenizer tokenizer(&input_stream, &error_collector); - compiler::Parser parser; - parser.RecordErrorsTo(&error_collector); - FileDescriptorProto proto; - ASSERT_TRUE(parser.Parse(&tokenizer, &proto)) - << error_collector.last_error() << "\n" - << kProtoFile; - ASSERT_EQ("", error_collector.last_error()); - proto.set_name("foo.proto"); - - BuildDescriptorMessagesInTestPool(); - BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); - const FileDescriptor* file = pool_.BuildFile(proto); + )schema"); ASSERT_THAT(file, NotNull()); const FieldDescriptor* map_field = file->message_type(0)->field(0); @@ -8744,7 +8747,8 @@ TEST_F(FeaturesTest, MapFieldFeaturesOverride) { } TEST_F(FeaturesTest, MapFieldFeaturesStringValidation) { - constexpr absl::string_view kProtoFile = R"schema( + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* file = ParseAndBuildFile("foo.proto", R"schema( edition = "2023"; message Foo { @@ -8758,21 +8762,7 @@ TEST_F(FeaturesTest, MapFieldFeaturesStringValidation) { features.utf8_validation = NONE ]; } - )schema"; - io::ArrayInputStream input_stream(kProtoFile.data(), kProtoFile.size()); - SimpleErrorCollector error_collector; - io::Tokenizer tokenizer(&input_stream, &error_collector); - compiler::Parser parser; - parser.RecordErrorsTo(&error_collector); - FileDescriptorProto proto; - ASSERT_TRUE(parser.Parse(&tokenizer, &proto)) - << error_collector.last_error() << "\n" - << kProtoFile; - ASSERT_EQ("", error_collector.last_error()); - proto.set_name("foo.proto"); - - BuildDescriptorMessagesInTestPool(); - const FileDescriptor* file = pool_.BuildFile(proto); + )schema"); ASSERT_THAT(file, NotNull()); auto validate_map_field = [](const FieldDescriptor* field) { @@ -8789,6 +8779,109 @@ TEST_F(FeaturesTest, MapFieldFeaturesStringValidation) { validate_map_field(file->message_type(0)->field(2)); } +TEST_F(FeaturesTest, MapFieldFeaturesImplicitPresence) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* editions = ParseAndBuildFile("editions.proto", R"schema( + edition = "2023"; + + option features.field_presence = IMPLICIT; + + message Foo { + map message_map = 1; + map string_map = 2; + } + )schema"); + ASSERT_THAT(editions, NotNull()); + const FileDescriptor* proto3 = ParseAndBuildFile("proto3.proto", R"schema( + syntax = "proto3"; + + message Bar { + map message_map = 1; + map string_map = 2; + } + )schema"); + ASSERT_THAT(proto3, NotNull()); + + auto validate_maps = [](const FileDescriptor* file) { + const FieldDescriptor* message_map = file->message_type(0)->field(0); + EXPECT_FALSE(message_map->has_presence()); + EXPECT_FALSE(message_map->message_type()->field(0)->has_presence()); + EXPECT_TRUE(message_map->message_type()->field(1)->has_presence()); + + const FieldDescriptor* string_map = file->message_type(0)->field(1); + EXPECT_FALSE(string_map->has_presence()); + EXPECT_FALSE(string_map->message_type()->field(0)->has_presence()); + EXPECT_FALSE(string_map->message_type()->field(1)->has_presence()); + }; + validate_maps(editions); + validate_maps(proto3); +} + +TEST_F(FeaturesTest, MapFieldFeaturesExplicitPresence) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* editions = ParseAndBuildFile("editions.proto", R"schema( + edition = "2023"; + + message Foo { + map message_map = 1; + map string_map = 2; + } + )schema"); + ASSERT_THAT(editions, NotNull()); + const FileDescriptor* proto2 = ParseAndBuildFile("google.protobuf.proto", R"schema( + syntax = "proto2"; + + message Bar { + map message_map = 1; + map string_map = 2; + } + )schema"); + ASSERT_THAT(proto2, NotNull()); + + auto validate_maps = [](const FileDescriptor* file) { + const FieldDescriptor* message_map = file->message_type(0)->field(0); + EXPECT_FALSE(message_map->has_presence()); + EXPECT_TRUE(message_map->message_type()->field(0)->has_presence()); + EXPECT_TRUE(message_map->message_type()->field(1)->has_presence()); + + const FieldDescriptor* string_map = file->message_type(0)->field(1); + EXPECT_FALSE(string_map->has_presence()); + EXPECT_TRUE(string_map->message_type()->field(0)->has_presence()); + EXPECT_TRUE(string_map->message_type()->field(1)->has_presence()); + }; + validate_maps(editions); + validate_maps(proto2); +} + +TEST_F(FeaturesTest, MapFieldFeaturesInheritedMessageEncoding) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* file = ParseAndBuildFile("foo.proto", R"schema( + edition = "2023"; + + option features.message_encoding = DELIMITED; + + message Foo { + map message_map = 1; + map string_map = 2; + } + )schema"); + ASSERT_THAT(file, NotNull()); + + const FieldDescriptor* message_map = file->message_type(0)->field(0); + EXPECT_EQ(message_map->type(), FieldDescriptor::TYPE_MESSAGE); + EXPECT_EQ(message_map->message_type()->field(0)->type(), + FieldDescriptor::TYPE_INT32); + EXPECT_EQ(message_map->message_type()->field(1)->type(), + FieldDescriptor::TYPE_MESSAGE); + + const FieldDescriptor* string_map = file->message_type(0)->field(1); + EXPECT_EQ(string_map->type(), FieldDescriptor::TYPE_MESSAGE); + EXPECT_EQ(string_map->message_type()->field(0)->type(), + FieldDescriptor::TYPE_STRING); + EXPECT_EQ(string_map->message_type()->field(1)->type(), + FieldDescriptor::TYPE_STRING); +} + TEST_F(FeaturesTest, RootExtensionFeaturesOverride) { BuildDescriptorMessagesInTestPool(); BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); From d6c283321e7b2e159a6b300522b3d9f850e7de40 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 May 2024 16:04:43 -0700 Subject: [PATCH 090/179] Fix validation checks of implicit presence. Instead of checking the resolved features, we should really be checking the has_presence helper. Repeated fields, oneofs, and extensions can trigger these conditions when they inherit IMPLICIT, even though it's ignored. Closes #16664 PiperOrigin-RevId: 630206208 --- src/google/protobuf/descriptor.cc | 20 ++-- src/google/protobuf/descriptor_unittest.cc | 119 ++++++++++++++++++++- 2 files changed, 128 insertions(+), 11 deletions(-) diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 96df6f02bd..47eeb9012d 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8084,16 +8084,16 @@ void DescriptorBuilder::ValidateFieldFeatures( } // Validate fully resolved features. - if (field->has_default_value() && - field->features().field_presence() == FeatureSet::IMPLICIT) { - AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, - "Implicit presence fields can't specify defaults."); - } - if (field->enum_type() != nullptr && - field->enum_type()->features().enum_type() != FeatureSet::OPEN && - field->features().field_presence() == FeatureSet::IMPLICIT) { - AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, - "Implicit presence enum fields must always be open."); + if (!field->is_repeated() && !field->has_presence()) { + if (field->has_default_value()) { + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, + "Implicit presence fields can't specify defaults."); + } + if (field->enum_type() != nullptr && + field->enum_type()->features().enum_type() != FeatureSet::OPEN) { + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, + "Implicit presence enum fields must always be open."); + } } if (field->is_extension() && field->features().field_presence() == FeatureSet::LEGACY_REQUIRED) { diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index f2797b9f43..0c173c97d3 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -9830,7 +9830,62 @@ TEST_F(FeaturesTest, InvalidFieldImplicitDefault) { "defaults.\n"); } -TEST_F(FeaturesTest, InvalidFieldImplicitOpen) { +TEST_F(FeaturesTest, ValidExtensionFieldImplicitDefault) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* file = BuildFile( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2023 + options { features { field_presence: IMPLICIT } } + message_type { + name: "Foo" + extension_range { start: 1 end: 100 } + } + extension { + name: "bar" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + default_value: "Hello world" + extendee: "Foo" + } + )pb"); + ASSERT_THAT(file, NotNull()); + + EXPECT_TRUE(file->extension(0)->has_presence()); + EXPECT_EQ(file->extension(0)->default_value_string(), "Hello world"); +} + +TEST_F(FeaturesTest, ValidOneofFieldImplicitDefault) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* file = BuildFile( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2023 + options { features { field_presence: IMPLICIT } } + message_type { + name: "Foo" + field { + name: "bar" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + default_value: "Hello world" + oneof_index: 0 + } + oneof_decl { name: "_foo" } + } + )pb"); + ASSERT_THAT(file, NotNull()); + + EXPECT_TRUE(file->message_type(0)->field(0)->has_presence()); + EXPECT_EQ(file->message_type(0)->field(0)->default_value_string(), + "Hello world"); +} + +TEST_F(FeaturesTest, InvalidFieldImplicitClosed) { BuildDescriptorMessagesInTestPool(); BuildFileWithErrors( R"pb( @@ -9858,6 +9913,68 @@ TEST_F(FeaturesTest, InvalidFieldImplicitOpen) { "be open.\n"); } +TEST_F(FeaturesTest, ValidRepeatedFieldImplicitClosed) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* file = BuildFile( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2023 + options { features { field_presence: IMPLICIT } } + message_type { + name: "Foo" + field { + name: "bar" + number: 1 + label: LABEL_REPEATED + type: TYPE_ENUM + type_name: "Enum" + } + } + enum_type { + name: "Enum" + value { name: "BAR" number: 0 } + options { features { enum_type: CLOSED } } + } + )pb"); + ASSERT_THAT(file, NotNull()); + + EXPECT_FALSE(file->message_type(0)->field(0)->has_presence()); + EXPECT_TRUE(file->enum_type(0)->is_closed()); +} + +TEST_F(FeaturesTest, ValidOneofFieldImplicitClosed) { + BuildDescriptorMessagesInTestPool(); + const FileDescriptor* file = BuildFile( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2023 + options { features { field_presence: IMPLICIT } } + message_type { + name: "Foo" + field { + name: "bar" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_ENUM + type_name: "Enum" + oneof_index: 0 + } + oneof_decl { name: "_foo" } + } + enum_type { + name: "Enum" + value { name: "BAR" number: 0 } + options { features { enum_type: CLOSED } } + } + )pb"); + ASSERT_THAT(file, NotNull()); + + EXPECT_TRUE(file->message_type(0)->field(0)->has_presence()); + EXPECT_TRUE(file->enum_type(0)->is_closed()); +} + TEST_F(FeaturesTest, InvalidFieldRequiredExtension) { BuildDescriptorMessagesInTestPool(); BuildFileWithErrors( From cc79f776dcb09164e7a0b6c11951679244a7828b Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Thu, 2 May 2024 17:47:08 -0700 Subject: [PATCH 091/179] Proto java full runtime: avoid allocating Integers accessing enum lists Previously, we would allocate a boxed Integer when accessing repeated enums, because we used ArrayList, which must box the int. Use IntArrayList and the primitive "int getInt()" method instead. PiperOrigin-RevId: 630229119 --- .../compiler/java/immutable/enum_field.cc | 99 +++++++++---------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/immutable/enum_field.cc index 4d0f65539d..71207a9d49 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/immutable/enum_field.cc @@ -687,13 +687,13 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( printer->Print( variables_, "@SuppressWarnings(\"serial\")\n" - "private java.util.List $name$_;\n" + "private com.google.protobuf.Internal.IntList $name$_;\n" "private static final " - "com.google.protobuf.Internal.ListAdapter.Converter<\n" - " java.lang.Integer, $type$> $name$_converter_ =\n" - " new com.google.protobuf.Internal.ListAdapter.Converter<\n" - " java.lang.Integer, $type$>() {\n" - " public $type$ convert(java.lang.Integer from) {\n" + "com.google.protobuf.Internal.IntListAdapter.IntConverter<\n" + " $type$> $name$_converter_ =\n" + " new com.google.protobuf.Internal.IntListAdapter.IntConverter<\n" + " $type$>() {\n" + " public $type$ convert(int from) {\n" " $type$ result = $type$.forNumber(from);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -701,14 +701,13 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( PrintExtraFieldInfo(variables_, printer); WriteFieldAccessorDocComment(printer, descriptor_, LIST_GETTER, context_->options()); - printer->Print( - variables_, - "@java.lang.Override\n" - "$deprecation$public java.util.List<$type$> " - "${$get$capitalized_name$List$}$() {\n" - " return new com.google.protobuf.Internal.ListAdapter<\n" - " java.lang.Integer, $type$>($name$_, $name$_converter_);\n" - "}\n"); + printer->Print(variables_, + "@java.lang.Override\n" + "$deprecation$public java.util.List<$type$> " + "${$get$capitalized_name$List$}$() {\n" + " return new com.google.protobuf.Internal.IntListAdapter<\n" + " $type$>($name$_, $name$_converter_);\n" + "}\n"); printer->Annotate("{", "}", descriptor_); WriteFieldAccessorDocComment(printer, descriptor_, LIST_COUNT, context_->options()); @@ -725,7 +724,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$(int index) {\n" - " return $name$_converter_.convert($name$_.get(index));\n" + " return $name$_converter_.convert($name$_.getInt(index));\n" "}\n"); printer->Annotate("{", "}", descriptor_); if (SupportUnknownEnumValue(descriptor_)) { @@ -744,7 +743,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( "@java.lang.Override\n" "$deprecation$public int " "${$get$capitalized_name$Value$}$(int index) {\n" - " return $name$_.get(index);\n" + " return $name$_.getInt(index);\n" "}\n"); printer->Annotate("{", "}", descriptor_); } @@ -760,19 +759,18 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( variables_, // One field is the list and the other field keeps track of whether the // list is immutable. If it's immutable, the invariant is that it must - // either an instance of Collections.emptyList() or it's an ArrayList - // wrapped in a Collections.unmodifiableList() wrapper and nobody else has - // a reference to the underlying ArrayList. This invariant allows us to - // share instances of lists between protocol buffers avoiding expensive - // memory allocations. Note, immutable is a strong guarantee here -- not - // just that the list cannot be modified via the reference but that the - // list can never be modified. - "private java.util.List $name$_ =\n" - " java.util.Collections.emptyList();\n" + // either an instance of emptyIntList() or it's an immutable IntArrayList + // and nobody else has a reference to the underlying ArrayList. This + // invariant allows us to share instances of lists between protocol + // buffers avoiding expensive memory allocations. Note, immutable is a + // strong guarantee here -- not just that the list cannot be modified via + // the reference but that the list can never be modified. + "private com.google.protobuf.Internal.IntList $name$_ =\n" + " emptyIntList();\n" "private void ensure$capitalized_name$IsMutable() {\n" " if (!$get_mutable_bit_builder$) {\n" - " $name$_ = new java.util.ArrayList($name$_);\n" + " $name$_ = makeMutableCopy($name$_);\n" " $set_mutable_bit_builder$;\n" " }\n" "}\n"); @@ -787,8 +785,8 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( // immutable. "$deprecation$public java.util.List<$type$> " "${$get$capitalized_name$List$}$() {\n" - " return new com.google.protobuf.Internal.ListAdapter<\n" - " java.lang.Integer, $type$>($name$_, $name$_converter_);\n" + " return new com.google.protobuf.Internal.IntListAdapter<\n" + " $type$>($name$_, $name$_converter_);\n" "}\n"); printer->Annotate("{", "}", descriptor_); WriteFieldAccessorDocComment(printer, descriptor_, LIST_COUNT, @@ -804,7 +802,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( printer->Print( variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$(int index) {\n" - " return $name$_converter_.convert($name$_.get(index));\n" + " return $name$_converter_.convert($name$_.getInt(index));\n" "}\n"); printer->Annotate("{", "}", descriptor_); WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, @@ -817,7 +815,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " throw new NullPointerException();\n" " }\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.set(index, value.getNumber());\n" + " $name$_.setInt(index, value.getNumber());\n" " onChanged();\n" " return this;\n" "}\n"); @@ -832,7 +830,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " throw new NullPointerException();\n" " }\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.add(value.getNumber());\n" + " $name$_.addInt(value.getNumber());\n" " onChanged();\n" " return this;\n" "}\n"); @@ -845,7 +843,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " java.lang.Iterable values) {\n" " ensure$capitalized_name$IsMutable();\n" " for ($type$ value : values) {\n" - " $name$_.add(value.getNumber());\n" + " $name$_.addInt(value.getNumber());\n" " }\n" " onChanged();\n" " return this;\n" @@ -857,7 +855,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( printer->Print( variables_, "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" - " $name$_ = java.util.Collections.emptyList();\n" + " $name$_ = emptyIntList();\n" " $clear_mutable_bit_builder$;\n" " onChanged();\n" " return this;\n" @@ -878,7 +876,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( printer->Print(variables_, "$deprecation$public int " "${$get$capitalized_name$Value$}$(int index) {\n" - " return $name$_.get(index);\n" + " return $name$_.getInt(index);\n" "}\n"); printer->Annotate("{", "}", descriptor_); WriteFieldEnumValueAccessorDocComment( @@ -889,7 +887,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "$deprecation$public Builder ${$set$capitalized_name$Value$}$(\n" " int index, int value) {\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.set(index, value);\n" + " $name$_.setInt(index, value);\n" " onChanged();\n" " return this;\n" "}\n"); @@ -901,7 +899,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "$deprecation$public Builder " "${$add$capitalized_name$Value$}$(int value) {\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.add(value);\n" + " $name$_.addInt(value);\n" " onChanged();\n" " return this;\n" "}\n"); @@ -915,7 +913,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " java.lang.Iterable values) {\n" " ensure$capitalized_name$IsMutable();\n" " for (int value : values) {\n" - " $name$_.add(value);\n" + " $name$_.addInt(value);\n" " }\n" " onChanged();\n" " return this;\n" @@ -931,13 +929,13 @@ void RepeatedImmutableEnumFieldGenerator:: void RepeatedImmutableEnumFieldGenerator::GenerateInitializationCode( io::Printer* printer) const { - printer->Print(variables_, "$name$_ = java.util.Collections.emptyList();\n"); + printer->Print(variables_, "$name$_ = emptyIntList();\n"); } void RepeatedImmutableEnumFieldGenerator::GenerateBuilderClearCode( io::Printer* printer) const { printer->Print(variables_, - "$name$_ = java.util.Collections.emptyList();\n" + "$name$_ = emptyIntList();\n" "$clear_mutable_bit_builder$;\n"); } @@ -965,13 +963,12 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuildingCode( io::Printer* printer) const { // The code below ensures that the result has an immutable list. If our // list is immutable, we can just reuse it. If not, we make it immutable. - printer->Print( - variables_, - "if ($get_mutable_bit_builder$) {\n" - " $name$_ = java.util.Collections.unmodifiableList($name$_);\n" - " $clear_mutable_bit_builder$;\n" - "}\n" - "result.$name$_ = $name$_;\n"); + printer->Print(variables_, + "if ($get_mutable_bit_builder$) {\n" + " $name$_.makeImmutable();\n" + " $clear_mutable_bit_builder$;\n" + "}\n" + "result.$name$_ = $name$_;\n"); } void RepeatedImmutableEnumFieldGenerator::GenerateBuilderParsingCode( @@ -981,7 +978,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderParsingCode( printer->Print(variables_, "int tmpRaw = input.readEnum();\n" "ensure$capitalized_name$IsMutable();\n" - "$name$_.add(tmpRaw);\n"); + "$name$_.addInt(tmpRaw);\n"); } else { printer->Print(variables_, "int tmpRaw = input.readEnum();\n" @@ -991,7 +988,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderParsingCode( " mergeUnknownVarintField($number$, tmpRaw);\n" "} else {\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.add(tmpRaw);\n" + " $name$_.addInt(tmpRaw);\n" "}\n"); } } @@ -1022,12 +1019,12 @@ void RepeatedImmutableEnumFieldGenerator::GenerateSerializationCode( " output.writeUInt32NoTag($name$MemoizedSerializedSize);\n" "}\n" "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.writeEnumNoTag($name$_.get(i));\n" + " output.writeEnumNoTag($name$_.getInt(i));\n" "}\n"); } else { printer->Print(variables_, "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.writeEnum($number$, $name$_.get(i));\n" + " output.writeEnum($number$, $name$_.getInt(i));\n" "}\n"); } } @@ -1042,7 +1039,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateSerializedSizeCode( printer->Print(variables_, "for (int i = 0; i < $name$_.size(); i++) {\n" " dataSize += com.google.protobuf.CodedOutputStream\n" - " .computeEnumSizeNoTag($name$_.get(i));\n" + " .computeEnumSizeNoTag($name$_.getInt(i));\n" "}\n"); printer->Print("size += dataSize;\n"); if (descriptor_->is_packed()) { From 4f12891b87a99ecabdc1e1039d0587094e9b57cd Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Thu, 2 May 2024 18:13:35 -0700 Subject: [PATCH 092/179] Java full runtime: Avoid allocating iterator if UnknownFieldSet's TreeMap is empty I also considered using Collections.emptyNavigableMap(), which I thought might use a specialized allocation free empty iterator, but it allocates, and its clone() method isn't nicely exposed. PiperOrigin-RevId: 630234849 --- .../com/google/protobuf/UnknownFieldSet.java | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index 10fd7a7f62..38aecac42f 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -102,6 +102,10 @@ public final class UnknownFieldSet implements MessageLite { /** Serializes the set and writes it to {@code output}. */ @Override public void writeTo(CodedOutputStream output) throws IOException { + if (fields.isEmpty()) { + // Avoid allocating an iterator. + return; + } for (Map.Entry entry : fields.entrySet()) { Field field = entry.getValue(); field.writeTo(entry.getKey(), output); @@ -174,16 +178,22 @@ public final class UnknownFieldSet implements MessageLite { @Override public int getSerializedSize() { int result = 0; - if (!fields.isEmpty()) { - for (Map.Entry entry : fields.entrySet()) { - result += entry.getValue().getSerializedSize(entry.getKey()); - } + if (fields.isEmpty()) { + // Avoid allocating an iterator. + return result; + } + for (Map.Entry entry : fields.entrySet()) { + result += entry.getValue().getSerializedSize(entry.getKey()); } return result; } /** Serializes the set and writes it to {@code output} using {@code MessageSet} wire format. */ public void writeAsMessageSetTo(CodedOutputStream output) throws IOException { + if (fields.isEmpty()) { + // Avoid allocating an iterator. + return; + } for (Map.Entry entry : fields.entrySet()) { entry.getValue().writeAsMessageSetExtensionTo(entry.getKey(), output); } @@ -191,6 +201,10 @@ public final class UnknownFieldSet implements MessageLite { /** Serializes the set and writes it to {@code writer}. */ void writeTo(Writer writer) throws IOException { + if (fields.isEmpty()) { + // Avoid allocating an iterator. + return; + } if (writer.fieldOrder() == Writer.FieldOrder.DESCENDING) { // Write fields in descending order. for (Map.Entry entry : fields.descendingMap().entrySet()) { @@ -206,6 +220,10 @@ public final class UnknownFieldSet implements MessageLite { /** Serializes the set and writes it to {@code writer} using {@code MessageSet} wire format. */ void writeAsMessageSetTo(Writer writer) throws IOException { + if (fields.isEmpty()) { + // Avoid allocating an iterator. + return; + } if (writer.fieldOrder() == Writer.FieldOrder.DESCENDING) { // Write fields in descending order. for (Map.Entry entry : fields.descendingMap().entrySet()) { @@ -222,6 +240,10 @@ public final class UnknownFieldSet implements MessageLite { /** Get the number of bytes required to encode this set using {@code MessageSet} wire format. */ public int getSerializedSizeAsMessageSet() { int result = 0; + if (fields.isEmpty()) { + // Avoid allocating an iterator. + return result; + } for (Map.Entry entry : fields.entrySet()) { result += entry.getValue().getSerializedSizeAsMessageSetExtension(entry.getKey()); } From d9ff109888b60a37de1c4f69cea023f89886e86b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 2 May 2024 21:46:06 -0700 Subject: [PATCH 093/179] internal change PiperOrigin-RevId: 630273635 --- .../protobuf/compiler/java/immutable/BUILD.bazel | 2 ++ src/google/protobuf/compiler/java/immutable/enum.cc | 10 ++++++++++ src/google/protobuf/compiler/java/immutable/enum.h | 1 + src/google/protobuf/compiler/java/internal_helpers.h | 1 + 4 files changed, 14 insertions(+) diff --git a/src/google/protobuf/compiler/java/immutable/BUILD.bazel b/src/google/protobuf/compiler/java/immutable/BUILD.bazel index baa679ef42..0f01091bcd 100644 --- a/src/google/protobuf/compiler/java/immutable/BUILD.bazel +++ b/src/google/protobuf/compiler/java/immutable/BUILD.bazel @@ -85,9 +85,11 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/io:printer", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", ], ) diff --git a/src/google/protobuf/compiler/java/immutable/enum.cc b/src/google/protobuf/compiler/java/immutable/enum.cc index 61b307914c..9ceefcb1a8 100644 --- a/src/google/protobuf/compiler/java/immutable/enum.cc +++ b/src/google/protobuf/compiler/java/immutable/enum.cc @@ -11,13 +11,20 @@ #include "google/protobuf/compiler/java/immutable/enum.h" +#include +#include #include +#include #include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" #include "google/protobuf/compiler/java/context.h" #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" @@ -30,6 +37,7 @@ namespace protobuf { namespace compiler { namespace java { + EnumNonLiteGenerator::EnumNonLiteGenerator(const EnumDescriptor* descriptor, bool immutable_api, Context* context) : descriptor_(descriptor), @@ -56,6 +64,7 @@ void EnumNonLiteGenerator::Generate(io::Printer* printer) { WriteEnumDocComment(printer, descriptor_, context_->options()); MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_); + if (!context_->options().opensource_runtime) { printer->Print("@com.google.protobuf.Internal.ProtoNonnullApi\n"); } @@ -374,6 +383,7 @@ void EnumNonLiteGenerator::Generate(io::Printer* printer) { printer->Print("}\n\n"); } + bool EnumNonLiteGenerator::CanUseEnumValues() { if (canonical_values_.size() != descriptor_->value_count()) { return false; diff --git a/src/google/protobuf/compiler/java/immutable/enum.h b/src/google/protobuf/compiler/java/immutable/enum.h index 18ccc01e95..8528d7f81f 100644 --- a/src/google/protobuf/compiler/java/immutable/enum.h +++ b/src/google/protobuf/compiler/java/immutable/enum.h @@ -71,6 +71,7 @@ class EnumNonLiteGenerator : public EnumGenerator { ClassNameResolver* name_resolver_; bool CanUseEnumValues(); + }; } // namespace java diff --git a/src/google/protobuf/compiler/java/internal_helpers.h b/src/google/protobuf/compiler/java/internal_helpers.h index e2534b95bf..607669b26a 100644 --- a/src/google/protobuf/compiler/java/internal_helpers.h +++ b/src/google/protobuf/compiler/java/internal_helpers.h @@ -50,6 +50,7 @@ inline bool CheckUtf8(const FieldDescriptor* descriptor) { descriptor->file()->options().java_string_check_utf8(); } + // Only the lowest two bytes of the return value are used. The lowest byte // is the integer value of a j/c/g/protobuf/FieldType enum. For the other // byte: From 733b9c54e9ce720bb2ba77e980778616da45479d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Sad=C5=82ocha?= Date: Fri, 3 May 2024 03:48:49 -0700 Subject: [PATCH 094/179] Implement `is_known` method on the `Enum` trait This is equivalent to C++'s `_IsValid` functions. Since proto3 enums are open, `is_known` seems to be a better name than the misleading `is_valid`. C++-specific Protocol Buffers documentation already uses "known fields" and "unknown enum values" expressions. PiperOrigin-RevId: 630344180 --- rust/enum.rs | 9 ++++++++- rust/shared.rs | 2 +- rust/test/shared/enum_test.rs | 22 +++++++++++++++++++++ src/google/protobuf/compiler/rust/enum.cc | 24 ++++++++++++----------- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/rust/enum.rs b/rust/enum.rs index 847ef41df5..d9b2394e81 100644 --- a/rust/enum.rs +++ b/rust/enum.rs @@ -19,9 +19,16 @@ use std::{ /// representation as erased enums in the runtime. /// - For C++, this is `proto2::RepeatedField` /// - For UPB, this is an array compatible with `int32` -pub unsafe trait Enum { +pub unsafe trait Enum: TryFrom { /// The name of the enum. const NAME: &'static str; + + /// Returns `true` if the given numeric value matches one of the `Self`'s + /// defined values. + /// + /// If `Self` is a closed enum, then `TryFrom` for `value` succeeds if + /// and only if this function returns `true`. + fn is_known(value: i32) -> bool; } /// An integer value wasn't known for an enum while converting. diff --git a/rust/shared.rs b/rust/shared.rs index 1e0c645291..9c839fa47a 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -22,7 +22,7 @@ use std::fmt; /// These are the items protobuf users can access directly. #[doc(hidden)] pub mod __public { - pub use crate::r#enum::UnknownEnumValue; + pub use crate::r#enum::{Enum, UnknownEnumValue}; pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; pub use crate::optional::Optional; pub use crate::proto; diff --git a/rust/test/shared/enum_test.rs b/rust/test/shared/enum_test.rs index c11a4906c0..7181e59d47 100644 --- a/rust/test/shared/enum_test.rs +++ b/rust/test/shared/enum_test.rs @@ -9,6 +9,7 @@ use enums_proto::*; use googletest::prelude::*; +use protobuf::Enum; use unittest_proto::*; #[test] @@ -167,3 +168,24 @@ fn test_enum_conversion_failure_impls_std_error() { let err = TestSparseEnum::try_from(1).unwrap_err(); let _test_compiles: &dyn std::error::Error = &err; } + +#[test] +fn test_is_known_for_closed_enum() { + assert_that!(test_all_types::NestedEnum::is_known(-2), eq(false)); + assert_that!(test_all_types::NestedEnum::is_known(-1), eq(true)); + assert_that!(test_all_types::NestedEnum::is_known(0), eq(false)); + assert_that!(test_all_types::NestedEnum::is_known(1), eq(true)); + assert_that!(test_all_types::NestedEnum::is_known(2), eq(true)); + assert_that!(test_all_types::NestedEnum::is_known(3), eq(true)); + assert_that!(test_all_types::NestedEnum::is_known(4), eq(false)); +} + +#[test] +fn test_is_known_for_open_enum() { + assert_that!(TestEnumWithNumericNames::is_known(-1), eq(false)); + assert_that!(TestEnumWithNumericNames::is_known(0), eq(true)); + assert_that!(TestEnumWithNumericNames::is_known(1), eq(true)); + assert_that!(TestEnumWithNumericNames::is_known(2), eq(true)); + assert_that!(TestEnumWithNumericNames::is_known(3), eq(true)); + assert_that!(TestEnumWithNumericNames::is_known(4), eq(false)); +} diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 521dcd3ce3..857d283538 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -325,23 +325,21 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { // The default value of an enum is the first listed value. // The compiler checks that this is equal to 0 for open enums. {"default_int_value", absl::StrCat(desc.value(0)->number())}, + {"known_values_pattern", + // TODO: Check validity in UPB/C++. + absl::StrJoin(values, "|", + [](std::string* o, const RustEnumValue& val) { + absl::StrAppend(o, val.number); + })}, {"impl_from_i32", [&] { if (desc.is_closed()) { - ctx.Emit({{"name", name}, - {"known_values_pattern", - // TODO: Check validity in UPB/C++. - absl::StrJoin( - values, "|", - [](std::string* o, const RustEnumValue& val) { - absl::StrAppend(o, val.number); - })}}, - R"rs( + ctx.Emit(R"rs( impl $std$::convert::TryFrom for $name$ { type Error = $pb$::UnknownEnumValue; fn try_from(val: i32) -> Result<$name$, Self::Error> { - if matches!(val, $known_values_pattern$) { + if ::is_known(val) { Ok(Self(val)) } else { Err($pb$::UnknownEnumValue::new($pbi$::Private, val)) @@ -350,7 +348,7 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } )rs"); } else { - ctx.Emit({{"name", name}}, R"rs( + ctx.Emit(R"rs( impl $std$::convert::From for $name$ { fn from(val: i32) -> $name$ { Self(val) @@ -457,6 +455,10 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { // SAFETY: this is an enum type unsafe impl $pbi$::Enum for $name$ { const NAME: &'static str = "$name$"; + + fn is_known(value: i32) -> bool { + matches!(value, $known_values_pattern$) + } } $impl_proxied_in_map$ From 353b4e003183d88e1e95776046c463461d59f6db Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Fri, 3 May 2024 08:13:17 -0700 Subject: [PATCH 095/179] Ensure overflow values for float/double map to inf/-inf. PiperOrigin-RevId: 630393984 --- conformance/text_format_conformance_suite.cc | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index 3eca94304c..6025199991 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -481,6 +481,27 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { ExpectParseFailure("FloatFieldNoOctal", REQUIRED, "optional_float: 012"); ExpectParseFailure("FloatFieldNoNegativeOctal", REQUIRED, "optional_float: -012"); + // https://protobuf.dev/reference/protobuf/textformat-spec/#value says + // overflows are mapped to infinity/-infinity. + RunValidTextFormatTestWithExpected("FloatFieldOverflowInfinity", REQUIRED, + "optional_float: 1e50", + "optional_float: inf"); + RunValidTextFormatTestWithExpected("FloatFieldOverflowNegativeInfinity", + REQUIRED, "optional_float: -1e50", + "optional_float: -inf"); + RunValidTextFormatTestWithExpected("DoubleFieldOverflowInfinity", REQUIRED, + "optional_double: 1e9999", + "optional_double: inf"); + RunValidTextFormatTestWithExpected("DoubleFieldOverflowNegativeInfinity", + REQUIRED, "optional_double: -1e9999", + "optional_double: -inf"); + // Exponent is one more than uint64 max. + RunValidTextFormatTestWithExpected( + "FloatFieldOverflowInfinityHugeExponent", REQUIRED, + "optional_float: 1e18446744073709551616", "optional_float: inf"); + RunValidTextFormatTestWithExpected( + "DoubleFieldOverflowInfinityHugeExponent", REQUIRED, + "optional_double: 1e18446744073709551616", "optional_double: inf"); // String literals x {Strings, Bytes} for (const auto& field_type : std::vector{"String", "Bytes"}) { From 7b3682fcf24e825b3eddbed309c3ace5ddfcaa2e Mon Sep 17 00:00:00 2001 From: Bastien Jacot-Guillarmod Date: Fri, 3 May 2024 09:50:45 -0700 Subject: [PATCH 096/179] Pin the rev of googletest to the GitHub main branch PiperOrigin-RevId: 630417312 --- .github/workflows/test_rust.yml | 2 +- Cargo.bazel.lock | 72 +++++++++++---------- Cargo.lock | 14 ++-- WORKSPACE | 3 +- rust/map.rs | 31 +++++---- rust/repeated.rs | 10 +-- rust/test/shared/accessors_map_test.rs | 53 +++++++-------- rust/test/shared/accessors_repeated_test.rs | 39 ++++++----- rust/test/shared/accessors_test.rs | 2 +- rust/test/shared/enum_test.rs | 12 ++-- 10 files changed, 119 insertions(+), 119 deletions(-) diff --git a/.github/workflows/test_rust.yml b/.github/workflows/test_rust.yml index cab8c4d874..00c2e4b7bb 100644 --- a/.github/workflows/test_rust.yml +++ b/.github/workflows/test_rust.yml @@ -23,7 +23,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel-docker@v3 with: - image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.1-75f2a85ece6526cc3d54087018c0f1097d78d42b" + image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.1-97f82260fd504923d8af642d567afb2d83a1959d" credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: rust_linux bazel: >- diff --git a/Cargo.bazel.lock b/Cargo.bazel.lock index a287466e2e..3855576c78 100644 --- a/Cargo.bazel.lock +++ b/Cargo.bazel.lock @@ -1,12 +1,12 @@ { - "checksum": "f93f5d1848bc00c6384273f9fb5273cc1b7fc0cb4dbc2afd776d2feb7b37f3ae", + "checksum": "8863e5b8f3da7cf4502f68bea0d455dec4834bf25ff070caaa58a8e1c5ea1a3d", "crates": { "aho-corasick 1.1.2": { "name": "aho-corasick", "version": "1.1.2", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/aho-corasick/1.1.2/download", + "url": "https://static.crates.io/crates/aho-corasick/1.1.2/download", "sha256": "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" } }, @@ -53,7 +53,7 @@ "version": "1.1.0", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/autocfg/1.1.0/download", + "url": "https://static.crates.io/crates/autocfg/1.1.0/download", "sha256": "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" } }, @@ -101,7 +101,7 @@ "deps": { "common": [ { - "id": "googletest 0.10.0", + "id": "googletest 0.11.0", "target": "googletest" } ], @@ -121,13 +121,16 @@ }, "license": null }, - "googletest 0.10.0": { + "googletest 0.11.0": { "name": "googletest", - "version": "0.10.0", + "version": "0.11.0", "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/googletest/0.10.0/download", - "sha256": "09213705c85aa0e4b4fff44a3a826a556979a34a266df6bcda703a49c69fb61e" + "Git": { + "remote": "https://github.com/google/googletest-rust", + "commitish": { + "Rev": "471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f" + }, + "strip_prefix": "googletest" } }, "targets": [ @@ -163,7 +166,7 @@ "proc_macro_deps": { "common": [ { - "id": "googletest_macro 0.10.0", + "id": "googletest_macro 0.11.0", "target": "googletest_macro" }, { @@ -173,17 +176,20 @@ ], "selects": {} }, - "version": "0.10.0" + "version": "0.11.0" }, "license": "Apache-2.0" }, - "googletest_macro 0.10.0": { + "googletest_macro 0.11.0": { "name": "googletest_macro", - "version": "0.10.0", + "version": "0.11.0", "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/googletest_macro/0.10.0/download", - "sha256": "005e4cb962c56efd249bdeeb4ac232b11e1c45a2e49793bba2b2982dcc3f2e9d" + "Git": { + "remote": "https://github.com/google/googletest-rust", + "commitish": { + "Rev": "471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f" + }, + "strip_prefix": "googletest_macro" } }, "targets": [ @@ -209,14 +215,14 @@ "target": "quote" }, { - "id": "syn 2.0.38", + "id": "syn 2.0.43", "target": "syn" } ], "selects": {} }, "edition": "2021", - "version": "0.10.0" + "version": "0.11.0" }, "license": "Apache-2.0" }, @@ -225,7 +231,7 @@ "version": "2.6.4", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/memchr/2.6.4/download", + "url": "https://static.crates.io/crates/memchr/2.6.4/download", "sha256": "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" } }, @@ -263,7 +269,7 @@ "version": "0.2.17", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/num-traits/0.2.17/download", + "url": "https://static.crates.io/crates/num-traits/0.2.17/download", "sha256": "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" } }, @@ -332,7 +338,7 @@ "version": "1.0.14", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/paste/1.0.14/download", + "url": "https://static.crates.io/crates/paste/1.0.14/download", "sha256": "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" } }, @@ -385,7 +391,7 @@ "version": "1.0.69", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/proc-macro2/1.0.69/download", + "url": "https://static.crates.io/crates/proc-macro2/1.0.69/download", "sha256": "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" } }, @@ -448,7 +454,7 @@ "version": "1.0.33", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/quote/1.0.33/download", + "url": "https://static.crates.io/crates/quote/1.0.33/download", "sha256": "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" } }, @@ -494,7 +500,7 @@ "version": "1.10.0", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/regex/1.10.0/download", + "url": "https://static.crates.io/crates/regex/1.10.0/download", "sha256": "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" } }, @@ -567,7 +573,7 @@ "version": "0.4.1", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/regex-automata/0.4.1/download", + "url": "https://static.crates.io/crates/regex-automata/0.4.1/download", "sha256": "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" } }, @@ -641,7 +647,7 @@ "version": "0.8.1", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/regex-syntax/0.8.1/download", + "url": "https://static.crates.io/crates/regex-syntax/0.8.1/download", "sha256": "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" } }, @@ -686,7 +692,7 @@ "version": "1.0.14", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/rustversion/1.0.14/download", + "url": "https://static.crates.io/crates/rustversion/1.0.14/download", "sha256": "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" } }, @@ -734,13 +740,13 @@ }, "license": "MIT OR Apache-2.0" }, - "syn 2.0.38": { + "syn 2.0.43": { "name": "syn", - "version": "2.0.38", + "version": "2.0.43", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/syn/2.0.38/download", - "sha256": "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" + "url": "https://static.crates.io/crates/syn/2.0.43/download", + "sha256": "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" } }, "targets": [ @@ -790,7 +796,7 @@ "selects": {} }, "edition": "2021", - "version": "2.0.38" + "version": "2.0.43" }, "license": "MIT OR Apache-2.0" }, @@ -799,7 +805,7 @@ "version": "1.0.12", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/unicode-ident/1.0.12/download", + "url": "https://static.crates.io/crates/unicode-ident/1.0.12/download", "sha256": "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" } }, diff --git a/Cargo.lock b/Cargo.lock index ea70571b3c..be9bb77bad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,9 +27,8 @@ dependencies = [ [[package]] name = "googletest" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09213705c85aa0e4b4fff44a3a826a556979a34a266df6bcda703a49c69fb61e" +version = "0.11.0" +source = "git+https://github.com/google/googletest-rust?rev=471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f#471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f" dependencies = [ "googletest_macro", "num-traits", @@ -39,9 +38,8 @@ dependencies = [ [[package]] name = "googletest_macro" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005e4cb962c56efd249bdeeb4ac232b11e1c45a2e49793bba2b2982dcc3f2e9d" +version = "0.11.0" +source = "git+https://github.com/google/googletest-rust?rev=471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f#471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f" dependencies = [ "quote", "syn", @@ -123,9 +121,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "syn" -version = "2.0.38" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", diff --git a/WORKSPACE b/WORKSPACE index 0e620deda3..def9ed993f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -215,7 +215,8 @@ crates_repository( lockfile = "//:Cargo.bazel.lock", packages = { "googletest": crate.spec( - version = ">0.0.0", + git = "https://github.com/google/googletest-rust", + rev = "471d4a2a8e8bc74f6d7d9c8eecb4d4e3157b2a9f", ), "paste": crate.spec( version = ">=1", diff --git a/rust/map.rs b/rust/map.rs index f63aa3b1c5..948e5bae52 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -521,13 +521,13 @@ mod tests { assert_that!( map.as_view().iter().collect::>(), unordered_elements_are![ - eq((3, ProtoStr::from_str("fizz"))), - eq((5, ProtoStr::from_str("buzz"))), - eq((15, ProtoStr::from_str("fizzbuzz"))) + eq(&(3, ProtoStr::from_str("fizz"))), + eq(&(5, ProtoStr::from_str("buzz"))), + eq(&(15, ProtoStr::from_str("fizzbuzz"))) ] ); assert_that!( - map.as_view().into_iter().collect::>(), + map.as_view(), unordered_elements_are![ eq((3, ProtoStr::from_str("fizz"))), eq((5, ProtoStr::from_str("buzz"))), @@ -535,7 +535,15 @@ mod tests { ] ); assert_that!( - map.as_view().into_iter().collect::>(), + map.as_mut().iter().collect::>(), + unordered_elements_are![ + eq(&(3, ProtoStr::from_str("fizz"))), + eq(&(5, ProtoStr::from_str("buzz"))), + eq(&(15, ProtoStr::from_str("fizzbuzz"))) + ] + ); + assert_that!( + map.as_mut(), unordered_elements_are![ eq((3, ProtoStr::from_str("fizz"))), eq((5, ProtoStr::from_str("buzz"))), @@ -551,10 +559,7 @@ mod tests { assert!(map_mut.insert(0, "fizz")); // insert should return false when the key is already present assert!(!map_mut.insert(0, "buzz")); - assert_that!( - map.as_mut().iter().collect::>(), - unordered_elements_are![eq((0, ProtoStr::from_str("buzz"))),] - ); + assert_that!(map.as_mut(), unordered_elements_are![eq((0, ProtoStr::from_str("buzz"))),]); } #[test] @@ -568,7 +573,7 @@ mod tests { map_mut.extend([(0, "fizz"), (1, "buzz"), (2, "fizzbuzz")]); assert_that!( - map.as_view().into_iter().collect::>(), + map.as_view(), unordered_elements_are![ eq((0, ProtoStr::from_str("fizz"))), eq((1, ProtoStr::from_str("buzz"))), @@ -584,7 +589,7 @@ mod tests { map_mut.extend(&map_2); assert_that!( - map.as_view().into_iter().collect::>(), + map.as_view(), unordered_elements_are![ eq((0, ProtoStr::from_str("fizz"))), eq((1, ProtoStr::from_str("buzz"))), @@ -601,7 +606,7 @@ mod tests { map_mut.copy_from([(0, "fizz"), (1, "buzz"), (2, "fizzbuzz")]); assert_that!( - map.as_view().into_iter().collect::>(), + map.as_view(), unordered_elements_are![ eq((0, ProtoStr::from_str("fizz"))), eq((1, ProtoStr::from_str("buzz"))), @@ -617,7 +622,7 @@ mod tests { map_mut.copy_from(&map_2); assert_that!( - map.as_view().into_iter().collect::>(), + map.as_view(), unordered_elements_are![ eq((2, ProtoStr::from_str("bing"))), eq((3, ProtoStr::from_str("bong"))) diff --git a/rust/repeated.rs b/rust/repeated.rs index edc59472dc..617f1f3a1e 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -479,8 +479,7 @@ mod tests { assert_that!(r.len(), eq(expected_len)); )* - assert_that!( - r.iter().collect::>(), elements_are![$(eq($vals)),*]); + assert_that!(r, elements_are![$(eq($vals)),*]); r.set(0, <$t as Default>::default()); assert_that!(r.get(0).expect("elem 0"), eq(<$t as Default>::default())); @@ -507,15 +506,12 @@ mod tests { assert_that!(r.as_mut().len(), eq(0)); r.as_mut().extend([0, 1]); - assert_that!(r.as_mut().iter().collect::>(), elements_are![eq(0), eq(1)]); + assert_that!(r.as_mut(), elements_are![eq(0), eq(1)]); let mut x = Repeated::::new(); x.as_mut().extend([2, 3]); r.as_mut().extend(&x.as_mut()); - assert_that!( - r.as_mut().iter().collect::>(), - elements_are![eq(0), eq(1), eq(2), eq(3)] - ); + assert_that!(r.as_mut(), elements_are![eq(0), eq(1), eq(2), eq(3)]); } } diff --git a/rust/test/shared/accessors_map_test.rs b/rust/test/shared/accessors_map_test.rs index 50341b0c3a..3120bd8f42 100644 --- a/rust/test/shared/accessors_map_test.rs +++ b/rust/test/shared/accessors_map_test.rs @@ -24,7 +24,7 @@ macro_rules! generate_map_primitives_tests { let mut msg = TestMap::new(); assert_that!(msg.[< map_ $k_field _ $v_field >]().len(), eq(0)); assert_that!( - msg.[< map_ $k_field _ $v_field >]().iter().collect::>(), + msg.[< map_ $k_field _ $v_field >](), elements_are![] ); assert_that!( @@ -41,24 +41,24 @@ macro_rules! generate_map_primitives_tests { assert_that!(msg.[< map_ $k_field _ $v_field _mut>]().insert(k, v), eq(false)); assert_that!(msg.[< map_ $k_field _ $v_field >]().len(), eq(1)); assert_that!( - msg.[< map_ $k_field _ $v_field >]().iter().collect::>(), + msg.[< map_ $k_field _ $v_field >](), elements_are![eq((k, v))] ); assert_that!( msg.[< map_ $k_field _ $v_field >]().keys().collect::>(), - elements_are![eq(k)] + elements_are![eq(&k)] ); assert_that!( msg.[< map_ $k_field _ $v_field >]().values().collect::>(), - elements_are![eq(v)] + elements_are![eq(&v)] ); let k2: $k_type = $k_nonzero; let v2: $v_type = $v_nonzero; assert_that!(msg.[< map_ $k_field _ $v_field _mut>]().insert(k2, v2), eq(true)); - assert_that!(msg.[< map_ $k_field _ $v_field >]().len(), eq(2)); + assert_that!(msg.[< map_ $k_field _ $v_field >](), len(eq(2))); assert_that!( - msg.[< map_ $k_field _ $v_field >]().iter().collect::>(), + msg.[< map_ $k_field _ $v_field >](), unordered_elements_are![ eq((k, v)), eq((k2, v2)), @@ -66,11 +66,11 @@ macro_rules! generate_map_primitives_tests { ); assert_that!( msg.[< map_ $k_field _ $v_field >]().keys().collect::>(), - unordered_elements_are![eq(k), eq(k2)] + unordered_elements_are![eq(&k), eq(&k2)] ); assert_that!( msg.[< map_ $k_field _ $v_field >]().values().collect::>(), - unordered_elements_are![eq(v), eq(v2)] + unordered_elements_are![eq(&v), eq(&v2)] ); } )* } @@ -105,11 +105,11 @@ fn collect_as_hashmap() { let hashmap: HashMap = msg.map_string_string().iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); assert_that!( - hashmap.into_iter().collect::>(), + hashmap, unordered_elements_are![ - eq(("hello".to_owned(), "world".to_owned())), - eq(("fizz".to_owned(), "buzz".to_owned())), - eq(("boo".to_owned(), "blah".to_owned())), + (eq("hello"), eq("world")), + (eq("fizz"), eq("buzz")), + (eq("boo"), eq("blah")), ] ); } @@ -150,10 +150,7 @@ fn test_bytes_and_string_copied() { } assert_that!(msg.map_string_string_mut().get("hello").unwrap(), eq("world")); - assert_that!( - msg.map_string_string().iter().collect::>(), - unordered_elements_are![eq(("hello".into(), "world".into()))] - ); + assert_that!(msg.map_string_string(), unordered_elements_are![(eq("hello"), eq("world"))]); assert_that!(msg.map_int32_bytes_mut().get(1).unwrap(), eq(b"world")); } @@ -166,7 +163,7 @@ fn test_map_setter() { let mut msg2 = TestMap::new(); msg2.set_map_string_string(msg.map_string_string()); assert_that!( - msg2.map_string_string().iter().collect::>(), + msg2.map_string_string(), unordered_elements_are![ eq(("hello".into(), "world".into())), eq(("fizz".into(), "buzz".into())) @@ -242,7 +239,7 @@ macro_rules! generate_map_with_msg_values_tests { msg.[< map_ $k_field _all_types_mut >]().remove($k_nonzero), eq(true), "`remove` should return true when key was present."); - assert_that!(msg.[< map_ $k_field _all_types >]().len(), eq(0)); + assert_that!(msg.[< map_ $k_field _all_types >](), empty()); assert_that!( msg.[< map_ $k_field _all_types_mut >]().remove($k_nonzero), eq(false), @@ -255,13 +252,13 @@ macro_rules! generate_map_with_msg_values_tests { // "`iter` should work when empty." // ); assert_that!( - msg.[< map_ $k_field _all_types_mut >]().keys().collect::>(), - elements_are![], + msg.[< map_ $k_field _all_types_mut >]().keys().count(), + eq(0), "`iter` should work when empty." ); assert_that!( - msg.[< map_ $k_field _all_types_mut >]().values().collect::>(), - elements_are![], + msg.[< map_ $k_field _all_types_mut >]().values().count(), + eq(0), "`iter` should work when empty." ); @@ -277,10 +274,10 @@ macro_rules! generate_map_with_msg_values_tests { // ); assert_that!( msg.[< map_ $k_field _all_types >]().keys().collect::>(), - unordered_elements_are![eq($k_nonzero)] + unordered_elements_are![eq(&$k_nonzero)] ); assert_that!( - msg.[< map_ $k_field _all_types >]().values().collect::>().len(), + msg.[< map_ $k_field _all_types >]().values().count(), eq(1)); @@ -292,15 +289,15 @@ macro_rules! generate_map_with_msg_values_tests { eq(true)); assert_that!( - msg.[< map_ $k_field _all_types >]().iter().collect::>().len(), - eq(2) + msg.[< map_ $k_field _all_types >](), + len(eq(2)) ); assert_that!( msg.[< map_ $k_field _all_types >]().keys().collect::>(), - unordered_elements_are![eq($k_nonzero), eq($k_other)] + unordered_elements_are![eq(&$k_nonzero), eq(&$k_other)] ); assert_that!( - msg.[< map_ $k_field _all_types >]().values().collect::>().len(), + msg.[< map_ $k_field _all_types >]().values().count(), eq(2) ); } diff --git a/rust/test/shared/accessors_repeated_test.rs b/rust/test/shared/accessors_repeated_test.rs index 99ae45374c..4c5356a86a 100644 --- a/rust/test/shared/accessors_repeated_test.rs +++ b/rust/test/shared/accessors_repeated_test.rs @@ -35,20 +35,20 @@ macro_rules! generate_repeated_numeric_test { mutator.set(2, 0 as $t); assert_that!( - mutator.iter().collect::>(), - elements_are![eq(2 as $t), eq(1 as $t), eq(0 as $t)] + mutator, + elements_are![eq(2 as $t), eq(1 as $t), eq(0 as $t)] ); assert_that!( - mutator.as_view().into_iter().collect::>(), - elements_are![eq(2 as $t), eq(1 as $t), eq(0 as $t)] + mutator.as_view(), + elements_are![eq(2 as $t), eq(1 as $t), eq(0 as $t)] ); for i in 0..mutator.len() { mutator.set(i, 0 as $t); } assert_that!( - msg.[]().iter().collect::>(), - each(eq(0 as $t)) + msg.[](), + each(eq(0 as $t)) ); } @@ -66,7 +66,7 @@ macro_rules! generate_repeated_numeric_test { let view = msg.[](); assert_that!( view.iter().collect::>(), - eq(mutator2.iter().collect::>()) + eq(&mutator2.iter().collect::>()) ); } @@ -125,16 +125,13 @@ fn test_repeated_bool_accessors() { mutator.set(2, false); assert_that!(mutator.get(2), some(eq(false))); - assert_that!(mutator.iter().collect::>(), elements_are![eq(false), eq(true), eq(false)]); - assert_that!( - mutator.as_view().into_iter().collect::>(), - elements_are![eq(false), eq(true), eq(false)] - ); + assert_that!(mutator, elements_are![eq(false), eq(true), eq(false)]); + assert_that!(mutator.as_view(), elements_are![eq(false), eq(true), eq(false)]); for i in 0..mutator.len() { mutator.set(i, false); } - assert_that!(msg.repeated_bool().iter().collect::>(), each(eq(false))); + assert_that!(msg.repeated_bool(), each(eq(false))); } #[test] @@ -161,18 +158,18 @@ fn test_repeated_enum_accessors() { assert_that!(mutator.get(2), some(eq(NestedEnum::Foo))); assert_that!( - mutator.iter().collect::>(), + mutator, elements_are![eq(NestedEnum::Bar), eq(NestedEnum::Baz), eq(NestedEnum::Foo)] ); assert_that!( - mutator.as_view().into_iter().collect::>(), + mutator.as_view(), elements_are![eq(NestedEnum::Bar), eq(NestedEnum::Baz), eq(NestedEnum::Foo)] ); for i in 0..mutator.len() { mutator.set(i, NestedEnum::Foo); } - assert_that!(msg.repeated_nested_enum().iter().collect::>(), each(eq(NestedEnum::Foo))); + assert_that!(msg.repeated_nested_enum(), each(eq(NestedEnum::Foo))); } #[test] @@ -187,7 +184,7 @@ fn test_repeated_bool_set() { msg.set_repeated_bool(mutator2.as_view()); let view = msg.repeated_bool(); - assert_that!(view.iter().collect::>(), eq(mutator2.iter().collect::>())); + assert_that!(&view.iter().collect::>(), eq(&mutator2.iter().collect::>())); } #[test] @@ -211,8 +208,8 @@ fn test_repeated_message() { assert_that!(msg.repeated_nested_message().get(0).unwrap().bb(), eq(2)); assert_that!( - msg.repeated_nested_message().iter().map(|m| m.bb()).collect::>(), - eq(vec![2]), + msg.repeated_nested_message(), + elements_are![predicate(|m: protobuf::View| m.bb() == 2)], ); drop(msg); @@ -243,7 +240,7 @@ fn test_repeated_strings() { assert_that!(msg.repeated_string().get(0).unwrap(), eq("set from Mut")); assert_that!(msg.repeated_string().get(1).unwrap(), eq("set second str")); assert_that!( - msg.repeated_string().iter().collect::>(), + msg.repeated_string(), elements_are![eq("set from Mut"), eq("set second str")] ); older_msg.repeated_string_mut().copy_from(msg.repeated_string()); @@ -251,7 +248,7 @@ fn test_repeated_strings() { assert_that!(older_msg.repeated_string().len(), eq(2)); assert_that!( - older_msg.repeated_string().iter().collect::>(), + older_msg.repeated_string(), elements_are![eq("set from Mut"), eq("set second str")] ); diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index 36caf8b00c..7a170364e0 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -16,7 +16,7 @@ fn test_default_accessors() { let msg: TestAllTypes = Default::default(); assert_that!( msg, - matches_pattern!(TestAllTypes{ + matches_pattern!(&TestAllTypes{ default_int32(): eq(41), default_int64(): eq(42), default_uint32(): eq(43), diff --git a/rust/test/shared/enum_test.rs b/rust/test/shared/enum_test.rs index 7181e59d47..1c8e7fda84 100644 --- a/rust/test/shared/enum_test.rs +++ b/rust/test/shared/enum_test.rs @@ -61,12 +61,12 @@ fn test_closed_enum_is_nonexhaustive() { #[test] fn test_closed_enum_conversion() { assert_that!(i32::from(TestSparseEnum::SparseA), eq(123)); - assert_that!(TestSparseEnum::try_from(123), ok(eq(TestSparseEnum::SparseA))); + assert_that!(TestSparseEnum::try_from(123), ok(eq(&TestSparseEnum::SparseA))); assert_that!(i32::from(TestSparseEnum::SparseD), eq(-15)); - assert_that!(TestSparseEnum::try_from(-15), ok(eq(TestSparseEnum::SparseD))); + assert_that!(TestSparseEnum::try_from(-15), ok(eq(&TestSparseEnum::SparseD))); - assert_that!(TestSparseEnum::try_from(0), ok(eq(TestSparseEnum::SparseF))); + assert_that!(TestSparseEnum::try_from(0), ok(eq(&TestSparseEnum::SparseF))); assert_that!(TestSparseEnum::try_from(1), err(anything())); } @@ -78,9 +78,9 @@ fn test_closed_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDupValue::Bar2), eq(2)); assert_that!(i32::from(TestEnumWithDupValue::Baz), eq(3)); - assert_that!(TestEnumWithDupValue::try_from(1), ok(eq(TestEnumWithDupValue::Foo1))); - assert_that!(TestEnumWithDupValue::try_from(2), ok(eq(TestEnumWithDupValue::Bar1))); - assert_that!(TestEnumWithDupValue::try_from(3), ok(eq(TestEnumWithDupValue::Baz))); + assert_that!(TestEnumWithDupValue::try_from(1), ok(eq(&TestEnumWithDupValue::Foo1))); + assert_that!(TestEnumWithDupValue::try_from(2), ok(eq(&TestEnumWithDupValue::Bar1))); + assert_that!(TestEnumWithDupValue::try_from(3), ok(eq(&TestEnumWithDupValue::Baz))); assert_that!(TestEnumWithDupValue::try_from(0), err(anything())); assert_that!(TestEnumWithDupValue::try_from(4), err(anything())); From 9ae3d81052a11abc4f164ec488250ca18f1f5640 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 May 2024 10:35:45 -0700 Subject: [PATCH 097/179] Internal change. PiperOrigin-RevId: 630431165 --- src/google/protobuf/text_format.cc | 12 +++--------- src/google/protobuf/text_format.h | 1 - 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index dd41e5600f..71b7ed6d03 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -94,9 +94,6 @@ namespace internal { const char kDebugStringSilentMarker[] = ""; const char kDebugStringSilentMarkerForDetection[] = "\t "; -// Controls insertion of kDebugStringSilentMarker into DebugString() output. -PROTOBUF_EXPORT std::atomic enable_debug_text_format_marker; - // Controls insertion of a marker making debug strings non-parseable, and // redacting annotated fields in Protobuf's DebugString APIs. PROTOBUF_EXPORT std::atomic enable_debug_string_safe_format{false}; @@ -158,8 +155,7 @@ std::string Message::DebugString() const { TextFormat::Printer printer; printer.SetExpandAny(true); - printer.SetInsertSilentMarker(internal::enable_debug_text_format_marker.load( - std::memory_order_relaxed)); + printer.SetInsertSilentMarker(true); printer.SetReportSensitiveFields(FieldReporterLevel::kDebugString); printer.PrintToString(*this, &debug_string); @@ -181,8 +177,7 @@ std::string Message::ShortDebugString() const { TextFormat::Printer printer; printer.SetSingleLineMode(true); printer.SetExpandAny(true); - printer.SetInsertSilentMarker(internal::enable_debug_text_format_marker.load( - std::memory_order_relaxed)); + printer.SetInsertSilentMarker(true); printer.SetReportSensitiveFields(FieldReporterLevel::kShortDebugString); printer.PrintToString(*this, &debug_string); @@ -205,8 +200,7 @@ std::string Message::Utf8DebugString() const { TextFormat::Printer printer; printer.SetUseUtf8StringEscaping(true); printer.SetExpandAny(true); - printer.SetInsertSilentMarker(internal::enable_debug_text_format_marker.load( - std::memory_order_relaxed)); + printer.SetInsertSilentMarker(true); printer.SetReportSensitiveFields(FieldReporterLevel::kUtf8DebugString); printer.PrintToString(*this, &debug_string); diff --git a/src/google/protobuf/text_format.h b/src/google/protobuf/text_format.h index ba9210f344..da891beb97 100644 --- a/src/google/protobuf/text_format.h +++ b/src/google/protobuf/text_format.h @@ -44,7 +44,6 @@ namespace internal { PROTOBUF_EXPORT extern const char kDebugStringSilentMarker[1]; PROTOBUF_EXPORT extern const char kDebugStringSilentMarkerForDetection[3]; -PROTOBUF_EXPORT extern std::atomic enable_debug_text_format_marker; PROTOBUF_EXPORT extern std::atomic enable_debug_string_safe_format; PROTOBUF_EXPORT int64_t GetRedactedFieldCount(); PROTOBUF_EXPORT bool ShouldRedactField(const FieldDescriptor* field); From a91d76bb8a0d07c479f1dc8dd3e032c632755149 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 May 2024 11:30:44 -0700 Subject: [PATCH 098/179] Internal changes to extension declarations. PiperOrigin-RevId: 630448683 --- src/google/protobuf/compiler/command_line_interface.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h index 43ab8eee1d..86af621e37 100644 --- a/src/google/protobuf/compiler/command_line_interface.h +++ b/src/google/protobuf/compiler/command_line_interface.h @@ -470,6 +470,7 @@ class PROTOC_EXPORT CommandLineInterface { bool deterministic_output_ = false; bool opensource_runtime_ = PROTO2_IS_OSS; + }; } // namespace compiler From aa70a3576b579d5481f9bb996b86ee6bbb8847b4 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Fri, 3 May 2024 12:33:45 -0700 Subject: [PATCH 099/179] upb: clean up message/accessors.h and message/internal/accessors.h PiperOrigin-RevId: 630466865 --- upb/message/accessors.h | 410 ++++++------------------------- upb/message/copy.c | 8 +- upb/message/internal/accessors.h | 392 +++++++++++++++++++++++++++-- 3 files changed, 451 insertions(+), 359 deletions(-) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index f812e2da47..c2af8df6e9 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -68,263 +68,100 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} +UPB_API_INLINE upb_MessageValue +upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, + upb_MessageValue default_val); -// NOTE: The default_val is only used for fields that support presence. -// For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a -// upb_Array/upb_Map has not been allocated yet. Array/map fields do not have -// presence, so this is semantically identical to a pointer to an empty -// array/map, and must be treated the same for all semantic purposes. -UPB_INLINE upb_MessageValue -upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* field, - upb_MessageValue default_val) { - upb_MessageValue ret; - if (upb_MiniTableField_IsExtension(field)) { - _upb_Message_GetExtensionField(msg, (upb_MiniTableExtension*)field, - &default_val, &ret); - } else { - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - } - return ret; -} - -// Sets the value of the given field in the given msg. The return value is true -// if the operation completed successfully, or false if memory allocation -// failed. -UPB_INLINE bool UPB_PRIVATE(_upb_Message_SetField)( - upb_Message* msg, const upb_MiniTableField* field, upb_MessageValue val, - upb_Arena* a) { - if (upb_MiniTableField_IsExtension(field)) { - const upb_MiniTableExtension* ext = (const upb_MiniTableExtension*)field; - return upb_Message_SetExtension(msg, ext, &val, a); - } else { - upb_Message_SetBaseField(msg, field, &val); - return true; - } -} +UPB_API_INLINE const upb_Array* upb_Message_GetArray( + const upb_Message* msg, const upb_MiniTableField* f); UPB_API_INLINE bool upb_Message_GetBool(const upb_Message* msg, - const upb_MiniTableField* field, - bool default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_1Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue def; - def.bool_val = default_val; - return upb_Message_GetField(msg, field, def).bool_val; -} + const upb_MiniTableField* f, + bool default_val); -UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, - const upb_MiniTableField* field, - bool value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_1Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.bool_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE double upb_Message_GetDouble(const upb_Message* msg, + const upb_MiniTableField* field, + double default_val); + +UPB_API_INLINE float upb_Message_GetFloat(const upb_Message* msg, + const upb_MiniTableField* f, + float default_val); UPB_API_INLINE int32_t upb_Message_GetInt32(const upb_Message* msg, - const upb_MiniTableField* field, - int32_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || - upb_MiniTableField_CType(field) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); + const upb_MiniTableField* f, + int32_t default_val); - upb_MessageValue def; - def.int32_val = default_val; - return upb_Message_GetField(msg, field, def).int32_val; -} +UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, + const upb_MiniTableField* f, + int64_t default_val); -UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, - const upb_MiniTableField* field, - int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || - upb_MiniTableField_CType(field) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.int32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE const upb_Map* upb_Message_GetMap(const upb_Message* msg, + const upb_MiniTableField* f); -UPB_API_INLINE uint32_t upb_Message_GetUInt32(const upb_Message* msg, - const upb_MiniTableField* field, - uint32_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( + upb_Message* msg, const upb_MiniTableField* f); - upb_MessageValue def; - def.uint32_val = default_val; - return upb_Message_GetField(msg, field, def).uint32_val; -} +UPB_API_INLINE upb_Map* upb_Message_GetMutableMap(upb_Message* msg, + const upb_MiniTableField* f); -UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, - const upb_MiniTableField* field, - uint32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.uint32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( + upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena); -UPB_API_INLINE void upb_Message_SetClosedEnum( - upb_Message* msg, const upb_MiniTable* msg_mini_table, - const upb_MiniTableField* field, int32_t value) { - UPB_ASSERT(upb_MiniTableField_IsClosedEnum(field)); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - UPB_ASSERT(upb_MiniTableEnum_CheckValue( - upb_MiniTable_GetSubEnumTable(msg_mini_table, field), value)); - upb_Message_SetBaseField(msg, field, &value); -} +UPB_API_INLINE upb_Map* upb_Message_GetOrCreateMutableMap( + upb_Message* msg, const upb_MiniTable* map_entry_mini_table, + const upb_MiniTableField* f, upb_Arena* arena); -UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, - const upb_MiniTableField* field, - int64_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( + upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_Arena* arena); - upb_MessageValue def; - def.int64_val = default_val; - return upb_Message_GetField(msg, field, def).int64_val; -} +UPB_API_INLINE upb_StringView +upb_Message_GetString(const upb_Message* msg, const upb_MiniTableField* field, + upb_StringView default_val); -UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, - const upb_MiniTableField* field, - int64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.int64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE uint32_t upb_Message_GetUInt32(const upb_Message* msg, + const upb_MiniTableField* f, + uint32_t default_val); UPB_API_INLINE uint64_t upb_Message_GetUInt64(const upb_Message* msg, - const upb_MiniTableField* field, - uint64_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - - upb_MessageValue def; - def.uint64_val = default_val; - return upb_Message_GetField(msg, field, def).uint64_val; -} + const upb_MiniTableField* f, + uint64_t default_val); -UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, - const upb_MiniTableField* field, - uint64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.uint64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a); -UPB_API_INLINE float upb_Message_GetFloat(const upb_Message* msg, - const upb_MiniTableField* field, - float default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE void upb_Message_SetClosedEnum( + upb_Message* msg, const upb_MiniTable* msg_mini_table, + const upb_MiniTableField* f, int32_t value); - upb_MessageValue def; - def.float_val = default_val; - return upb_Message_GetField(msg, field, def).float_val; -} +UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, + const upb_MiniTableField* f, + double value, upb_Arena* a); UPB_API_INLINE bool upb_Message_SetFloat(upb_Message* msg, - const upb_MiniTableField* field, - float value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.float_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} - -UPB_API_INLINE double upb_Message_GetDouble(const upb_Message* msg, - const upb_MiniTableField* field, - double default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); + const upb_MiniTableField* f, + float value, upb_Arena* a); - upb_MessageValue def; - def.double_val = default_val; - return upb_Message_GetField(msg, field, def).double_val; -} +UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, + const upb_MiniTableField* f, + int32_t value, upb_Arena* a); -UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, - const upb_MiniTableField* field, - double value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.double_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, + const upb_MiniTableField* f, + int64_t value, upb_Arena* a); -UPB_API_INLINE upb_StringView -upb_Message_GetString(const upb_Message* msg, const upb_MiniTableField* field, - upb_StringView default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_String || - upb_MiniTableField_CType(field) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value, upb_Arena* a); - upb_MessageValue def; - def.str_val = default_val; - return upb_Message_GetField(msg, field, def).str_val; -} +UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value, upb_Arena* a); -// Sets the value of a `string` or `bytes` field. The bytes of the value are not -// copied, so it is the caller's responsibility to ensure that they remain valid -// for the lifetime of `msg`. That might be done by copying them into the given -// arena, or by fusing that arena with the arena the bytes live in, for example. -UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, - const upb_MiniTableField* field, - upb_StringView value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_String || - upb_MiniTableField_CType(field) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.str_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value, upb_Arena* a); UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( const upb_Message* msg, const upb_MiniTableField* field, @@ -353,14 +190,14 @@ UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( // For internal use only; users cannot set tagged messages because only the // parser and the message copier are allowed to directly create an empty // message. -UPB_API_INLINE void _upb_Message_SetTaggedMessagePtr( - upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* field, upb_TaggedMessagePtr sub_message) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == +UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( + struct upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_TaggedMessagePtr sub_message) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_Message_SetBaseField(msg, field, &sub_message); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_Message_SetBaseField(msg, f, &sub_message); } // Sets the value of a message-typed field. The `mini_table` and `field` @@ -370,102 +207,17 @@ UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, const upb_MiniTable* mini_table, const upb_MiniTableField* field, upb_Message* sub_message) { - _upb_Message_SetTaggedMessagePtr( - msg, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); -} - -UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( - upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSERT(arena); - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(!upb_MiniTableField_IsExtension(field)); - upb_Message* sub_message = - *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); - if (!sub_message) { - const upb_MiniTable* sub_mini_table = - upb_MiniTable_SubMessage(mini_table, field); - UPB_ASSERT(sub_mini_table); - sub_message = _upb_Message_New(sub_mini_table, arena); - *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*) = sub_message; - UPB_PRIVATE(_upb_Message_SetPresence)(msg, field); - } - return sub_message; -} - -UPB_API_INLINE const upb_Array* upb_Message_GetArray( - const upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* ret; - const upb_Array* default_val = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - return ret; -} - -UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( - upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - return (upb_Array*)upb_Message_GetArray(msg, field); -} - -UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( - upb_Message* msg, const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSERT(arena); - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* array = upb_Message_GetMutableArray(msg, field); - if (!array) { - array = UPB_PRIVATE(_upb_Array_New)( - arena, 4, UPB_PRIVATE(_upb_MiniTableField_ElemSizeLg2)(field)); - // Check again due to: https://godbolt.org/z/7WfaoKG1r - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_MessageValue val; - val.array_val = array; - UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, arena); - } - return array; + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (msg, mini_table, field, + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); } UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( - upb_Message* msg, const upb_MiniTableField* field, size_t size, - upb_Arena* arena) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* arr = upb_Message_GetOrCreateMutableArray(msg, field, arena); - if (!arr || !UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, size, arena)) { - return NULL; - } - return upb_Array_MutableDataPtr(arr); -} - -UPB_API_INLINE const upb_Map* upb_Message_GetMap( - const upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - _upb_Message_AssertMapIsUntagged(msg, field); - upb_Map* ret; - const upb_Map* default_val = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - return ret; -} + upb_Message* msg, const upb_MiniTableField* f, size_t size, + upb_Arena* arena); -UPB_API_INLINE upb_Map* upb_Message_GetMutableMap( - upb_Message* msg, const upb_MiniTableField* field) { - return (upb_Map*)upb_Message_GetMap(msg, field); -} - -UPB_API_INLINE upb_Map* upb_Message_GetOrCreateMutableMap( - upb_Message* msg, const upb_MiniTable* map_entry_mini_table, - const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - const upb_MiniTableField* map_entry_key_field = - &map_entry_mini_table->UPB_ONLYBITS(fields)[0]; - const upb_MiniTableField* map_entry_value_field = - &map_entry_mini_table->UPB_ONLYBITS(fields)[1]; - return _upb_Message_GetOrCreateMutableMap( - msg, field, - _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_key_field)), - _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_value_field)), - arena); -} +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const upb_Message* message, const upb_MiniTableField* oneof_field); // Updates a map entry given an entry message. bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* mini_table, diff --git a/upb/message/copy.c b/upb/message/copy.c index b3d6ef8624..dca2107779 100644 --- a/upb/message/copy.c +++ b/upb/message/copy.c @@ -214,10 +214,10 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, if (dst_sub_message == NULL) { return NULL; } - _upb_Message_SetTaggedMessagePtr( - dst, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, - is_empty)); + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (dst, mini_table, field, + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, + is_empty)); } } break; case kUpb_CType_String: diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index bfae12e601..a6c072278a 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -12,14 +12,18 @@ #include #include +#include "upb/base/descriptor_constants.h" #include "upb/base/internal/endian.h" #include "upb/base/string_view.h" #include "upb/mem/arena.h" +#include "upb/message/internal/array.h" #include "upb/message/internal/extension.h" #include "upb/message/internal/map.h" #include "upb/message/internal/message.h" #include "upb/message/internal/tagged_ptr.h" #include "upb/message/internal/types.h" +#include "upb/message/value.h" +#include "upb/mini_table/enum.h" #include "upb/mini_table/extension.h" #include "upb/mini_table/field.h" #include "upb/mini_table/internal/field.h" @@ -112,6 +116,12 @@ UPB_INLINE bool UPB_PRIVATE(_upb_Message_ClearOneofCase)( return true; } +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const struct upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} + // LINT.ThenChange(GoogleInternalName2) // Returns false if the message is missing any of its required fields. @@ -271,6 +281,24 @@ UPB_INLINE void _upb_Message_GetExtensionField( } } +// NOTE: The default_val is only used for fields that support presence. +// For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a +// upb_Array/upb_Map has not been allocated yet. Array/map fields do not have +// presence, so this is semantically identical to a pointer to an empty +// array/map, and must be treated the same for all semantic purposes. +UPB_API_INLINE upb_MessageValue upb_Message_GetField( + const struct upb_Message* msg, const upb_MiniTableField* field, + upb_MessageValue default_val) { + upb_MessageValue ret; + if (upb_MiniTableField_IsExtension(field)) { + _upb_Message_GetExtensionField(msg, (upb_MiniTableExtension*)field, + &default_val, &ret); + } else { + _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); + } + return ret; +} + UPB_API_INLINE void upb_Message_SetBaseField(struct upb_Message* msg, const upb_MiniTableField* f, const void* val) { @@ -294,6 +322,336 @@ UPB_API_INLINE bool upb_Message_SetExtension(struct upb_Message* msg, return true; } +// Sets the value of the given field in the given msg. The return value is true +// if the operation completed successfully, or false if memory allocation +// failed. +UPB_INLINE bool UPB_PRIVATE(_upb_Message_SetField)(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_MessageValue val, + upb_Arena* a) { + if (upb_MiniTableField_IsExtension(f)) { + const upb_MiniTableExtension* ext = (const upb_MiniTableExtension*)f; + return upb_Message_SetExtension(msg, ext, &val, a); + } else { + upb_Message_SetBaseField(msg, f, &val); + return true; + } +} + +UPB_API_INLINE const upb_Array* upb_Message_GetArray( + const struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* ret; + const upb_Array* default_val = NULL; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &ret); + return ret; +} + +UPB_API_INLINE bool upb_Message_GetBool(const struct upb_Message* msg, + const upb_MiniTableField* f, + bool default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue def; + def.bool_val = default_val; + return upb_Message_GetField(msg, f, def).bool_val; +} + +UPB_API_INLINE double upb_Message_GetDouble(const struct upb_Message* msg, + const upb_MiniTableField* f, + double default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.double_val = default_val; + return upb_Message_GetField(msg, f, def).double_val; +} + +UPB_API_INLINE float upb_Message_GetFloat(const struct upb_Message* msg, + const upb_MiniTableField* f, + float default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.float_val = default_val; + return upb_Message_GetField(msg, f, def).float_val; +} + +UPB_API_INLINE int32_t upb_Message_GetInt32(const struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.int32_val = default_val; + return upb_Message_GetField(msg, f, def).int32_val; +} + +UPB_API_INLINE int64_t upb_Message_GetInt64(const struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.int64_val = default_val; + return upb_Message_GetField(msg, f, def).int64_val; +} + +UPB_INLINE void UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)( + const struct upb_Message* msg, const upb_MiniTableField* field) { + UPB_UNUSED(msg); + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); +#ifndef NDEBUG + uintptr_t default_val = 0; + uintptr_t tagged; + _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); + UPB_ASSERT(!upb_TaggedMessagePtr_IsEmpty(tagged)); +#endif +} + +UPB_API_INLINE const struct upb_Map* upb_Message_GetMap( + const struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(f); + UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)(msg, f); + struct upb_Map* ret; + const struct upb_Map* default_val = NULL; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &ret); + return ret; +} + +UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( + struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + return (upb_Array*)upb_Message_GetArray(msg, f); +} + +UPB_API_INLINE struct upb_Map* upb_Message_GetMutableMap( + struct upb_Message* msg, const upb_MiniTableField* f) { + return (struct upb_Map*)upb_Message_GetMap(msg, f); +} + +UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( + struct upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSERT(arena); + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* array = upb_Message_GetMutableArray(msg, f); + if (!array) { + array = UPB_PRIVATE(_upb_Array_New)( + arena, 4, UPB_PRIVATE(_upb_MiniTableField_ElemSizeLg2)(f)); + // Check again due to: https://godbolt.org/z/7WfaoKG1r + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_MessageValue val; + val.array_val = array; + UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, arena); + } + return array; +} + +UPB_INLINE struct upb_Map* _upb_Message_GetOrCreateMutableMap( + struct upb_Message* msg, const upb_MiniTableField* field, size_t key_size, + size_t val_size, upb_Arena* arena) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); + UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)(msg, field); + struct upb_Map* map = NULL; + struct upb_Map* default_map_value = NULL; + _upb_Message_GetNonExtensionField(msg, field, &default_map_value, &map); + if (!map) { + map = _upb_Map_New(arena, key_size, val_size); + // Check again due to: https://godbolt.org/z/7WfaoKG1r + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); + upb_Message_SetBaseField(msg, field, &map); + } + return map; +} + +UPB_API_INLINE struct upb_Map* upb_Message_GetOrCreateMutableMap( + struct upb_Message* msg, const upb_MiniTable* map_entry_mini_table, + const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + const upb_MiniTableField* map_entry_key_field = + &map_entry_mini_table->UPB_ONLYBITS(fields)[0]; + const upb_MiniTableField* map_entry_value_field = + &map_entry_mini_table->UPB_ONLYBITS(fields)[1]; + return _upb_Message_GetOrCreateMutableMap( + msg, f, _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_key_field)), + _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_value_field)), + arena); +} + +UPB_API_INLINE struct upb_Message* upb_Message_GetOrCreateMutableMessage( + struct upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSERT(arena); + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(!upb_MiniTableField_IsExtension(f)); + struct upb_Message* sub_message = + *UPB_PTR_AT(msg, f->UPB_ONLYBITS(offset), struct upb_Message*); + if (!sub_message) { + const upb_MiniTable* sub_mini_table = + upb_MiniTable_SubMessage(mini_table, f); + UPB_ASSERT(sub_mini_table); + sub_message = _upb_Message_New(sub_mini_table, arena); + *UPB_PTR_AT(msg, f->UPB_ONLYBITS(offset), struct upb_Message*) = + sub_message; + UPB_PRIVATE(_upb_Message_SetPresence)(msg, f); + } + return sub_message; +} + +UPB_API_INLINE upb_StringView +upb_Message_GetString(const struct upb_Message* msg, + const upb_MiniTableField* f, upb_StringView default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.str_val = default_val; + return upb_Message_GetField(msg, f, def).str_val; +} + +UPB_API_INLINE uint32_t upb_Message_GetUInt32(const struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.uint32_val = default_val; + return upb_Message_GetField(msg, f, def).uint32_val; +} + +UPB_API_INLINE uint64_t upb_Message_GetUInt64(const struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.uint64_val = default_val; + return upb_Message_GetField(msg, f, def).uint64_val; +} + +UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.bool_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE void upb_Message_SetClosedEnum( + struct upb_Message* msg, const upb_MiniTable* msg_mini_table, + const upb_MiniTableField* f, int32_t value) { + UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSERT(upb_MiniTableEnum_CheckValue( + upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE bool upb_Message_SetDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.double_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.float_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.int32_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.int64_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +// Sets the value of a `string` or `bytes` field. The bytes of the value are not +// copied, so it is the caller's responsibility to ensure that they remain valid +// for the lifetime of `msg`. That might be done by copying them into the given +// arena, or by fusing that arena with the arena the bytes live in, for example. +UPB_API_INLINE bool upb_Message_SetString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.str_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.uint32_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.uint64_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + UPB_API_INLINE void upb_Message_Clear(struct upb_Message* msg, const upb_MiniTable* m) { UPB_ASSERT(!upb_Message_IsFrozen(msg)); @@ -335,33 +693,15 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } -UPB_INLINE void _upb_Message_AssertMapIsUntagged( - const struct upb_Message* msg, const upb_MiniTableField* field) { - UPB_UNUSED(msg); - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); -#ifndef NDEBUG - uintptr_t default_val = 0; - uintptr_t tagged; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); - UPB_ASSERT(!upb_TaggedMessagePtr_IsEmpty(tagged)); -#endif -} - -UPB_INLINE struct upb_Map* _upb_Message_GetOrCreateMutableMap( - struct upb_Message* msg, const upb_MiniTableField* field, size_t key_size, - size_t val_size, upb_Arena* arena) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - _upb_Message_AssertMapIsUntagged(msg, field); - struct upb_Map* map = NULL; - struct upb_Map* default_map_value = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_map_value, &map); - if (!map) { - map = _upb_Map_New(arena, key_size, val_size); - // Check again due to: https://godbolt.org/z/7WfaoKG1r - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - upb_Message_SetBaseField(msg, field, &map); +UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( + struct upb_Message* msg, const upb_MiniTableField* f, size_t size, + upb_Arena* arena) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* arr = upb_Message_GetOrCreateMutableArray(msg, f, arena); + if (!arr || !UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, size, arena)) { + return NULL; } - return map; + return upb_Array_MutableDataPtr(arr); } #ifdef __cplusplus From 7646fbfebd39b403f3c45540a510f15d1d6d05ef Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 May 2024 19:47:53 +0000 Subject: [PATCH 100/179] Auto-generate files after cl/630466865 --- php/ext/google/protobuf/php-upb.c | 8 +- php/ext/google/protobuf/php-upb.h | 798 ++++++++++++++------------ ruby/ext/google/protobuf_c/ruby-upb.c | 8 +- ruby/ext/google/protobuf_c/ruby-upb.h | 798 ++++++++++++++------------ 4 files changed, 894 insertions(+), 718 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 2d78a633f4..fe3bbb76cf 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -6157,10 +6157,10 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, if (dst_sub_message == NULL) { return NULL; } - _upb_Message_SetTaggedMessagePtr( - dst, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, - is_empty)); + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (dst, mini_table, field, + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, + is_empty)); } } break; case kUpb_CType_String: diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index c9fc8ec49e..a3d60bc576 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -2691,6 +2691,12 @@ UPB_INLINE bool UPB_PRIVATE(_upb_Message_ClearOneofCase)( return true; } +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const struct upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} + // LINT.ThenChange(GoogleInternalName2) // Returns false if the message is missing any of its required fields. @@ -2850,6 +2856,24 @@ UPB_INLINE void _upb_Message_GetExtensionField( } } +// NOTE: The default_val is only used for fields that support presence. +// For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a +// upb_Array/upb_Map has not been allocated yet. Array/map fields do not have +// presence, so this is semantically identical to a pointer to an empty +// array/map, and must be treated the same for all semantic purposes. +UPB_API_INLINE upb_MessageValue upb_Message_GetField( + const struct upb_Message* msg, const upb_MiniTableField* field, + upb_MessageValue default_val) { + upb_MessageValue ret; + if (upb_MiniTableField_IsExtension(field)) { + _upb_Message_GetExtensionField(msg, (upb_MiniTableExtension*)field, + &default_val, &ret); + } else { + _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); + } + return ret; +} + UPB_API_INLINE void upb_Message_SetBaseField(struct upb_Message* msg, const upb_MiniTableField* f, const void* val) { @@ -2873,6 +2897,336 @@ UPB_API_INLINE bool upb_Message_SetExtension(struct upb_Message* msg, return true; } +// Sets the value of the given field in the given msg. The return value is true +// if the operation completed successfully, or false if memory allocation +// failed. +UPB_INLINE bool UPB_PRIVATE(_upb_Message_SetField)(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_MessageValue val, + upb_Arena* a) { + if (upb_MiniTableField_IsExtension(f)) { + const upb_MiniTableExtension* ext = (const upb_MiniTableExtension*)f; + return upb_Message_SetExtension(msg, ext, &val, a); + } else { + upb_Message_SetBaseField(msg, f, &val); + return true; + } +} + +UPB_API_INLINE const upb_Array* upb_Message_GetArray( + const struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* ret; + const upb_Array* default_val = NULL; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &ret); + return ret; +} + +UPB_API_INLINE bool upb_Message_GetBool(const struct upb_Message* msg, + const upb_MiniTableField* f, + bool default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue def; + def.bool_val = default_val; + return upb_Message_GetField(msg, f, def).bool_val; +} + +UPB_API_INLINE double upb_Message_GetDouble(const struct upb_Message* msg, + const upb_MiniTableField* f, + double default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.double_val = default_val; + return upb_Message_GetField(msg, f, def).double_val; +} + +UPB_API_INLINE float upb_Message_GetFloat(const struct upb_Message* msg, + const upb_MiniTableField* f, + float default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.float_val = default_val; + return upb_Message_GetField(msg, f, def).float_val; +} + +UPB_API_INLINE int32_t upb_Message_GetInt32(const struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.int32_val = default_val; + return upb_Message_GetField(msg, f, def).int32_val; +} + +UPB_API_INLINE int64_t upb_Message_GetInt64(const struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.int64_val = default_val; + return upb_Message_GetField(msg, f, def).int64_val; +} + +UPB_INLINE void UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)( + const struct upb_Message* msg, const upb_MiniTableField* field) { + UPB_UNUSED(msg); + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); +#ifndef NDEBUG + uintptr_t default_val = 0; + uintptr_t tagged; + _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); + UPB_ASSERT(!upb_TaggedMessagePtr_IsEmpty(tagged)); +#endif +} + +UPB_API_INLINE const struct upb_Map* upb_Message_GetMap( + const struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(f); + UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)(msg, f); + struct upb_Map* ret; + const struct upb_Map* default_val = NULL; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &ret); + return ret; +} + +UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( + struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + return (upb_Array*)upb_Message_GetArray(msg, f); +} + +UPB_API_INLINE struct upb_Map* upb_Message_GetMutableMap( + struct upb_Message* msg, const upb_MiniTableField* f) { + return (struct upb_Map*)upb_Message_GetMap(msg, f); +} + +UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( + struct upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSERT(arena); + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* array = upb_Message_GetMutableArray(msg, f); + if (!array) { + array = UPB_PRIVATE(_upb_Array_New)( + arena, 4, UPB_PRIVATE(_upb_MiniTableField_ElemSizeLg2)(f)); + // Check again due to: https://godbolt.org/z/7WfaoKG1r + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_MessageValue val; + val.array_val = array; + UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, arena); + } + return array; +} + +UPB_INLINE struct upb_Map* _upb_Message_GetOrCreateMutableMap( + struct upb_Message* msg, const upb_MiniTableField* field, size_t key_size, + size_t val_size, upb_Arena* arena) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); + UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)(msg, field); + struct upb_Map* map = NULL; + struct upb_Map* default_map_value = NULL; + _upb_Message_GetNonExtensionField(msg, field, &default_map_value, &map); + if (!map) { + map = _upb_Map_New(arena, key_size, val_size); + // Check again due to: https://godbolt.org/z/7WfaoKG1r + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); + upb_Message_SetBaseField(msg, field, &map); + } + return map; +} + +UPB_API_INLINE struct upb_Map* upb_Message_GetOrCreateMutableMap( + struct upb_Message* msg, const upb_MiniTable* map_entry_mini_table, + const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + const upb_MiniTableField* map_entry_key_field = + &map_entry_mini_table->UPB_ONLYBITS(fields)[0]; + const upb_MiniTableField* map_entry_value_field = + &map_entry_mini_table->UPB_ONLYBITS(fields)[1]; + return _upb_Message_GetOrCreateMutableMap( + msg, f, _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_key_field)), + _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_value_field)), + arena); +} + +UPB_API_INLINE struct upb_Message* upb_Message_GetOrCreateMutableMessage( + struct upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSERT(arena); + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(!upb_MiniTableField_IsExtension(f)); + struct upb_Message* sub_message = + *UPB_PTR_AT(msg, f->UPB_ONLYBITS(offset), struct upb_Message*); + if (!sub_message) { + const upb_MiniTable* sub_mini_table = + upb_MiniTable_SubMessage(mini_table, f); + UPB_ASSERT(sub_mini_table); + sub_message = _upb_Message_New(sub_mini_table, arena); + *UPB_PTR_AT(msg, f->UPB_ONLYBITS(offset), struct upb_Message*) = + sub_message; + UPB_PRIVATE(_upb_Message_SetPresence)(msg, f); + } + return sub_message; +} + +UPB_API_INLINE upb_StringView +upb_Message_GetString(const struct upb_Message* msg, + const upb_MiniTableField* f, upb_StringView default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.str_val = default_val; + return upb_Message_GetField(msg, f, def).str_val; +} + +UPB_API_INLINE uint32_t upb_Message_GetUInt32(const struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.uint32_val = default_val; + return upb_Message_GetField(msg, f, def).uint32_val; +} + +UPB_API_INLINE uint64_t upb_Message_GetUInt64(const struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.uint64_val = default_val; + return upb_Message_GetField(msg, f, def).uint64_val; +} + +UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.bool_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE void upb_Message_SetClosedEnum( + struct upb_Message* msg, const upb_MiniTable* msg_mini_table, + const upb_MiniTableField* f, int32_t value) { + UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSERT(upb_MiniTableEnum_CheckValue( + upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE bool upb_Message_SetDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.double_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.float_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.int32_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.int64_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +// Sets the value of a `string` or `bytes` field. The bytes of the value are not +// copied, so it is the caller's responsibility to ensure that they remain valid +// for the lifetime of `msg`. That might be done by copying them into the given +// arena, or by fusing that arena with the arena the bytes live in, for example. +UPB_API_INLINE bool upb_Message_SetString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.str_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.uint32_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.uint64_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + UPB_API_INLINE void upb_Message_Clear(struct upb_Message* msg, const upb_MiniTable* m) { UPB_ASSERT(!upb_Message_IsFrozen(msg)); @@ -2914,33 +3268,15 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } -UPB_INLINE void _upb_Message_AssertMapIsUntagged( - const struct upb_Message* msg, const upb_MiniTableField* field) { - UPB_UNUSED(msg); - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); -#ifndef NDEBUG - uintptr_t default_val = 0; - uintptr_t tagged; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); - UPB_ASSERT(!upb_TaggedMessagePtr_IsEmpty(tagged)); -#endif -} - -UPB_INLINE struct upb_Map* _upb_Message_GetOrCreateMutableMap( - struct upb_Message* msg, const upb_MiniTableField* field, size_t key_size, - size_t val_size, upb_Arena* arena) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - _upb_Message_AssertMapIsUntagged(msg, field); - struct upb_Map* map = NULL; - struct upb_Map* default_map_value = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_map_value, &map); - if (!map) { - map = _upb_Map_New(arena, key_size, val_size); - // Check again due to: https://godbolt.org/z/7WfaoKG1r - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - upb_Message_SetBaseField(msg, field, &map); +UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( + struct upb_Message* msg, const upb_MiniTableField* f, size_t size, + upb_Arena* arena) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* arr = upb_Message_GetOrCreateMutableArray(msg, f, arena); + if (!arr || !UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, size, arena)) { + return NULL; } - return map; + return upb_Array_MutableDataPtr(arr); } #ifdef __cplusplus @@ -3225,263 +3561,100 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} - -// NOTE: The default_val is only used for fields that support presence. -// For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a -// upb_Array/upb_Map has not been allocated yet. Array/map fields do not have -// presence, so this is semantically identical to a pointer to an empty -// array/map, and must be treated the same for all semantic purposes. -UPB_INLINE upb_MessageValue -upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* field, - upb_MessageValue default_val) { - upb_MessageValue ret; - if (upb_MiniTableField_IsExtension(field)) { - _upb_Message_GetExtensionField(msg, (upb_MiniTableExtension*)field, - &default_val, &ret); - } else { - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - } - return ret; -} +UPB_API_INLINE upb_MessageValue +upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, + upb_MessageValue default_val); -// Sets the value of the given field in the given msg. The return value is true -// if the operation completed successfully, or false if memory allocation -// failed. -UPB_INLINE bool UPB_PRIVATE(_upb_Message_SetField)( - upb_Message* msg, const upb_MiniTableField* field, upb_MessageValue val, - upb_Arena* a) { - if (upb_MiniTableField_IsExtension(field)) { - const upb_MiniTableExtension* ext = (const upb_MiniTableExtension*)field; - return upb_Message_SetExtension(msg, ext, &val, a); - } else { - upb_Message_SetBaseField(msg, field, &val); - return true; - } -} +UPB_API_INLINE const upb_Array* upb_Message_GetArray( + const upb_Message* msg, const upb_MiniTableField* f); UPB_API_INLINE bool upb_Message_GetBool(const upb_Message* msg, - const upb_MiniTableField* field, - bool default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_1Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue def; - def.bool_val = default_val; - return upb_Message_GetField(msg, field, def).bool_val; -} + const upb_MiniTableField* f, + bool default_val); -UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, - const upb_MiniTableField* field, - bool value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_1Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.bool_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE double upb_Message_GetDouble(const upb_Message* msg, + const upb_MiniTableField* field, + double default_val); + +UPB_API_INLINE float upb_Message_GetFloat(const upb_Message* msg, + const upb_MiniTableField* f, + float default_val); UPB_API_INLINE int32_t upb_Message_GetInt32(const upb_Message* msg, - const upb_MiniTableField* field, - int32_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || - upb_MiniTableField_CType(field) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); + const upb_MiniTableField* f, + int32_t default_val); - upb_MessageValue def; - def.int32_val = default_val; - return upb_Message_GetField(msg, field, def).int32_val; -} +UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, + const upb_MiniTableField* f, + int64_t default_val); -UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, - const upb_MiniTableField* field, - int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || - upb_MiniTableField_CType(field) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.int32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE const upb_Map* upb_Message_GetMap(const upb_Message* msg, + const upb_MiniTableField* f); -UPB_API_INLINE uint32_t upb_Message_GetUInt32(const upb_Message* msg, - const upb_MiniTableField* field, - uint32_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( + upb_Message* msg, const upb_MiniTableField* f); - upb_MessageValue def; - def.uint32_val = default_val; - return upb_Message_GetField(msg, field, def).uint32_val; -} +UPB_API_INLINE upb_Map* upb_Message_GetMutableMap(upb_Message* msg, + const upb_MiniTableField* f); -UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, - const upb_MiniTableField* field, - uint32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.uint32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( + upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena); -UPB_API_INLINE void upb_Message_SetClosedEnum( - upb_Message* msg, const upb_MiniTable* msg_mini_table, - const upb_MiniTableField* field, int32_t value) { - UPB_ASSERT(upb_MiniTableField_IsClosedEnum(field)); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - UPB_ASSERT(upb_MiniTableEnum_CheckValue( - upb_MiniTable_GetSubEnumTable(msg_mini_table, field), value)); - upb_Message_SetBaseField(msg, field, &value); -} +UPB_API_INLINE upb_Map* upb_Message_GetOrCreateMutableMap( + upb_Message* msg, const upb_MiniTable* map_entry_mini_table, + const upb_MiniTableField* f, upb_Arena* arena); -UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, - const upb_MiniTableField* field, - int64_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( + upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_Arena* arena); - upb_MessageValue def; - def.int64_val = default_val; - return upb_Message_GetField(msg, field, def).int64_val; -} +UPB_API_INLINE upb_StringView +upb_Message_GetString(const upb_Message* msg, const upb_MiniTableField* field, + upb_StringView default_val); -UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, - const upb_MiniTableField* field, - int64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.int64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE uint32_t upb_Message_GetUInt32(const upb_Message* msg, + const upb_MiniTableField* f, + uint32_t default_val); UPB_API_INLINE uint64_t upb_Message_GetUInt64(const upb_Message* msg, - const upb_MiniTableField* field, - uint64_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - - upb_MessageValue def; - def.uint64_val = default_val; - return upb_Message_GetField(msg, field, def).uint64_val; -} + const upb_MiniTableField* f, + uint64_t default_val); -UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, - const upb_MiniTableField* field, - uint64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.uint64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a); -UPB_API_INLINE float upb_Message_GetFloat(const upb_Message* msg, - const upb_MiniTableField* field, - float default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE void upb_Message_SetClosedEnum( + upb_Message* msg, const upb_MiniTable* msg_mini_table, + const upb_MiniTableField* f, int32_t value); - upb_MessageValue def; - def.float_val = default_val; - return upb_Message_GetField(msg, field, def).float_val; -} +UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, + const upb_MiniTableField* f, + double value, upb_Arena* a); UPB_API_INLINE bool upb_Message_SetFloat(upb_Message* msg, - const upb_MiniTableField* field, - float value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.float_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} - -UPB_API_INLINE double upb_Message_GetDouble(const upb_Message* msg, - const upb_MiniTableField* field, - double default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); + const upb_MiniTableField* f, + float value, upb_Arena* a); - upb_MessageValue def; - def.double_val = default_val; - return upb_Message_GetField(msg, field, def).double_val; -} +UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, + const upb_MiniTableField* f, + int32_t value, upb_Arena* a); -UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, - const upb_MiniTableField* field, - double value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.double_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, + const upb_MiniTableField* f, + int64_t value, upb_Arena* a); -UPB_API_INLINE upb_StringView -upb_Message_GetString(const upb_Message* msg, const upb_MiniTableField* field, - upb_StringView default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_String || - upb_MiniTableField_CType(field) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value, upb_Arena* a); - upb_MessageValue def; - def.str_val = default_val; - return upb_Message_GetField(msg, field, def).str_val; -} +UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value, upb_Arena* a); -// Sets the value of a `string` or `bytes` field. The bytes of the value are not -// copied, so it is the caller's responsibility to ensure that they remain valid -// for the lifetime of `msg`. That might be done by copying them into the given -// arena, or by fusing that arena with the arena the bytes live in, for example. -UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, - const upb_MiniTableField* field, - upb_StringView value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_String || - upb_MiniTableField_CType(field) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.str_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value, upb_Arena* a); UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( const upb_Message* msg, const upb_MiniTableField* field, @@ -3510,14 +3683,14 @@ UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( // For internal use only; users cannot set tagged messages because only the // parser and the message copier are allowed to directly create an empty // message. -UPB_API_INLINE void _upb_Message_SetTaggedMessagePtr( - upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* field, upb_TaggedMessagePtr sub_message) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == +UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( + struct upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_TaggedMessagePtr sub_message) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_Message_SetBaseField(msg, field, &sub_message); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_Message_SetBaseField(msg, f, &sub_message); } // Sets the value of a message-typed field. The `mini_table` and `field` @@ -3527,102 +3700,17 @@ UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, const upb_MiniTable* mini_table, const upb_MiniTableField* field, upb_Message* sub_message) { - _upb_Message_SetTaggedMessagePtr( - msg, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); -} - -UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( - upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSERT(arena); - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(!upb_MiniTableField_IsExtension(field)); - upb_Message* sub_message = - *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); - if (!sub_message) { - const upb_MiniTable* sub_mini_table = - upb_MiniTable_SubMessage(mini_table, field); - UPB_ASSERT(sub_mini_table); - sub_message = _upb_Message_New(sub_mini_table, arena); - *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*) = sub_message; - UPB_PRIVATE(_upb_Message_SetPresence)(msg, field); - } - return sub_message; -} - -UPB_API_INLINE const upb_Array* upb_Message_GetArray( - const upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* ret; - const upb_Array* default_val = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - return ret; -} - -UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( - upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - return (upb_Array*)upb_Message_GetArray(msg, field); -} - -UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( - upb_Message* msg, const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSERT(arena); - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* array = upb_Message_GetMutableArray(msg, field); - if (!array) { - array = UPB_PRIVATE(_upb_Array_New)( - arena, 4, UPB_PRIVATE(_upb_MiniTableField_ElemSizeLg2)(field)); - // Check again due to: https://godbolt.org/z/7WfaoKG1r - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_MessageValue val; - val.array_val = array; - UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, arena); - } - return array; + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (msg, mini_table, field, + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); } UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( - upb_Message* msg, const upb_MiniTableField* field, size_t size, - upb_Arena* arena) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* arr = upb_Message_GetOrCreateMutableArray(msg, field, arena); - if (!arr || !UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, size, arena)) { - return NULL; - } - return upb_Array_MutableDataPtr(arr); -} - -UPB_API_INLINE const upb_Map* upb_Message_GetMap( - const upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - _upb_Message_AssertMapIsUntagged(msg, field); - upb_Map* ret; - const upb_Map* default_val = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - return ret; -} - -UPB_API_INLINE upb_Map* upb_Message_GetMutableMap( - upb_Message* msg, const upb_MiniTableField* field) { - return (upb_Map*)upb_Message_GetMap(msg, field); -} + upb_Message* msg, const upb_MiniTableField* f, size_t size, + upb_Arena* arena); -UPB_API_INLINE upb_Map* upb_Message_GetOrCreateMutableMap( - upb_Message* msg, const upb_MiniTable* map_entry_mini_table, - const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - const upb_MiniTableField* map_entry_key_field = - &map_entry_mini_table->UPB_ONLYBITS(fields)[0]; - const upb_MiniTableField* map_entry_value_field = - &map_entry_mini_table->UPB_ONLYBITS(fields)[1]; - return _upb_Message_GetOrCreateMutableMap( - msg, field, - _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_key_field)), - _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_value_field)), - arena); -} +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const upb_Message* message, const upb_MiniTableField* oneof_field); // Updates a map entry given an entry message. bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* mini_table, diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 1483124033..989c51d2a4 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -5649,10 +5649,10 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, if (dst_sub_message == NULL) { return NULL; } - _upb_Message_SetTaggedMessagePtr( - dst, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, - is_empty)); + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (dst, mini_table, field, + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, + is_empty)); } } break; case kUpb_CType_String: diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 277b52be07..f5cf050c34 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -2693,6 +2693,12 @@ UPB_INLINE bool UPB_PRIVATE(_upb_Message_ClearOneofCase)( return true; } +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const struct upb_Message* message, const upb_MiniTableField* oneof_field) { + UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); + return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); +} + // LINT.ThenChange(GoogleInternalName2) // Returns false if the message is missing any of its required fields. @@ -2852,6 +2858,24 @@ UPB_INLINE void _upb_Message_GetExtensionField( } } +// NOTE: The default_val is only used for fields that support presence. +// For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a +// upb_Array/upb_Map has not been allocated yet. Array/map fields do not have +// presence, so this is semantically identical to a pointer to an empty +// array/map, and must be treated the same for all semantic purposes. +UPB_API_INLINE upb_MessageValue upb_Message_GetField( + const struct upb_Message* msg, const upb_MiniTableField* field, + upb_MessageValue default_val) { + upb_MessageValue ret; + if (upb_MiniTableField_IsExtension(field)) { + _upb_Message_GetExtensionField(msg, (upb_MiniTableExtension*)field, + &default_val, &ret); + } else { + _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); + } + return ret; +} + UPB_API_INLINE void upb_Message_SetBaseField(struct upb_Message* msg, const upb_MiniTableField* f, const void* val) { @@ -2875,6 +2899,336 @@ UPB_API_INLINE bool upb_Message_SetExtension(struct upb_Message* msg, return true; } +// Sets the value of the given field in the given msg. The return value is true +// if the operation completed successfully, or false if memory allocation +// failed. +UPB_INLINE bool UPB_PRIVATE(_upb_Message_SetField)(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_MessageValue val, + upb_Arena* a) { + if (upb_MiniTableField_IsExtension(f)) { + const upb_MiniTableExtension* ext = (const upb_MiniTableExtension*)f; + return upb_Message_SetExtension(msg, ext, &val, a); + } else { + upb_Message_SetBaseField(msg, f, &val); + return true; + } +} + +UPB_API_INLINE const upb_Array* upb_Message_GetArray( + const struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* ret; + const upb_Array* default_val = NULL; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &ret); + return ret; +} + +UPB_API_INLINE bool upb_Message_GetBool(const struct upb_Message* msg, + const upb_MiniTableField* f, + bool default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue def; + def.bool_val = default_val; + return upb_Message_GetField(msg, f, def).bool_val; +} + +UPB_API_INLINE double upb_Message_GetDouble(const struct upb_Message* msg, + const upb_MiniTableField* f, + double default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.double_val = default_val; + return upb_Message_GetField(msg, f, def).double_val; +} + +UPB_API_INLINE float upb_Message_GetFloat(const struct upb_Message* msg, + const upb_MiniTableField* f, + float default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.float_val = default_val; + return upb_Message_GetField(msg, f, def).float_val; +} + +UPB_API_INLINE int32_t upb_Message_GetInt32(const struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.int32_val = default_val; + return upb_Message_GetField(msg, f, def).int32_val; +} + +UPB_API_INLINE int64_t upb_Message_GetInt64(const struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.int64_val = default_val; + return upb_Message_GetField(msg, f, def).int64_val; +} + +UPB_INLINE void UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)( + const struct upb_Message* msg, const upb_MiniTableField* field) { + UPB_UNUSED(msg); + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); +#ifndef NDEBUG + uintptr_t default_val = 0; + uintptr_t tagged; + _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); + UPB_ASSERT(!upb_TaggedMessagePtr_IsEmpty(tagged)); +#endif +} + +UPB_API_INLINE const struct upb_Map* upb_Message_GetMap( + const struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(f); + UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)(msg, f); + struct upb_Map* ret; + const struct upb_Map* default_val = NULL; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &ret); + return ret; +} + +UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( + struct upb_Message* msg, const upb_MiniTableField* f) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + return (upb_Array*)upb_Message_GetArray(msg, f); +} + +UPB_API_INLINE struct upb_Map* upb_Message_GetMutableMap( + struct upb_Message* msg, const upb_MiniTableField* f) { + return (struct upb_Map*)upb_Message_GetMap(msg, f); +} + +UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( + struct upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSERT(arena); + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* array = upb_Message_GetMutableArray(msg, f); + if (!array) { + array = UPB_PRIVATE(_upb_Array_New)( + arena, 4, UPB_PRIVATE(_upb_MiniTableField_ElemSizeLg2)(f)); + // Check again due to: https://godbolt.org/z/7WfaoKG1r + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_MessageValue val; + val.array_val = array; + UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, arena); + } + return array; +} + +UPB_INLINE struct upb_Map* _upb_Message_GetOrCreateMutableMap( + struct upb_Message* msg, const upb_MiniTableField* field, size_t key_size, + size_t val_size, upb_Arena* arena) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); + UPB_PRIVATE(_upb_Message_AssertMapIsUntagged)(msg, field); + struct upb_Map* map = NULL; + struct upb_Map* default_map_value = NULL; + _upb_Message_GetNonExtensionField(msg, field, &default_map_value, &map); + if (!map) { + map = _upb_Map_New(arena, key_size, val_size); + // Check again due to: https://godbolt.org/z/7WfaoKG1r + UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); + upb_Message_SetBaseField(msg, field, &map); + } + return map; +} + +UPB_API_INLINE struct upb_Map* upb_Message_GetOrCreateMutableMap( + struct upb_Message* msg, const upb_MiniTable* map_entry_mini_table, + const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + const upb_MiniTableField* map_entry_key_field = + &map_entry_mini_table->UPB_ONLYBITS(fields)[0]; + const upb_MiniTableField* map_entry_value_field = + &map_entry_mini_table->UPB_ONLYBITS(fields)[1]; + return _upb_Message_GetOrCreateMutableMap( + msg, f, _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_key_field)), + _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_value_field)), + arena); +} + +UPB_API_INLINE struct upb_Message* upb_Message_GetOrCreateMutableMessage( + struct upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_Arena* arena) { + UPB_ASSERT(arena); + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(!upb_MiniTableField_IsExtension(f)); + struct upb_Message* sub_message = + *UPB_PTR_AT(msg, f->UPB_ONLYBITS(offset), struct upb_Message*); + if (!sub_message) { + const upb_MiniTable* sub_mini_table = + upb_MiniTable_SubMessage(mini_table, f); + UPB_ASSERT(sub_mini_table); + sub_message = _upb_Message_New(sub_mini_table, arena); + *UPB_PTR_AT(msg, f->UPB_ONLYBITS(offset), struct upb_Message*) = + sub_message; + UPB_PRIVATE(_upb_Message_SetPresence)(msg, f); + } + return sub_message; +} + +UPB_API_INLINE upb_StringView +upb_Message_GetString(const struct upb_Message* msg, + const upb_MiniTableField* f, upb_StringView default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.str_val = default_val; + return upb_Message_GetField(msg, f, def).str_val; +} + +UPB_API_INLINE uint32_t upb_Message_GetUInt32(const struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.uint32_val = default_val; + return upb_Message_GetField(msg, f, def).uint32_val; +} + +UPB_API_INLINE uint64_t upb_Message_GetUInt64(const struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + + upb_MessageValue def; + def.uint64_val = default_val; + return upb_Message_GetField(msg, f, def).uint64_val; +} + +UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.bool_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE void upb_Message_SetClosedEnum( + struct upb_Message* msg, const upb_MiniTable* msg_mini_table, + const upb_MiniTableField* f, int32_t value) { + UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSERT(upb_MiniTableEnum_CheckValue( + upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE bool upb_Message_SetDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.double_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.float_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.int32_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.int64_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +// Sets the value of a `string` or `bytes` field. The bytes of the value are not +// copied, so it is the caller's responsibility to ensure that they remain valid +// for the lifetime of `msg`. That might be done by copying them into the given +// arena, or by fusing that arena with the arena the bytes live in, for example. +UPB_API_INLINE bool upb_Message_SetString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.str_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.uint32_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + +UPB_API_INLINE bool upb_Message_SetUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_MessageValue val; + val.uint64_val = value; + return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); +} + UPB_API_INLINE void upb_Message_Clear(struct upb_Message* msg, const upb_MiniTable* m) { UPB_ASSERT(!upb_Message_IsFrozen(msg)); @@ -2916,33 +3270,15 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } -UPB_INLINE void _upb_Message_AssertMapIsUntagged( - const struct upb_Message* msg, const upb_MiniTableField* field) { - UPB_UNUSED(msg); - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); -#ifndef NDEBUG - uintptr_t default_val = 0; - uintptr_t tagged; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); - UPB_ASSERT(!upb_TaggedMessagePtr_IsEmpty(tagged)); -#endif -} - -UPB_INLINE struct upb_Map* _upb_Message_GetOrCreateMutableMap( - struct upb_Message* msg, const upb_MiniTableField* field, size_t key_size, - size_t val_size, upb_Arena* arena) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - _upb_Message_AssertMapIsUntagged(msg, field); - struct upb_Map* map = NULL; - struct upb_Map* default_map_value = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_map_value, &map); - if (!map) { - map = _upb_Map_New(arena, key_size, val_size); - // Check again due to: https://godbolt.org/z/7WfaoKG1r - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - upb_Message_SetBaseField(msg, field, &map); +UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( + struct upb_Message* msg, const upb_MiniTableField* f, size_t size, + upb_Arena* arena) { + UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); + upb_Array* arr = upb_Message_GetOrCreateMutableArray(msg, f, arena); + if (!arr || !UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, size, arena)) { + return NULL; } - return map; + return upb_Array_MutableDataPtr(arr); } #ifdef __cplusplus @@ -3227,263 +3563,100 @@ UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, const upb_MiniTableExtension* e, const void* val, upb_Arena* a); -UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( - const upb_Message* message, const upb_MiniTableField* oneof_field) { - UPB_ASSUME(upb_MiniTableField_IsInOneof(oneof_field)); - return UPB_PRIVATE(_upb_Message_GetOneofCase)(message, oneof_field); -} - -// NOTE: The default_val is only used for fields that support presence. -// For repeated/map fields, the resulting upb_Array*/upb_Map* can be NULL if a -// upb_Array/upb_Map has not been allocated yet. Array/map fields do not have -// presence, so this is semantically identical to a pointer to an empty -// array/map, and must be treated the same for all semantic purposes. -UPB_INLINE upb_MessageValue -upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* field, - upb_MessageValue default_val) { - upb_MessageValue ret; - if (upb_MiniTableField_IsExtension(field)) { - _upb_Message_GetExtensionField(msg, (upb_MiniTableExtension*)field, - &default_val, &ret); - } else { - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - } - return ret; -} +UPB_API_INLINE upb_MessageValue +upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, + upb_MessageValue default_val); -// Sets the value of the given field in the given msg. The return value is true -// if the operation completed successfully, or false if memory allocation -// failed. -UPB_INLINE bool UPB_PRIVATE(_upb_Message_SetField)( - upb_Message* msg, const upb_MiniTableField* field, upb_MessageValue val, - upb_Arena* a) { - if (upb_MiniTableField_IsExtension(field)) { - const upb_MiniTableExtension* ext = (const upb_MiniTableExtension*)field; - return upb_Message_SetExtension(msg, ext, &val, a); - } else { - upb_Message_SetBaseField(msg, field, &val); - return true; - } -} +UPB_API_INLINE const upb_Array* upb_Message_GetArray( + const upb_Message* msg, const upb_MiniTableField* f); UPB_API_INLINE bool upb_Message_GetBool(const upb_Message* msg, - const upb_MiniTableField* field, - bool default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_1Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue def; - def.bool_val = default_val; - return upb_Message_GetField(msg, field, def).bool_val; -} + const upb_MiniTableField* f, + bool default_val); -UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, - const upb_MiniTableField* field, - bool value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_1Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.bool_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE double upb_Message_GetDouble(const upb_Message* msg, + const upb_MiniTableField* field, + double default_val); + +UPB_API_INLINE float upb_Message_GetFloat(const upb_Message* msg, + const upb_MiniTableField* f, + float default_val); UPB_API_INLINE int32_t upb_Message_GetInt32(const upb_Message* msg, - const upb_MiniTableField* field, - int32_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || - upb_MiniTableField_CType(field) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); + const upb_MiniTableField* f, + int32_t default_val); - upb_MessageValue def; - def.int32_val = default_val; - return upb_Message_GetField(msg, field, def).int32_val; -} +UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, + const upb_MiniTableField* f, + int64_t default_val); -UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, - const upb_MiniTableField* field, - int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int32 || - upb_MiniTableField_CType(field) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.int32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE const upb_Map* upb_Message_GetMap(const upb_Message* msg, + const upb_MiniTableField* f); -UPB_API_INLINE uint32_t upb_Message_GetUInt32(const upb_Message* msg, - const upb_MiniTableField* field, - uint32_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( + upb_Message* msg, const upb_MiniTableField* f); - upb_MessageValue def; - def.uint32_val = default_val; - return upb_Message_GetField(msg, field, def).uint32_val; -} +UPB_API_INLINE upb_Map* upb_Message_GetMutableMap(upb_Message* msg, + const upb_MiniTableField* f); -UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, - const upb_MiniTableField* field, - uint32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.uint32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( + upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena); -UPB_API_INLINE void upb_Message_SetClosedEnum( - upb_Message* msg, const upb_MiniTable* msg_mini_table, - const upb_MiniTableField* field, int32_t value) { - UPB_ASSERT(upb_MiniTableField_IsClosedEnum(field)); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - UPB_ASSERT(upb_MiniTableEnum_CheckValue( - upb_MiniTable_GetSubEnumTable(msg_mini_table, field), value)); - upb_Message_SetBaseField(msg, field, &value); -} +UPB_API_INLINE upb_Map* upb_Message_GetOrCreateMutableMap( + upb_Message* msg, const upb_MiniTable* map_entry_mini_table, + const upb_MiniTableField* f, upb_Arena* arena); -UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, - const upb_MiniTableField* field, - int64_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( + upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_Arena* arena); - upb_MessageValue def; - def.int64_val = default_val; - return upb_Message_GetField(msg, field, def).int64_val; -} +UPB_API_INLINE upb_StringView +upb_Message_GetString(const upb_Message* msg, const upb_MiniTableField* field, + upb_StringView default_val); -UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, - const upb_MiniTableField* field, - int64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.int64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE uint32_t upb_Message_GetUInt32(const upb_Message* msg, + const upb_MiniTableField* f, + uint32_t default_val); UPB_API_INLINE uint64_t upb_Message_GetUInt64(const upb_Message* msg, - const upb_MiniTableField* field, - uint64_t default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - - upb_MessageValue def; - def.uint64_val = default_val; - return upb_Message_GetField(msg, field, def).uint64_val; -} + const upb_MiniTableField* f, + uint64_t default_val); -UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, - const upb_MiniTableField* field, - uint64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.uint64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a); -UPB_API_INLINE float upb_Message_GetFloat(const upb_Message* msg, - const upb_MiniTableField* field, - float default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE void upb_Message_SetClosedEnum( + upb_Message* msg, const upb_MiniTable* msg_mini_table, + const upb_MiniTableField* f, int32_t value); - upb_MessageValue def; - def.float_val = default_val; - return upb_Message_GetField(msg, field, def).float_val; -} +UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, + const upb_MiniTableField* f, + double value, upb_Arena* a); UPB_API_INLINE bool upb_Message_SetFloat(upb_Message* msg, - const upb_MiniTableField* field, - float value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.float_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} - -UPB_API_INLINE double upb_Message_GetDouble(const upb_Message* msg, - const upb_MiniTableField* field, - double default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); + const upb_MiniTableField* f, + float value, upb_Arena* a); - upb_MessageValue def; - def.double_val = default_val; - return upb_Message_GetField(msg, field, def).double_val; -} +UPB_API_INLINE bool upb_Message_SetInt32(upb_Message* msg, + const upb_MiniTableField* f, + int32_t value, upb_Arena* a); -UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, - const upb_MiniTableField* field, - double value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.double_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, + const upb_MiniTableField* f, + int64_t value, upb_Arena* a); -UPB_API_INLINE upb_StringView -upb_Message_GetString(const upb_Message* msg, const upb_MiniTableField* field, - upb_StringView default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_String || - upb_MiniTableField_CType(field) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); +UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value, upb_Arena* a); - upb_MessageValue def; - def.str_val = default_val; - return upb_Message_GetField(msg, field, def).str_val; -} +UPB_API_INLINE bool upb_Message_SetUInt32(upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value, upb_Arena* a); -// Sets the value of a `string` or `bytes` field. The bytes of the value are not -// copied, so it is the caller's responsibility to ensure that they remain valid -// for the lifetime of `msg`. That might be done by copying them into the given -// arena, or by fusing that arena with the arena the bytes live in, for example. -UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, - const upb_MiniTableField* field, - upb_StringView value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_String || - upb_MiniTableField_CType(field) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_MessageValue val; - val.str_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, a); -} +UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value, upb_Arena* a); UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( const upb_Message* msg, const upb_MiniTableField* field, @@ -3512,14 +3685,14 @@ UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( // For internal use only; users cannot set tagged messages because only the // parser and the message copier are allowed to directly create an empty // message. -UPB_API_INLINE void _upb_Message_SetTaggedMessagePtr( - upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* field, upb_TaggedMessagePtr sub_message) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == +UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( + struct upb_Message* msg, const upb_MiniTable* mini_table, + const upb_MiniTableField* f, upb_TaggedMessagePtr sub_message) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_Message_SetBaseField(msg, field, &sub_message); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_Message_SetBaseField(msg, f, &sub_message); } // Sets the value of a message-typed field. The `mini_table` and `field` @@ -3529,102 +3702,17 @@ UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, const upb_MiniTable* mini_table, const upb_MiniTableField* field, upb_Message* sub_message) { - _upb_Message_SetTaggedMessagePtr( - msg, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); -} - -UPB_API_INLINE upb_Message* upb_Message_GetOrCreateMutableMessage( - upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSERT(arena); - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(!upb_MiniTableField_IsExtension(field)); - upb_Message* sub_message = - *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*); - if (!sub_message) { - const upb_MiniTable* sub_mini_table = - upb_MiniTable_SubMessage(mini_table, field); - UPB_ASSERT(sub_mini_table); - sub_message = _upb_Message_New(sub_mini_table, arena); - *UPB_PTR_AT(msg, field->UPB_ONLYBITS(offset), upb_Message*) = sub_message; - UPB_PRIVATE(_upb_Message_SetPresence)(msg, field); - } - return sub_message; -} - -UPB_API_INLINE const upb_Array* upb_Message_GetArray( - const upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* ret; - const upb_Array* default_val = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - return ret; -} - -UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( - upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - return (upb_Array*)upb_Message_GetArray(msg, field); -} - -UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( - upb_Message* msg, const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSERT(arena); - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* array = upb_Message_GetMutableArray(msg, field); - if (!array) { - array = UPB_PRIVATE(_upb_Array_New)( - arena, 4, UPB_PRIVATE(_upb_MiniTableField_ElemSizeLg2)(field)); - // Check again due to: https://godbolt.org/z/7WfaoKG1r - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_MessageValue val; - val.array_val = array; - UPB_PRIVATE(_upb_Message_SetField)(msg, field, val, arena); - } - return array; + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (msg, mini_table, field, + UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); } UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( - upb_Message* msg, const upb_MiniTableField* field, size_t size, - upb_Arena* arena) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(field); - upb_Array* arr = upb_Message_GetOrCreateMutableArray(msg, field, arena); - if (!arr || !UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, size, arena)) { - return NULL; - } - return upb_Array_MutableDataPtr(arr); -} - -UPB_API_INLINE const upb_Map* upb_Message_GetMap( - const upb_Message* msg, const upb_MiniTableField* field) { - UPB_PRIVATE(_upb_MiniTableField_CheckIsMap)(field); - _upb_Message_AssertMapIsUntagged(msg, field); - upb_Map* ret; - const upb_Map* default_val = NULL; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &ret); - return ret; -} - -UPB_API_INLINE upb_Map* upb_Message_GetMutableMap( - upb_Message* msg, const upb_MiniTableField* field) { - return (upb_Map*)upb_Message_GetMap(msg, field); -} + upb_Message* msg, const upb_MiniTableField* f, size_t size, + upb_Arena* arena); -UPB_API_INLINE upb_Map* upb_Message_GetOrCreateMutableMap( - upb_Message* msg, const upb_MiniTable* map_entry_mini_table, - const upb_MiniTableField* field, upb_Arena* arena) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - const upb_MiniTableField* map_entry_key_field = - &map_entry_mini_table->UPB_ONLYBITS(fields)[0]; - const upb_MiniTableField* map_entry_value_field = - &map_entry_mini_table->UPB_ONLYBITS(fields)[1]; - return _upb_Message_GetOrCreateMutableMap( - msg, field, - _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_key_field)), - _upb_Map_CTypeSize(upb_MiniTableField_CType(map_entry_value_field)), - arena); -} +UPB_API_INLINE uint32_t upb_Message_WhichOneofFieldNumber( + const upb_Message* message, const upb_MiniTableField* oneof_field); // Updates a map entry given an entry message. bool upb_Message_SetMapEntry(upb_Map* map, const upb_MiniTable* mini_table, From 0a2e6625e2d43296a39f5a992503b402a6da8211 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 May 2024 15:43:22 -0700 Subject: [PATCH 101/179] Add copyright notice to test protos PiperOrigin-RevId: 630517509 --- conformance/test_protos/test_messages_edition2023.proto | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conformance/test_protos/test_messages_edition2023.proto b/conformance/test_protos/test_messages_edition2023.proto index 7688c47321..a6e2bcc863 100644 --- a/conformance/test_protos/test_messages_edition2023.proto +++ b/conformance/test_protos/test_messages_edition2023.proto @@ -1,3 +1,10 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + edition = "2023"; package protobuf_test_messages.editions; From 2f6e7055952eeff2fef2f29edb092c97e7949793 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 3 May 2024 22:57:16 +0000 Subject: [PATCH 102/179] Auto-generate files after cl/630517509 --- csharp/src/Google.Protobuf.Test/testprotos.pb | Bin 452970 -> 453266 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 2116b58c003620c789d914cc7d0b0bbab8ba29ab..255dfb472b7d2ea0ee9e525d1da0e4785066efe3 100644 GIT binary patch delta 8815 zcmYkC*>7FN9mns?8Q*(-uGjItZ`b?od+o&gZpV%t8*FEDY{#)35~`4d5JCvy0AZ8Z zW5BFpUn*59G^wC!X$1(XLRGbu`c|oZ?pyzY_B(TCeuoE3`QDlD_cQz1@}Iw#-26w0 zvM5aVSe`d^mnY%1aI+0NJZpXKiyeql)x38SPcnxmUWl`zGLQ`(4 z%cIQyr~hMGs9OK%N=NK`BS#E)w?uM1(Ax}ElBFoKiBfd^iIyKO!pA7_+JJ1=K^BXq7I_{oHf_`G&WmLG*#d2oPQFOIFZk4q1rmExKp0D}Uin;<>7T`VDHNP>z-t?VW!yT)y}3|M0Z(qMuZCP_+ns zDo_Kinq9X0*ZLx_j7R)Y*G(^r=39lRvZ7*StLng(^Ac0UF#P3YQ(Y)08-uN!Y(Q4< zGGpVgRq&`)uvMfMdRD?$QBrDf5S8R};HxB`17Brop|LrtCfS5^s>mi_t0J3#t%_^{ zwkmD&RPRu;ntUl**=DA|S53YY_^JtqWmLn}3dWk{Vc2STs#UPnxW!h%R#R4PIv0^n zoNQhMzBu_J@Wsg&fiF%zY`4+VQE70g(y8OsrYIG*IX$v879=iE!z#H4!cywkE=*!`4({X&KUK z)}8az+-1PmOufs1ubFz60beuqE>k+Kyw$WM6APe~XIceYYgWBgu(dX|m{w*}K+K7JO}`Nm~eC+rD1o%a%?Dw~^WKb?`J(F&nmybh{j~Ve5#iV^*zaDV^S+&J*WLYBi|h5$ zj>ty?dt+H9D<4_C*>;ELBdfPMZkkXaojzV`Di+}6>7$SWg!EBJ0YdsHqyQm(6jCUi zeqL|Sz(S<-Q%WIH`eWIKP>7U%X{WmoDg89vMba6dRZ@hI0SYNX$N+^DA!L9;iV!lO zLOi`_ilsA1DaA+`q?BT$3{pxlQU)od7%79)loII-QAi0whA5;2Awv{Wf{-B-;)Qq# zQidp{R64`F(e%6&DZ`Xfij?7)v}H&c&avyT6d}WP4Q7JMq%%SxWe6FekTQgfP)M1F zlo3iPL&^xHluKunmTx&yMk%EnDWkC*)0A?AjON+}QI3$&h9=XL3h9hdNCiU1D5L@* z-Wa7+AZ3hFDv&ZpDV5S0rxjO;lyOR_M9Mg=xJrbK(~7G^$T+RID(Otn1XUqqg1dJ8 zRUu`9W~d4&6Ew|LNSUAss+P_qx5HkIlu2qzH9{t1_E4-w$Yhqi1E@yGWM#D(l^W?x z>89v2xCSXxlv0C~DQZd$Ql_XWHAtDFro^Q)O`{S=$TWq-5i(7q5=Y21jY=FL(`Hn# z{%WN&!+XqB)*@wwQfhI$%*5>ZS&NjJq7GXVDKm*~)1f-)%<^$lRELmR3aLZLY%Ffe z5HedkXqAVQ*^x1mQZJo3ZUOa3nWL0?q|C+gjISOkbNNkHA!M$#-GnqqXP#R?148C0 zqyZuGF?*6UAjF%m-e*!8kTTylU{V^Tv%n`zQ6o|oD5Vi&yFe4vh?E6sFI0^PS-`c= zKad-7kT0~3n}fVj9pnpbwzEyrS>&7cQq=^+A|aZ9SR_Ofe2dba#Z5pgV)}eU6A+8F z34>@-h{d`tH&35y&C*%g{X;UKBY6pH)K6|k@>0zq|8|}? zN7eOYDL!w;r$su;e8E(0LF1PR(E`LWAzFa&mWk1Vlx1SHfU!)B7R^`&Lq8euRw-8a z;VuJd1!RRFt$?i12(<#TLL<})#tMy4YZP~Shr^wwBdr>8IHH!Bx?yOO;s`%#A#H#h zAxIk_M+nje$Pt3H0dj;OZRp5RVzg<-QDS)QQXJ#QEuLogiIOZ19s7(gnx{LAn6h zAV?P=8wL{MU0`ewqYI3aKEuO2cWKB;f+VEa1ikS&5F0ofu*5|Ax|BmvnXND`23f+RI$8xa4F zw_7?pv=h65*datW5Ifk3hTjdw4(-HlFm`Asc7w4)JF#0cc4#N|NO79qWe2MVkkbU| z0pv6-tsXE=)6(hz<8*c0US)d#xr;i|qak-uNA^kQ3|D(!-OKI+;|wwOfpLbetoy(? zlV$Ig_knSyywduBI8!xeHv2w}IOA0xM6ufK`=vO`@3tM;56D@9?8nvNEL~#u19CRs z&hvgi&NkbppZ$Q`P5s!fA$Ox6g__YT#W{XY68G9G_gvWA`q|F*?!o2{H)CWr7R>a+x55fZR`zK@GXzLimsrSNH>#F$BgH!@wCh1jrSF3;}Y5AVYv$ zA;=IQ4-jNXLmnW=uoPGMgBCIj$W`jdFc??qd>jVjs(*X0&cIo z2{IzZHU5x=i~w?tAR~ZWBghCK*XS8y1dMCM7y;uU&G7L4l8;yQoWLPi0( zPLNSRuG9HA3dnVOIv54yI-U;vvtksGhp8i@n(;6g>3Wxp$rO)2!dP~RYV z@d?d(JYac~GW-PNPZDerF#KR)lYl+JaMb$glYl+J@DyVtU{A2h8Y=;NGJs8L*pmTl zN`{|e{AprM3ArZli-k>vB_xhqKYa?Yrx*V)# z9#%;8kwWc_d01a!h0RvtnGPCnp8TuSAt`PK-b3(0TJIrvZwB5&u-*);3sUSd-fe;x zV1+EQ64qT-+-@bTyQ~XmiT_S#QHqxX?;^aA*1HJr%YkK&`{S*@@mu)fKr4jAiEDc)juW#Jbdg%#3H?NR0_U2Knyqwv1P@S?)U z9+Tp2hBp8vSRh*t-PyJyvKw+O;*t=NIUhbu)fD+ zooUr7tnaZ#^^QSVPe}2;#%k&bP(5giJpu3g0rmvE?+4g5DL!DL)C8}=3z=#qtRFDE zN$`hi4b~4>p}Ol<*mWsBWTMy1tM}B(yCKC#L6bM& zg|sc+fc2xG#T&4G6twuH6dwmIJ_#$NZShH1KW46Hn|u=9kAo&}O7Tg6-Gmp?VmIOa zB*1RM`bmI2CB>&qRGQwN!u&vHSPAQ=EYmK)Q}`eOU0UH5sc*f&{9^B){}Ux0zhLQR z`cu@>fi(9IDfnywnOopLKsdQF{7c4vwfEoq!v)?i8Q$P$=&!-K_;e{gQ{C5hEeK4*Hr zg~5QdHw;^F-wJLRwqU&#+%Rn68h(q#5Bk}BE1C!TEmk{gR>rmzw;4ZVqPG$4LEG$Y zWZ!04_6*!c_H9;eH}5vGZ?n32lfA98Z?pP2WOIB24S&Hxzrh#y7-oeb=?fNBZzqIc zYFNKy+@DVaQ|LV83K(=_)n_XTon;=nsCZp4Q*86un0zhpD(-hIs@};8!Bua+w}S|UNJxJWAyM~xJBE-*{a%fs-^M>N{U0z~m*XR__(3f$ z1y{ZH{$v};uo_U5=@;pGxG&Z9koJRGWLjVu{GgT?Mm>Vop&dV{1*XE0_oEG0!$jE} z*(mX|Rlj~zOIXEK;r|stnQ)HqA8k0-;XmG3Kk(H00fs-0KL>vTe<@$A9|)(t_kVM5 B&>8>$ delta 8841 zcmYkCd250IyOy5HdsRqZ?7&+~h``<-3KgnfTsl0|q<%%EBSN!ow#%c;|LGsh3Z?Wx zS32sae=r%6V0hnbxVVW4`zKmol|1y#>$jV3tXgRMmG7ecekJ<>b^6YZYhR~HK zk_LyMf^-ga6{K^ZtH>@lGDr0!nUGGLWCF4{$pmC^k_pJ-TIQ+Ip=W}0Svs<5W9VKKf+@3HJvX+W=V}eZ&VONjQ z6-lR!kC?V1OiCL~W)XC4*>+uvfNQI*vprFnZRufugX2Z&%FtGChfpk?3|IT8n&@J9 zX2`P`9lcDp-EoRxn!!w)t}x9cTa0P3GR;&rn?y^b)4{t;&k_uzOeJ3xR+`4q~gzMh^%N5t_rZb`x33kT{j8`eVx(n?YUJ9@7RNACa zCY>H$Z#tG?=jkDnGMMy`Nf}If$fOJ=J!DcYonGE(_rP+v^pZ*7`CMk&Y~;;qBcE%xgKd${Jm0pL zsum#T3DE+?JRw@3o0oPkZUJH*%jY9nfS9lEF^Cq0m~ZHIOZ2|hDxC$s=RI~RT;eT| zc`M8ps5V-mSSYk*+6wapl&Npt3iE}!CI5Dww?@?^Wg)$2rl?Iii+qpi+J^iW3DE|` zA|cv<@D_>D2A4%*w1Kfmj5f_!1VcX=@r)Epd{4K5WB^$rNCuE4nxqUMOEgItFqUYN zGEv;@?Fo08fn+pfPefIkx@TyYVwvx?kaj?p3DOS8GC|q_Stdw3Aj<@4$3XTHqg^xh z62t3|VjtgcAsvA1BS;4z`v}qj$UcH}0J4uD9f0g7NQZ{(CrGChEBt_Ebb_&B86NKh zWQ8D|fUFRt6Oa{xbOLgKAe|a=fFNB`9OQ>AqzjOP7Q(y0I7nBQE-(%j`KKE;nl3;N zV)OEAs|%1r1nJU{Lj>uTVwE4ZkZwR$3DOP7DnYsdSv8Ol?*?O)7~NnT_8A`5xm!aH z6QoCqHGafGdH`7?NDm-ubZzPZWUbil`aOWG;WFtDqz8~AG>{(6I6{nGDc1SXZrAfW z_X4s`kX}I63DOJ5Izf5?Stm#@AV&$(t06}L@o#$jq_aV1VjmD2gy;ie181V)_kpoN zXJQ{18+0c2fw4hnVxMMg(3#jT#U?*yC#xTjO@j0TvPq@Y55^{yRzDbpTd^@mBi)GziExjbl(lwlR)!%@~s6 z6o0rEx7!EKU)b6{d-BBX)u9~k6oG~?np3nv4gqwEuFpdNouZR*2%v|nmAj2B+2th`qIK$6c$Os^3 z2r>f58Cv=g4CD;mt&D(ihEB$jC?2@Z5@bX}&Jtu)igWx?3mFCE96?3_IY*FDK+X|l z6p(WS8O1;zCB~>`JZc#pACuxdf6PM0069;PF+k1}WDJn=1Q`S5JVC|)d5j=q8uFNh z@Np?F@W(A<9E=Nwfjw{>kP8GE2jl`l#sRrNka0jBC&;*lJWh}aDK7FSEMx+Zi!_i4 zFfP*mI042*{|;a6ffImSq-|vakS7Q-p&?HYWKxPt{7DO$1mqGyCIPubkV!x;(PP9U z7?+4K3C5F};o&78pVW{i39>_q%ls(|*#XF9g6sg~GVPB$0J%&L2s;3|j0XgNuh;>| zQ#6nrn(-7drlh#SpYFAL&=eq72r>o86@p9wa)lsMfLtNS6d+F%WJ*JxCdjlDSNSt` zAk$!6CB`%uS7V-ixSj^&DxHecfLx_haT<_k2r{i9&k$s%6xaB(7P1qNYXsQ|$Th01 zonTy}+S&=mH7{*%Q+ERLEJ1c^$g>2Qk>Wb{EMx|d>jaqr9003GvmTP-dB&S>|8HD+_)%nJ55e?#hAXab`Vd6VGhMf8Yn_wg z1&B=J9P&YErE`$J5J=}BeIby}OL2qYU77DY4=F-_DA)t$p}oOM8*MYRH<*5AP}p5k z+zhn4phak>VHcz~1L-bEZwAucQoI;QcSDNMN_Rv0A}jSQb~m&y2HFKFUJA4e&?297iw7-A$^&Zx7rX7e2DQ{%D;XsN%2acU4j;& z)hRQ?>#p^*??1dDeEsMR7z8;jtUPxaL%3_}sw>*ZoA^z0tgZ38V#WsZY z7AvW@A+)!c-g{J8?3dyVhKGG$x*t-6>N|{@|`Z2$PHS7d5HYf^l`M3tNC>73WHbZj@fHFW-f#cbWL!TJLh zSI^{1dqj#4nXtRb5oCqXF8vWmKV-7OWOW464_UK%0imSpQhcPbnz|0EhuC7*q5UYp zu0#7#fITY3$4n%R@lj|IX4??bj~QMw_)~Qh(vMlWx*u2A4JkfhB5oQtAVnC}A@;uw zXg^`u_MF*(_7hgBUScclrWBtBDQ`lH&}O^|>8C-)n~;7QWPD7D&w`APL5k33d<@dh znCsb;k3suckn)xkp9k12Xc1cM7POxS*eysu53t9j_=1TVGuq==AB1@}g!BuRZwv4^ zK1Cuqi4-|a*L>H>qZ7J?BzF?+c8#97XeUMh?>o(MP7;YL&2t?2wi&>7K>L1(2!XpQmN93hNjE{n+UPb~Cz{cj08f_#2v_@;;% zh^UhLiA79+5u+vIx4`aa7A^Awbw&T#UwbaN4FAGH|1guNxWBNddTT2LSEK#aP9h>$ zBJ?K_A^QGm=MW)We^qnn=lC1b|C-^t9G{8BJyl#PM>T5iH=88>vq3efe>gys?uRnd z0o?AXBGUoN;GQZm%z6abA&+~iz?3=s?pt#;P2|ntjTYZqjqAQDVP#i~e>*^A%sI^O gTXU|>@BjYB{LLHl41XMd4*mrGvVOcVAI|yW|ErU_x&QzG From b6e0a48b02ecc8ad3240009502886fbf8d1c5279 Mon Sep 17 00:00:00 2001 From: Jakob Buchgraber Date: Mon, 6 May 2024 05:16:05 -0700 Subject: [PATCH 103/179] Implement `IntoProxied` for repeated field setters We modify set_ to accept the IntoProxied type as the value and move the value (avoid copying) whenever possible. For UPB: - We fuse the arena of Repeated with the parent message arena. - We use upb_Message_SetBaseField to set the upb_Array contained in the Repeated. For C++: - We generate an additional setter thunk that moves the value. - The move assignment operator of RepeatedField/RepeatedPtrField is specialized. In order to adhere to the layering check we need to add '#include' statements for all .proto imports to the generated thunks.pb.cc. PiperOrigin-RevId: 631010333 --- rust/aspects.bzl | 6 ++- rust/cpp.rs | 4 ++ rust/repeated.rs | 47 ++++++++++++++++- rust/upb.rs | 8 +++ rust/upb/BUILD | 1 + rust/upb/lib.rs | 8 ++- rust/upb/message.rs | 8 ++- rust/upb/mini_table.rs | 10 ++++ rust/upb/upb_api.c | 10 ++-- .../compiler/rust/accessors/repeated_field.cc | 50 ++++++++++++++++--- .../protobuf/compiler/rust/generator.cc | 18 +++++-- src/google/protobuf/compiler/rust/message.cc | 16 +++++- 12 files changed, 166 insertions(+), 20 deletions(-) diff --git a/rust/aspects.bzl b/rust/aspects.bzl index ab72dc5aa9..bd5f1ca74f 100644 --- a/rust/aspects.bzl +++ b/rust/aspects.bzl @@ -324,13 +324,17 @@ def _rust_proto_aspect_common(target, ctx, is_upb): if is_upb: thunks_cc_info = target[UpbWrappedCcInfo].cc_info_with_thunks else: + dep_cc_infos = [] + for dep in proto_deps: + dep_cc_infos.append(dep[CcInfo]) + thunks_cc_info = cc_common.merge_cc_infos(cc_infos = [_compile_cc( feature_configuration = feature_configuration, src = thunk, ctx = ctx, attr = attr, cc_toolchain = cc_toolchain, - cc_infos = [target[CcInfo], ctx.attr._cpp_thunks_deps[CcInfo]], + cc_infos = [target[CcInfo], ctx.attr._cpp_thunks_deps[CcInfo]] + dep_cc_infos, ) for thunk in thunks]) runtime = proto_lang_toolchain.runtime diff --git a/rust/cpp.rs b/rust/cpp.rs index 2ad2a4ab30..1bd15266b8 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -301,6 +301,10 @@ impl InnerRepeated { pub fn as_mut(&mut self) -> InnerRepeatedMut<'_> { InnerRepeatedMut::new(Private, self.raw) } + + pub fn raw(&self) -> RawRepeatedField { + self.raw + } } /// The raw type-erased pointer version of `RepeatedMut`. diff --git a/rust/repeated.rs b/rust/repeated.rs index 617f1f3a1e..3183e42117 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -15,7 +15,7 @@ use std::iter::FusedIterator; use std::marker::PhantomData; use crate::{ - Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, + IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, __internal::Private, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -208,6 +208,49 @@ where } } +impl Repeated +where + T: ?Sized + ProxiedInRepeated, +{ + pub fn as_view(&self) -> View> { + RepeatedView { raw: self.inner.raw(), _phantom: PhantomData } + } + + #[doc(hidden)] + pub fn inner(&self, _private: Private) -> &InnerRepeated { + &self.inner + } +} + +impl IntoProxied> for Repeated +where + T: ?Sized + ProxiedInRepeated, +{ + fn into(self, _private: Private) -> Repeated { + self + } +} + +impl<'msg, T> IntoProxied> for RepeatedView<'msg, T> +where + T: 'msg + ?Sized + ProxiedInRepeated, +{ + fn into(self, _private: Private) -> Repeated { + let mut repeated: Repeated = Repeated::new(); + T::repeated_copy_from(self, repeated.as_mut()); + repeated + } +} + +impl<'msg, T> IntoProxied> for RepeatedMut<'msg, T> +where + T: 'msg + ?Sized + ProxiedInRepeated, +{ + fn into(self, _private: Private) -> Repeated { + IntoProxied::into(self.as_view(), _private) + } +} + /// Types that can appear in a `Repeated`. /// /// This trait is implemented by generated code to communicate how the proxied @@ -275,7 +318,7 @@ impl<'msg, T: ?Sized> Debug for RepeatedIter<'msg, T> { /// Users will generally write [`View>`](RepeatedView) or /// [`Mut>`](RepeatedMut) to access the repeated elements pub struct Repeated { - inner: InnerRepeated, + pub(crate) inner: InnerRepeated, _phantom: PhantomData, } diff --git a/rust/upb.rs b/rust/upb.rs index 6072349df1..895f47b417 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -155,6 +155,14 @@ impl InnerRepeated { pub fn as_mut(&mut self) -> InnerRepeatedMut<'_> { InnerRepeatedMut::new(Private, self.raw, &self.arena) } + + pub fn raw(&self) -> RawRepeatedField { + self.raw + } + + pub fn arena(&self) -> &Arena { + &self.arena + } } /// The raw type-erased pointer version of `RepeatedMut`. diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 13cea4d7f5..563794994b 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -46,5 +46,6 @@ cc_library( "//upb:mem", "//upb:message", "//upb:message_copy", + "//upb/mini_table", ], ) diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 040d98d736..299044f226 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -22,14 +22,18 @@ pub use map::{ mod message; pub use message::{ - upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_New, RawMessage, + upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_New, + upb_Message_SetBaseField, RawMessage, }; mod message_value; pub use message_value::{upb_MessageValue, upb_MutableMessageValue}; mod mini_table; -pub use mini_table::{upb_MiniTable, RawMiniTable}; +pub use mini_table::{ + upb_MiniTable, upb_MiniTableField, upb_MiniTable_FindFieldByNumber, RawMiniTable, + RawMiniTableField, +}; mod opaque_pointee; diff --git a/rust/upb/message.rs b/rust/upb/message.rs index fd831d7b6d..a4159a1fb6 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -1,5 +1,5 @@ use crate::opaque_pointee::opaque_pointee; -use crate::{upb_MiniTable, RawArena}; +use crate::{upb_MiniTable, upb_MiniTableField, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Message); @@ -22,4 +22,10 @@ extern "C" { mini_table: *const upb_MiniTable, arena: RawArena, ) -> Option; + + pub fn upb_Message_SetBaseField( + m: RawMessage, + mini_table: *const upb_MiniTableField, + val: *const std::ffi::c_void, + ); } diff --git a/rust/upb/mini_table.rs b/rust/upb/mini_table.rs index dc704caf0e..9aeca3e63a 100644 --- a/rust/upb/mini_table.rs +++ b/rust/upb/mini_table.rs @@ -3,3 +3,13 @@ use std::ptr::NonNull; opaque_pointee!(upb_MiniTable); pub type RawMiniTable = NonNull; + +opaque_pointee!(upb_MiniTableField); +pub type RawMiniTableField = NonNull; + +extern "C" { + pub fn upb_MiniTable_FindFieldByNumber( + m: *const upb_MiniTable, + number: u32, + ) -> *const upb_MiniTableField; +} diff --git a/rust/upb/upb_api.c b/rust/upb/upb_api.c index 6208be2464..e6d508db1d 100644 --- a/rust/upb/upb_api.c +++ b/rust/upb/upb_api.c @@ -10,9 +10,11 @@ #define UPB_BUILD_API -#include "upb/mem/arena.h" // IWYU pragma: keep -#include "upb/message/array.h" // IWYU pragma: keep -#include "upb/message/copy.h" // IWYU pragma: keep -#include "upb/message/map.h" // IWYU pragma: keep +#include "upb/mem/arena.h" // IWYU pragma: keep +#include "upb/message/array.h" // IWYU pragma: keep +#include "upb/message/copy.h" // IWYU pragma: keep +#include "upb/message/map.h" // IWYU pragma: keep +#include "upb/message/accessors.h" // IWYU pragma: keep +#include "upb/mini_table/message.h" // IWYU pragma: keep const size_t __rust_proto_kUpb_Map_Begin = kUpb_Map_Begin; diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index 11a85852de..82445759fd 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -104,17 +104,47 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, )rs"); } }}, + {"move_setter_thunk", ThunkName(ctx, field, "move_set")}, {"setter", [&] { if (accessor_case == AccessorCase::VIEW) { return; } - ctx.Emit({}, R"rs( - pub fn set_$raw_field_name$(&mut self, src: $pb$::RepeatedView<'_, $RsType$>) { - // TODO: Implement IntoProxied and avoid copying. - self.$field$_mut().copy_from(src); - } - )rs"); + if (ctx.is_upb()) { + ctx.Emit({{"field_number", field.number()}}, R"rs( + pub fn set_$raw_field_name$(&mut self, src: impl $pb$::IntoProxied<$pb$::Repeated<$RsType$>>) { + let minitable_field = unsafe { + $pbr$::upb_MiniTable_FindFieldByNumber( + Self::raw_minitable($pbi$::Private), + $field_number$ + ) + }; + let val = src.into($pbi$::Private); + let inner = val.inner($pbi$::Private); + + self.arena().fuse(inner.arena()); + unsafe { + let value_ptr: *const *const std::ffi::c_void = + &(inner.raw().as_ptr() as *const std::ffi::c_void); + $pbr$::upb_Message_SetBaseField(self.raw_msg(), + minitable_field, + value_ptr as *const std::ffi::c_void); + } + } + )rs"); + } else { + ctx.Emit({}, R"rs( + pub fn set_$raw_field_name$(&mut self, src: impl $pb$::IntoProxied<$pb$::Repeated<$RsType$>>) { + // Prevent the memory from being deallocated. The setter + // transfers ownership of the memory to the parent message. + let val = std::mem::ManuallyDrop::new(src.into($pbi$::Private)); + unsafe { + $move_setter_thunk$(self.raw_msg(), + val.inner($pbi$::Private).raw()); + } + } + )rs"); + } }}, }, R"rs( @@ -128,6 +158,7 @@ void RepeatedField::InExternC(Context& ctx, const FieldDescriptor& field) const { ctx.Emit({{"getter_thunk", ThunkName(ctx, field, "get")}, {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, + {"move_setter_thunk", ThunkName(ctx, field, "move_set")}, {"getter", [&] { if (ctx.is_upb()) { @@ -147,6 +178,7 @@ void RepeatedField::InExternC(Context& ctx, ctx.Emit(R"rs( fn $getter_mut_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::RawRepeatedField; fn $getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::RawRepeatedField; + fn $move_setter_thunk$(raw_msg: $pbr$::RawMessage, value: $pbr$::RawRepeatedField); )rs"); } }}, @@ -199,6 +231,7 @@ void RepeatedField::InThunkCc(Context& ctx, {"getter_mut_thunk", ThunkName(ctx, field, "get_mut")}, {"repeated_copy_from_thunk", ThunkName(ctx, field, "repeated_copy_from")}, + {"move_setter_thunk", ThunkName(ctx, field, "move_set")}, {"impls", [&] { ctx.Emit( @@ -213,6 +246,11 @@ void RepeatedField::InThunkCc(Context& ctx, const $QualifiedMsg$* msg) { return &msg->$field$(); } + void $move_setter_thunk$( + $QualifiedMsg$* msg, + $ContainerType$<$ElementType$>* value) { + *msg->mutable_$field$() = std::move(*value); + } )cc"); }}}, "$impls$"); diff --git a/src/google/protobuf/compiler/rust/generator.cc b/src/google/protobuf/compiler/rust/generator.cc index 71a8c1b107..49f1e65897 100644 --- a/src/google/protobuf/compiler/rust/generator.cc +++ b/src/google/protobuf/compiler/rust/generator.cc @@ -202,11 +202,23 @@ bool RustGenerator::Generate(const FileDescriptor* file, thunks_cc.reset(generator_context->Open(GetThunkCcFile(ctx, *file))); thunks_printer = std::make_unique(thunks_cc.get()); - thunks_printer->Emit({{"proto_h", GetHeaderFile(ctx, *file)}}, - R"cc( + thunks_printer->Emit( + {{"proto_h", GetHeaderFile(ctx, *file)}, + {"proto_deps_h", + [&] { + for (int i = 0; i < file->dependency_count(); i++) { + thunks_printer->Emit( + {{"proto_dep_h", GetHeaderFile(ctx, *file->dependency(i))}}, + R"cc( +#include "$proto_dep_h$" + )cc"); + } + }}}, + R"cc( #include "$proto_h$" + $proto_deps_h$ #include "google/protobuf/rust/cpp_kernel/cpp_api.h" - )cc"); + )cc"); } for (int i = 0; i < file->message_type_count(); ++i) { diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index ec0ab962f1..39236062ef 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -285,6 +285,16 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { ABSL_LOG(FATAL) << "unreachable"; } +void MessageGetMinitable(Context& ctx, const Descriptor& msg) { + if (ctx.opts().kernel == Kernel::kUpb) { + ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( + pub fn raw_minitable(_private: $pbi$::Private) -> *const $pbr$::upb_MiniTable { + unsafe { $std$::ptr::addr_of!($minitable$) } + } + )rs"); + } +} + void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: @@ -363,7 +373,6 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { } } } - )rs"); return; case Kernel::kUpb: @@ -821,6 +830,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } }}, {"into_proxied_impl", [&] { IntoProxiedForMessage(ctx, msg); }}, + {"get_upb_minitable", [&] { MessageGetMinitable(ctx, msg); }}, {"repeated_impl", [&] { MessageProxiedInRepeated(ctx, msg); }}, {"map_value_impl", [&] { MessageProxiedInMapValue(ctx, msg); }}, {"unwrap_upb", @@ -976,6 +986,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $pb$::ViewProxy::as_view(self).serialize() } + $get_upb_minitable$ + $raw_arena_getter_for_msgmut$ $accessor_fns_for_muts$ @@ -1043,6 +1055,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $Msg$Mut::new($pbi$::Private, &mut self.inner) } + $get_upb_minitable$ + $accessor_fns$ } // impl $Msg$ From 18306666b902274dfad3cca47e69c4bddbead9e3 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Mon, 6 May 2024 06:19:10 -0700 Subject: [PATCH 104/179] upb: clean up the UPB_TRACING_ENABLED code PiperOrigin-RevId: 631023575 --- upb/message/internal/message.c | 20 +++++++++----------- upb/message/internal/message.h | 15 ++++++++------- upb/message/message.h | 16 ++++++---------- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/upb/message/internal/message.c b/upb/message/internal/message.c index dc31f13e22..57ab2d7371 100644 --- a/upb/message/internal/message.c +++ b/upb/message/internal/message.c @@ -60,18 +60,16 @@ bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, } #if UPB_TRACING_ENABLED -static void (*_new_message_trace_handler)(const upb_MiniTable*, - const upb_Arena*); +static void (*_message_trace_handler)(const upb_MiniTable*, const upb_Arena*); -void UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)( - void (*new_message_trace_handler)(const upb_MiniTable*, const upb_Arena*)) { - _new_message_trace_handler = new_message_trace_handler; +void upb_Message_LogNewMessage(const upb_MiniTable* m, const upb_Arena* arena) { + if (_message_trace_handler) { + _message_trace_handler(m, arena); + } } -void UPB_PRIVATE(upb_Message_LogNewMessage)(const upb_MiniTable* mini_table, - const upb_Arena* arena) { - if (_new_message_trace_handler) { - _new_message_trace_handler(mini_table, arena); - } +void upb_Message_SetNewMessageTraceHandler(void (*handler)(const upb_MiniTable*, + const upb_Arena*)) { + _message_trace_handler = handler; } -#endif +#endif // UPB_TRACING_ENABLED diff --git a/upb/message/internal/message.h b/upb/message/internal/message.h index a81bb4f0b0..b371bbe0c8 100644 --- a/upb/message/internal/message.h +++ b/upb/message/internal/message.h @@ -60,18 +60,19 @@ typedef struct upb_Message_Internal { } upb_Message_Internal; #ifdef UPB_TRACING_ENABLED -void UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)( - void (*newMessageTraceHandler)(const upb_MiniTable*, const upb_Arena*)); -void UPB_PRIVATE(upb_Message_LogNewMessage)(const upb_MiniTable* mini_table, - const upb_Arena* arena); -#endif +UPB_API void upb_Message_LogNewMessage(const upb_MiniTable* m, + const upb_Arena* arena); +UPB_API void upb_Message_SetNewMessageTraceHandler( + void (*handler)(const upb_MiniTable*, const upb_Arena*)); +#endif // UPB_TRACING_ENABLED // Inline version upb_Message_New(), for internal use. UPB_INLINE struct upb_Message* _upb_Message_New(const upb_MiniTable* m, upb_Arena* a) { #ifdef UPB_TRACING_ENABLED - UPB_PRIVATE(upb_Message_LogNewMessage)(m, a); -#endif + upb_Message_LogNewMessage(m, a); +#endif // UPB_TRACING_ENABLED + const int size = m->UPB_PRIVATE(size); struct upb_Message* msg = (struct upb_Message*)upb_Arena_Malloc(a, size); if (UPB_UNLIKELY(!msg)) return NULL; diff --git a/upb/message/message.h b/upb/message/message.h index cbcca84a72..50388fcb43 100644 --- a/upb/message/message.h +++ b/upb/message/message.h @@ -47,16 +47,12 @@ UPB_API void upb_Message_Freeze(upb_Message* msg, const upb_MiniTable* m); UPB_API_INLINE bool upb_Message_IsFrozen(const upb_Message* msg); #ifdef UPB_TRACING_ENABLED -UPB_INLINE void upb_Message_SetNewMessageTraceHandler( - void (*newMessageTraceHandler)(const upb_MiniTable* mini_table, - const upb_Arena* arena)) { - UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)(newMessageTraceHandler); -} -UPB_INLINE void upb_Message_LogNewMessage(const upb_MiniTable* mini_table, - const upb_Arena* arena) { - UPB_PRIVATE(upb_Message_LogNewMessage)(mini_table, arena); -} -#endif +UPB_API void upb_Message_LogNewMessage(const upb_MiniTable* m, + const upb_Arena* arena); + +UPB_API void upb_Message_SetNewMessageTraceHandler( + void (*handler)(const upb_MiniTable* m, const upb_Arena* arena)); +#endif // UPB_TRACING_ENABLED #ifdef __cplusplus } /* extern "C" */ From 51cba7ce170cf5ed88cc408ff7aa4b35aa0dd3b5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 May 2024 13:34:40 +0000 Subject: [PATCH 105/179] Auto-generate files after cl/631023575 --- php/ext/google/protobuf/php-upb.c | 20 ++++++++--------- php/ext/google/protobuf/php-upb.h | 31 ++++++++++++--------------- ruby/ext/google/protobuf_c/ruby-upb.c | 20 ++++++++--------- ruby/ext/google/protobuf_c/ruby-upb.h | 31 ++++++++++++--------------- 4 files changed, 46 insertions(+), 56 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index fe3bbb76cf..fa3be023d6 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -12061,21 +12061,19 @@ bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, } #if UPB_TRACING_ENABLED -static void (*_new_message_trace_handler)(const upb_MiniTable*, - const upb_Arena*); +static void (*_message_trace_handler)(const upb_MiniTable*, const upb_Arena*); -void UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)( - void (*new_message_trace_handler)(const upb_MiniTable*, const upb_Arena*)) { - _new_message_trace_handler = new_message_trace_handler; +void upb_Message_LogNewMessage(const upb_MiniTable* m, const upb_Arena* arena) { + if (_message_trace_handler) { + _message_trace_handler(m, arena); + } } -void UPB_PRIVATE(upb_Message_LogNewMessage)(const upb_MiniTable* mini_table, - const upb_Arena* arena) { - if (_new_message_trace_handler) { - _new_message_trace_handler(mini_table, arena); - } +void upb_Message_SetNewMessageTraceHandler(void (*handler)(const upb_MiniTable*, + const upb_Arena*)) { + _message_trace_handler = handler; } -#endif +#endif // UPB_TRACING_ENABLED const char _kUpb_ToBase92[] = { diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index a3d60bc576..21448b174c 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -2470,18 +2470,19 @@ typedef struct upb_Message_Internal { } upb_Message_Internal; #ifdef UPB_TRACING_ENABLED -void UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)( - void (*newMessageTraceHandler)(const upb_MiniTable*, const upb_Arena*)); -void UPB_PRIVATE(upb_Message_LogNewMessage)(const upb_MiniTable* mini_table, - const upb_Arena* arena); -#endif +UPB_API void upb_Message_LogNewMessage(const upb_MiniTable* m, + const upb_Arena* arena); +UPB_API void upb_Message_SetNewMessageTraceHandler( + void (*handler)(const upb_MiniTable*, const upb_Arena*)); +#endif // UPB_TRACING_ENABLED // Inline version upb_Message_New(), for internal use. UPB_INLINE struct upb_Message* _upb_Message_New(const upb_MiniTable* m, upb_Arena* a) { #ifdef UPB_TRACING_ENABLED - UPB_PRIVATE(upb_Message_LogNewMessage)(m, a); -#endif + upb_Message_LogNewMessage(m, a); +#endif // UPB_TRACING_ENABLED + const int size = m->UPB_PRIVATE(size); struct upb_Message* msg = (struct upb_Message*)upb_Arena_Malloc(a, size); if (UPB_UNLIKELY(!msg)) return NULL; @@ -3436,16 +3437,12 @@ UPB_API void upb_Message_Freeze(upb_Message* msg, const upb_MiniTable* m); UPB_API_INLINE bool upb_Message_IsFrozen(const upb_Message* msg); #ifdef UPB_TRACING_ENABLED -UPB_INLINE void upb_Message_SetNewMessageTraceHandler( - void (*newMessageTraceHandler)(const upb_MiniTable* mini_table, - const upb_Arena* arena)) { - UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)(newMessageTraceHandler); -} -UPB_INLINE void upb_Message_LogNewMessage(const upb_MiniTable* mini_table, - const upb_Arena* arena) { - UPB_PRIVATE(upb_Message_LogNewMessage)(mini_table, arena); -} -#endif +UPB_API void upb_Message_LogNewMessage(const upb_MiniTable* m, + const upb_Arena* arena); + +UPB_API void upb_Message_SetNewMessageTraceHandler( + void (*handler)(const upb_MiniTable* m, const upb_Arena* arena)); +#endif // UPB_TRACING_ENABLED #ifdef __cplusplus } /* extern "C" */ diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 989c51d2a4..91fada38f9 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -11553,21 +11553,19 @@ bool UPB_PRIVATE(_upb_Message_Realloc)(struct upb_Message* msg, size_t need, } #if UPB_TRACING_ENABLED -static void (*_new_message_trace_handler)(const upb_MiniTable*, - const upb_Arena*); +static void (*_message_trace_handler)(const upb_MiniTable*, const upb_Arena*); -void UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)( - void (*new_message_trace_handler)(const upb_MiniTable*, const upb_Arena*)) { - _new_message_trace_handler = new_message_trace_handler; +void upb_Message_LogNewMessage(const upb_MiniTable* m, const upb_Arena* arena) { + if (_message_trace_handler) { + _message_trace_handler(m, arena); + } } -void UPB_PRIVATE(upb_Message_LogNewMessage)(const upb_MiniTable* mini_table, - const upb_Arena* arena) { - if (_new_message_trace_handler) { - _new_message_trace_handler(mini_table, arena); - } +void upb_Message_SetNewMessageTraceHandler(void (*handler)(const upb_MiniTable*, + const upb_Arena*)) { + _message_trace_handler = handler; } -#endif +#endif // UPB_TRACING_ENABLED const char _kUpb_ToBase92[] = { diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index f5cf050c34..ddb5106d45 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -2472,18 +2472,19 @@ typedef struct upb_Message_Internal { } upb_Message_Internal; #ifdef UPB_TRACING_ENABLED -void UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)( - void (*newMessageTraceHandler)(const upb_MiniTable*, const upb_Arena*)); -void UPB_PRIVATE(upb_Message_LogNewMessage)(const upb_MiniTable* mini_table, - const upb_Arena* arena); -#endif +UPB_API void upb_Message_LogNewMessage(const upb_MiniTable* m, + const upb_Arena* arena); +UPB_API void upb_Message_SetNewMessageTraceHandler( + void (*handler)(const upb_MiniTable*, const upb_Arena*)); +#endif // UPB_TRACING_ENABLED // Inline version upb_Message_New(), for internal use. UPB_INLINE struct upb_Message* _upb_Message_New(const upb_MiniTable* m, upb_Arena* a) { #ifdef UPB_TRACING_ENABLED - UPB_PRIVATE(upb_Message_LogNewMessage)(m, a); -#endif + upb_Message_LogNewMessage(m, a); +#endif // UPB_TRACING_ENABLED + const int size = m->UPB_PRIVATE(size); struct upb_Message* msg = (struct upb_Message*)upb_Arena_Malloc(a, size); if (UPB_UNLIKELY(!msg)) return NULL; @@ -3438,16 +3439,12 @@ UPB_API void upb_Message_Freeze(upb_Message* msg, const upb_MiniTable* m); UPB_API_INLINE bool upb_Message_IsFrozen(const upb_Message* msg); #ifdef UPB_TRACING_ENABLED -UPB_INLINE void upb_Message_SetNewMessageTraceHandler( - void (*newMessageTraceHandler)(const upb_MiniTable* mini_table, - const upb_Arena* arena)) { - UPB_PRIVATE(upb_Message_SetNewMessageTraceHandler)(newMessageTraceHandler); -} -UPB_INLINE void upb_Message_LogNewMessage(const upb_MiniTable* mini_table, - const upb_Arena* arena) { - UPB_PRIVATE(upb_Message_LogNewMessage)(mini_table, arena); -} -#endif +UPB_API void upb_Message_LogNewMessage(const upb_MiniTable* m, + const upb_Arena* arena); + +UPB_API void upb_Message_SetNewMessageTraceHandler( + void (*handler)(const upb_MiniTable* m, const upb_Arena* arena)); +#endif // UPB_TRACING_ENABLED #ifdef __cplusplus } /* extern "C" */ From e03c1ce3493318565b376999391fee4259846f03 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 May 2024 11:16:11 -0700 Subject: [PATCH 106/179] In SerialArenaChunk, (a) avoid recomputing offset of ids/arenas on each element access in constructor, (b) add a no-arg Layout() function for convenience, (c) fix an incorrect construction type in `new (&arena(i)) std::atomic`, where std::atomic should be std::atomic. PiperOrigin-RevId: 631125248 --- src/google/protobuf/arena.cc | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 4137615964..720e0924a9 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -373,16 +373,21 @@ struct SerialArenaChunkHeader { class ThreadSafeArena::SerialArenaChunk { public: SerialArenaChunk(uint32_t capacity, void* me, SerialArena* serial) { - new (&header()) SerialArenaChunkHeader{capacity, 1}; + // We use `layout`/`ids`/`arenas` local variables to avoid recomputing + // offsets if we were to call id(i)/arena(i) repeatedly. + const layout_type layout = Layout(capacity); + new (layout.Pointer(ptr())) SerialArenaChunkHeader{capacity, 1}; - new (&id(0)) std::atomic{me}; + std::atomic* ids = layout.Pointer(ptr()); + new (&ids[0]) std::atomic{me}; for (uint32_t i = 1; i < capacity; ++i) { - new (&id(i)) std::atomic{nullptr}; + new (&ids[i]) std::atomic{nullptr}; } - new (&arena(0)) std::atomic{serial}; + std::atomic* arenas = layout.Pointer(ptr()); + new (&arenas[0]) std::atomic{serial}; for (uint32_t i = 1; i < capacity; ++i) { - new (&arena(i)) std::atomic{nullptr}; + new (&arenas[i]) std::atomic{nullptr}; } } @@ -401,30 +406,30 @@ class ThreadSafeArena::SerialArenaChunk { // ids: returns up to size(). absl::Span> ids() const { - return Layout(capacity()).Slice(ptr()).first(safe_size()); + return Layout().Slice(ptr()).first(safe_size()); } absl::Span> ids() { - return Layout(capacity()).Slice(ptr()).first(safe_size()); + return Layout().Slice(ptr()).first(safe_size()); } std::atomic& id(uint32_t i) { ABSL_DCHECK_LT(i, capacity()); - return Layout(capacity()).Pointer(ptr())[i]; + return Layout().Pointer(ptr())[i]; } // arenas: returns up to size(). absl::Span> arenas() const { - return Layout(capacity()).Slice(ptr()).first(safe_size()); + return Layout().Slice(ptr()).first(safe_size()); } absl::Span> arenas() { - return Layout(capacity()).Slice(ptr()).first(safe_size()); + return Layout().Slice(ptr()).first(safe_size()); } const std::atomic& arena(uint32_t i) const { ABSL_DCHECK_LT(i, capacity()); - return Layout(capacity()).Pointer(ptr())[i]; + return Layout().Pointer(ptr())[i]; } std::atomic& arena(uint32_t i) { ABSL_DCHECK_LT(i, capacity()); - return Layout(capacity()).Pointer(ptr())[i]; + return Layout().Pointer(ptr())[i]; } // Tries to insert {id, serial} to head chunk. Returns false if the head is @@ -481,6 +486,7 @@ class ThreadSafeArena::SerialArenaChunk { constexpr static layout_type Layout(size_t n) { return layout_type(/*header*/ 1, /*ids*/ n, /*arenas*/ n); } + layout_type Layout() const { return Layout(capacity()); } }; constexpr SerialArenaChunkHeader kSentryArenaChunk = {0, 0}; From e949bba22a3b9fe4280ec924d61e98a65c884c94 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 May 2024 11:31:42 -0700 Subject: [PATCH 107/179] Clean up port_def.inc by inlining BSWAP macros that are only used once. PiperOrigin-RevId: 631130592 --- src/google/protobuf/endian.h | 14 ++++++++------ src/google/protobuf/port_def.inc | 21 --------------------- src/google/protobuf/port_undef.inc | 3 --- 3 files changed, 8 insertions(+), 30 deletions(-) diff --git a/src/google/protobuf/endian.h b/src/google/protobuf/endian.h index 781954e499..6bc90c0dce 100644 --- a/src/google/protobuf/endian.h +++ b/src/google/protobuf/endian.h @@ -14,6 +14,8 @@ #include +#include "absl/base/config.h" + // Must be included last. #include "google/protobuf/port_def.inc" @@ -22,8 +24,8 @@ namespace protobuf { namespace internal { inline uint64_t BSwap64(uint64_t host_int) { -#if defined(PROTOBUF_BUILTIN_BSWAP64) - return PROTOBUF_BUILTIN_BSWAP64(host_int); +#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_bswap64) + return __builtin_bswap64(host_int); #elif defined(_MSC_VER) return _byteswap_uint64(host_int); #else @@ -39,8 +41,8 @@ inline uint64_t BSwap64(uint64_t host_int) { } inline uint32_t BSwap32(uint32_t host_int) { -#if defined(PROTOBUF_BUILTIN_BSWAP32) - return PROTOBUF_BUILTIN_BSWAP32(host_int); +#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_bswap32) + return __builtin_bswap32(host_int); #elif defined(_MSC_VER) return _byteswap_ulong(host_int); #else @@ -52,8 +54,8 @@ inline uint32_t BSwap32(uint32_t host_int) { } inline uint16_t BSwap16(uint16_t host_int) { -#if defined(PROTOBUF_BUILTIN_BSWAP16) - return PROTOBUF_BUILTIN_BSWAP16(host_int); +#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_bswap16) + return __builtin_bswap16(host_int); #elif defined(_MSC_VER) return _byteswap_ushort(host_int); #else diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 2b1371865e..bc6e18b039 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -67,27 +67,6 @@ // - MSVC: https://docs.microsoft.com/en-us/visualstudio/releases/2019/release-notes-history // https://docs.microsoft.com/en-us/visualstudio/releasenotes/vs2017-relnotes-history -// Portable PROTOBUF_BUILTIN_BSWAPxx definitions -// Code must check for availability, e.g.: `defined(PROTOBUF_BUILTIN_BSWAP32)` -#ifdef PROTOBUF_BUILTIN_BSWAP16 -#error PROTOBUF_BUILTIN_BSWAP16 was previously defined -#endif -#ifdef PROTOBUF_BUILTIN_BSWAP32 -#error PROTOBUF_BUILTIN_BSWAP32 was previously defined -#endif -#ifdef PROTOBUF_BUILTIN_BSWAP64 -#error PROTOBUF_BUILTIN_BSWAP64 was previously defined -#endif -#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_bswap16) -#define PROTOBUF_BUILTIN_BSWAP16(x) __builtin_bswap16(x) -#endif -#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_bswap32) -#define PROTOBUF_BUILTIN_BSWAP32(x) __builtin_bswap32(x) -#endif -#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_bswap64) -#define PROTOBUF_BUILTIN_BSWAP64(x) __builtin_bswap64(x) -#endif - // Portable check for gcc-style atomic built-ins #if ABSL_HAVE_BUILTIN(__atomic_load_n) #define PROTOBUF_BUILTIN_ATOMIC 1 diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 93c13dfdbf..eb9fa83cb8 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -16,9 +16,6 @@ #undef PROTOBUF_POISON_MEMORY_REGION #undef PROTOBUF_UNPOISON_MEMORY_REGION -#undef PROTOBUF_BUILTIN_BSWAP16 -#undef PROTOBUF_BUILTIN_BSWAP32 -#undef PROTOBUF_BUILTIN_BSWAP64 #undef PROTOBUF_BUILTIN_ATOMIC #undef PROTOBUF_GNUC_MIN #undef PROTOBUF_CLANG_MIN From 24f27c3b88708a735968688d34aa02a28f403866 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Mon, 6 May 2024 12:10:59 -0700 Subject: [PATCH 108/179] Nextgen Proto Pythonic API: Add 'in' operator (Second attempt. The first attempt missed ListValue) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The “in” operator will be consistent with HasField but a little different with Proto Plus. The detail behavior of “in” operator in Nextgen * For WKT Struct (to be consist with old Struct behavior): -Raise TypeError if not pass a string -Check if the key is in the struct.fields * For WKT ListValue (to be consist with old behavior): -Check if the key is in the list_value.values * For other messages: -Raise ValueError if not pass a string -Raise ValueError if the string is not a field -For Oneof: Check any field under the oneof is set -For has-presence field: check if set -For non-has-presence field (include repeated fields): raise ValueError PiperOrigin-RevId: 631143378 --- .../google/protobuf/internal/message_test.py | 31 ++++++++++ .../protobuf/internal/python_message.py | 16 +++++ .../protobuf/internal/well_known_types.py | 3 - .../internal/well_known_types_test.py | 31 +++++++++- python/google/protobuf/message.py | 28 +++++++++ python/google/protobuf/pyext/message.cc | 59 ++++++++++++++++++- python/message.c | 31 ++++++++++ 7 files changed, 194 insertions(+), 5 deletions(-) diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 7b4478e245..e42538e1a7 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -1336,6 +1336,24 @@ class MessageTest(unittest.TestCase): union.DESCRIPTOR, message_module.TestAllTypes.NestedEnum.DESCRIPTOR ) + def testIn(self, message_module): + m = message_module.TestAllTypes() + self.assertNotIn('optional_nested_message', m) + self.assertNotIn('oneof_bytes', m) + self.assertNotIn('oneof_string', m) + with self.assertRaises(ValueError) as e: + 'repeated_int32' in m + with self.assertRaises(ValueError) as e: + 'repeated_nested_message' in m + with self.assertRaises(ValueError) as e: + 1 in m + with self.assertRaises(ValueError) as e: + 'not_a_field' in m + test_util.SetAllFields(m) + self.assertIn('optional_nested_message', m) + self.assertIn('oneof_bytes', m) + self.assertNotIn('oneof_string', m) + # Class to test proto2-only features (required, extensions, etc.) @testing_refleaks.TestCase @@ -1367,6 +1385,9 @@ class Proto2Test(unittest.TestCase): self.assertTrue(message.HasField('optional_int32')) self.assertTrue(message.HasField('optional_bool')) self.assertTrue(message.HasField('optional_nested_message')) + self.assertIn('optional_int32', message) + self.assertIn('optional_bool', message) + self.assertIn('optional_nested_message', message) # Set the fields to non-default values. message.optional_int32 = 5 @@ -1385,6 +1406,9 @@ class Proto2Test(unittest.TestCase): self.assertFalse(message.HasField('optional_int32')) self.assertFalse(message.HasField('optional_bool')) self.assertFalse(message.HasField('optional_nested_message')) + self.assertNotIn('optional_int32', message) + self.assertNotIn('optional_bool', message) + self.assertNotIn('optional_nested_message', message) self.assertEqual(0, message.optional_int32) self.assertEqual(False, message.optional_bool) self.assertEqual(0, message.optional_nested_message.bb) @@ -1711,6 +1735,12 @@ class Proto3Test(unittest.TestCase): with self.assertRaises(ValueError): message.HasField('repeated_nested_message') + # Can not test "in" operator. + with self.assertRaises(ValueError): + 'repeated_int32' in message + with self.assertRaises(ValueError): + 'repeated_nested_message' in message + # Fields should default to their type-specific default. self.assertEqual(0, message.optional_int32) self.assertEqual(0, message.optional_float) @@ -1721,6 +1751,7 @@ class Proto3Test(unittest.TestCase): # Setting a submessage should still return proper presence information. message.optional_nested_message.bb = 0 self.assertTrue(message.HasField('optional_nested_message')) + self.assertIn('optional_nested_message', message) # Set the fields to non-default values. message.optional_int32 = 5 diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 34efbfea06..869f2aa731 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -1000,6 +1000,21 @@ def _AddUnicodeMethod(unused_message_descriptor, cls): cls.__unicode__ = __unicode__ +def _AddContainsMethod(message_descriptor, cls): + + if message_descriptor.full_name == 'google.protobuf.Struct': + def __contains__(self, key): + return key in self.fields + elif message_descriptor.full_name == 'google.protobuf.ListValue': + def __contains__(self, value): + return value in self.items() + else: + def __contains__(self, field): + return self.HasField(field) + + cls.__contains__ = __contains__ + + def _BytesForNonRepeatedElement(value, field_number, field_type): """Returns the number of bytes needed to serialize a non-repeated element. The returned byte count includes space for tag information and any @@ -1394,6 +1409,7 @@ def _AddMessageMethods(message_descriptor, cls): _AddStrMethod(message_descriptor, cls) _AddReprMethod(message_descriptor, cls) _AddUnicodeMethod(message_descriptor, cls) + _AddContainsMethod(message_descriptor, cls) _AddByteSizeMethod(message_descriptor, cls) _AddSerializeToStringMethod(message_descriptor, cls) _AddSerializePartialToStringMethod(message_descriptor, cls) diff --git a/python/google/protobuf/internal/well_known_types.py b/python/google/protobuf/internal/well_known_types.py index 8a0a3b13dd..ea90be3be6 100644 --- a/python/google/protobuf/internal/well_known_types.py +++ b/python/google/protobuf/internal/well_known_types.py @@ -497,9 +497,6 @@ class Struct(object): def __getitem__(self, key): return _GetStructValue(self.fields[key]) - def __contains__(self, item): - return item in self.fields - def __setitem__(self, key, value): _SetStructValue(self.fields[key], value) diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 498d3f28ab..592776634d 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -515,7 +515,6 @@ class StructTest(unittest.TestCase): self.assertEqual(False, struct_list[3]) self.assertEqual(None, struct_list[4]) self.assertEqual(inner_struct, struct_list[5]) - self.assertIn(6, struct_list) struct_list[1] = 7 self.assertEqual(7, struct_list[1]) @@ -570,6 +569,36 @@ class StructTest(unittest.TestCase): self.assertEqual([6, True, False, None, inner_struct], list(struct['key5'].items())) + def testInOperator(self): + # in operator for Struct + struct = struct_pb2.Struct() + struct['key'] = 5 + + self.assertIn('key', struct) + self.assertNotIn('fields', struct) + with self.assertRaises(TypeError) as e: + 1 in struct + + # in operator for ListValue + struct_list = struct.get_or_create_list('key2') + self.assertIsInstance(struct_list, collections_abc.Sequence) + struct_list.extend([6, 'seven', True, False, None]) + struct_list.add_struct()['subkey'] = 9 + inner_struct = struct.__class__() + inner_struct['subkey'] = 9 + + self.assertIn(6, struct_list) + self.assertIn('seven', struct_list) + self.assertIn(True, struct_list) + self.assertIn(False, struct_list) + self.assertIn(None, struct_list) + self.assertIn(inner_struct, struct_list) + self.assertNotIn('values', struct_list) + self.assertNotIn(10, struct_list) + + for item in struct_list: + self.assertIn(item, struct_list) + def testStructAssignment(self): # Tests struct assignment from another struct s1 = struct_pb2.Struct() diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py index 3226b6e776..ae9cb147e3 100755 --- a/python/google/protobuf/message.py +++ b/python/google/protobuf/message.py @@ -75,6 +75,34 @@ class Message(object): """Outputs a human-readable representation of the message.""" raise NotImplementedError + def __contains__(self, field_name_or_key): + """Checks if a certain field is set for the message. + + Has presence fields return true if the field is set, false if the field is + not set. Fields without presence do raise `ValueError` (this includes + repeated fields, map fields, and implicit presence fields). + + If field_name is not defined in the message descriptor, `ValueError` will + be raised. + Note: WKT Struct checks if the key is contained in fields. ListValue checks + if the item is contained in the list. + + Args: + field_name_or_key: For Struct, the key (str) of the fields map. For + ListValue, any type that may be contained in the list. For other + messages, name of the field (str) to check for presence. + + Returns: + bool: For Struct, whether the item is contained in fields. For ListValue, + whether the item is contained in the list. For other message, + whether a value has been set for the named field. + + Raises: + ValueError: For normal messages, if the `field_name_or_key` is not a + member of this message or `field_name_or_key` is not a string. + """ + raise NotImplementedError + def MergeFrom(self, other_msg): """Merges the contents of the specified message into current message. diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 06f95157c8..39fe35a941 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -10,6 +10,7 @@ #include "google/protobuf/pyext/message.h" +#include #include // A Python header file. #include @@ -36,6 +37,7 @@ #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/strtod.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/map_field.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" #include "google/protobuf/unknown_field_set.h" @@ -85,6 +87,12 @@ class MessageReflectionFriend { return reflection->IsLazyField(field) || reflection->IsLazyExtension(message, field); } + static bool ContainsMapKey(const Reflection* reflection, + const Message& message, + const FieldDescriptor* field, + const MapKey& map_key) { + return reflection->ContainsMapKey(message, field, map_key); + } }; static PyObject* kDESCRIPTOR; @@ -1293,11 +1301,16 @@ PyObject* HasField(CMessage* self, PyObject* arg) { char* field_name; Py_ssize_t size; field_name = const_cast(PyUnicode_AsUTF8AndSize(arg, &size)); + Message* message = self->message; + if (!field_name) { + PyErr_Format(PyExc_ValueError, + "The field name passed to message %s" + " is not a str.", + message->GetDescriptor()->name().c_str()); return nullptr; } - Message* message = self->message; bool is_in_oneof; const FieldDescriptor* field_descriptor = FindFieldWithOneofs( message, absl::string_view(field_name, size), &is_in_oneof); @@ -2290,6 +2303,48 @@ PyObject* ToUnicode(CMessage* self) { return decoded; } +PyObject* Contains(CMessage* self, PyObject* arg) { + Message* message = self->message; + const Descriptor* descriptor = message->GetDescriptor(); + switch (descriptor->well_known_type()) { + case Descriptor::WELLKNOWNTYPE_STRUCT: { + // For WKT Struct, check if the key is in the fields. + const Reflection* reflection = message->GetReflection(); + const FieldDescriptor* map_field = descriptor->FindFieldByName("fields"); + const FieldDescriptor* key_field = map_field->message_type()->map_key(); + PyObject* py_string = CheckString(arg, key_field); + if (!py_string) { + PyErr_SetString(PyExc_TypeError, + "The key passed to Struct message must be a str."); + return nullptr; + } + char* value; + Py_ssize_t value_len; + if (PyBytes_AsStringAndSize(py_string, &value, &value_len) < 0) { + Py_DECREF(py_string); + Py_RETURN_FALSE; + } + std::string key_str; + key_str.assign(value, value_len); + Py_DECREF(py_string); + + MapKey map_key; + map_key.SetStringValue(key_str); + return PyBool_FromLong(MessageReflectionFriend::ContainsMapKey( + reflection, *message, map_field, map_key)); + } + case Descriptor::WELLKNOWNTYPE_LISTVALUE: { + // For WKT ListValue, check if the key is in the items. + PyObject* items = PyObject_CallMethod(reinterpret_cast(self), + "items", nullptr); + return PyBool_FromLong(PySequence_Contains(items, arg)); + } + default: + // For other messages, check with HasField. + return HasField(self, arg); + } +} + // CMessage static methods: PyObject* _CheckCalledFromGeneratedFile(PyObject* unused, PyObject* unused_arg) { @@ -2338,6 +2393,8 @@ static PyMethodDef Methods[] = { "Makes a deep copy of the class."}, {"__unicode__", (PyCFunction)ToUnicode, METH_NOARGS, "Outputs a unicode representation of the message."}, + {"__contains__", (PyCFunction)Contains, METH_O, + "Checks if a message field is set."}, {"ByteSize", (PyCFunction)ByteSize, METH_NOARGS, "Returns the size of the message in bytes."}, {"Clear", (PyCFunction)Clear, METH_NOARGS, "Clears the message."}, diff --git a/python/message.c b/python/message.c index c0c0882d24..d5213ab1af 100644 --- a/python/message.c +++ b/python/message.c @@ -1044,6 +1044,35 @@ static PyObject* PyUpb_Message_HasField(PyObject* _self, PyObject* arg) { NULL); } +static PyObject* PyUpb_Message_Contains(PyObject* _self, PyObject* arg) { + const upb_MessageDef* msgdef = PyUpb_Message_GetMsgdef(_self); + switch (upb_MessageDef_WellKnownType(msgdef)) { + case kUpb_WellKnown_Struct: { + // For WKT Struct, check if the key is in the fields. + PyUpb_Message* self = (void*)_self; + if (PyUpb_Message_IsStub(self)) Py_RETURN_FALSE; + upb_Message* msg = PyUpb_Message_GetMsg(self); + const upb_FieldDef* f = upb_MessageDef_FindFieldByName(msgdef, "fields"); + const upb_Map* map = upb_Message_GetFieldByDef(msg, f).map_val; + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + upb_MessageValue u_key; + if (!PyUpb_PyToUpb(arg, key_f, &u_key, NULL)) return NULL; + return PyBool_FromLong(upb_Map_Get(map, u_key, NULL)); + } + case kUpb_WellKnown_ListValue: { + // For WKT ListValue, check if the key is in the items. + PyUpb_Message* self = (void*)_self; + if (PyUpb_Message_IsStub(self)) Py_RETURN_FALSE; + PyObject* items = PyObject_CallMethod(_self, "items", NULL); + return PyBool_FromLong(PySequence_Contains(items, arg)); + } + default: + // For other messages, check with HasField. + return PyUpb_Message_HasField(_self, arg); + } +} + static PyObject* PyUpb_Message_FindInitializationErrors(PyObject* _self, PyObject* arg); @@ -1642,6 +1671,8 @@ static PyMethodDef PyUpb_Message_Methods[] = { // TODO //{ "__unicode__", (PyCFunction)ToUnicode, METH_NOARGS, // "Outputs a unicode representation of the message." }, + {"__contains__", PyUpb_Message_Contains, METH_O, + "Checks if a message field is set."}, {"ByteSize", (PyCFunction)PyUpb_Message_ByteSize, METH_NOARGS, "Returns the size of the message in bytes."}, {"Clear", (PyCFunction)PyUpb_Message_Clear, METH_NOARGS, From f70d90b4743dd9eed05e5bf33ef7196a99894e29 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 May 2024 13:49:32 -0700 Subject: [PATCH 109/179] Put arena cleanup nodes on a separate chunked list instead of at the ends of arena blocks. The motivation is (a) better data locality during SerialArena::CleanupList and (b) simplification of arena layout. PiperOrigin-RevId: 631173641 --- src/google/protobuf/BUILD.bazel | 1 + src/google/protobuf/arena.cc | 191 +++++++++++++++--------- src/google/protobuf/arena_cleanup.h | 86 +++++++---- src/google/protobuf/arena_unittest.cc | 42 +++++- src/google/protobuf/serial_arena.h | 113 ++++++-------- src/google/protobuf/thread_safe_arena.h | 2 + 6 files changed, 253 insertions(+), 182 deletions(-) diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 917d9d35d0..e33b4292d5 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -1185,6 +1185,7 @@ cc_test( }), deps = [ ":arena", + ":arena_cleanup", ":cc_test_protos", ":lite_test_util", ":port", diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 720e0924a9..55d87ab6bb 100644 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -60,27 +60,43 @@ ArenaBlock* SentryArenaBlock() { } #endif -SizedPtr AllocateMemory(const AllocationPolicy* policy_ptr, size_t last_size, - size_t min_bytes) { +inline size_t AllocationSize(size_t last_size, size_t start_size, + size_t max_size) { + if (last_size == 0) return start_size; + // Double the current block size, up to a limit. + return std::min(2 * last_size, max_size); +} + +SizedPtr AllocateMemory(const AllocationPolicy& policy, size_t size) { + if (policy.block_alloc == nullptr) { + return AllocateAtLeast(size); + } + return {policy.block_alloc(size), size}; +} + +SizedPtr AllocateBlock(const AllocationPolicy* policy_ptr, size_t last_size, + size_t min_bytes) { AllocationPolicy policy; // default policy if (policy_ptr) policy = *policy_ptr; - size_t size; - if (last_size != 0) { - // Double the current block size, up to a limit. - auto max_size = policy.max_block_size; - size = std::min(2 * last_size, max_size); - } else { - size = policy.start_block_size; - } + size_t size = + AllocationSize(last_size, policy.start_block_size, policy.max_block_size); // Verify that min_bytes + kBlockHeaderSize won't overflow. ABSL_CHECK_LE(min_bytes, std::numeric_limits::max() - SerialArena::kBlockHeaderSize); size = std::max(size, SerialArena::kBlockHeaderSize + min_bytes); - if (policy.block_alloc == nullptr) { - return AllocateAtLeast(size); - } - return {policy.block_alloc(size), size}; + return AllocateMemory(policy, size); +} + +SizedPtr AllocateCleanupChunk(const AllocationPolicy* policy_ptr, + size_t last_size) { + constexpr size_t kStartSize = 64; + constexpr size_t kMaxSize = 4 << 10; + static_assert(kStartSize % sizeof(cleanup::CleanupNode) == 0, ""); + + const size_t size = AllocationSize(last_size, kStartSize, kMaxSize); + if (policy_ptr == nullptr) return AllocateAtLeast(size); + return AllocateMemory(*policy_ptr, size); } class GetDeallocator { @@ -102,6 +118,88 @@ class GetDeallocator { } // namespace +namespace cleanup { +struct ChunkList::Chunk { + CleanupNode* First() { return reinterpret_cast(this + 1); } + CleanupNode* Last() { return First() + Capacity() - 1; } + static size_t Capacity(size_t size) { + return (size - sizeof(Chunk)) / sizeof(CleanupNode); + } + size_t Capacity() const { return Capacity(size); } + + Chunk* next; + size_t size; + // Cleanup nodes follow. +}; + +void ChunkList::AddFallback(void* elem, void (*destructor)(void*), + SerialArena& arena) { + ABSL_DCHECK_EQ(next_, limit_); + SizedPtr mem = AllocateCleanupChunk(arena.parent_.AllocPolicy(), + head_ == nullptr ? 0 : head_->size); + arena.AddSpaceAllocated(mem.n); + head_ = new (mem.p) Chunk{head_, mem.n}; + next_ = head_->First(); + prefetch_ptr_ = reinterpret_cast(next_); + limit_ = next_ + Chunk::Capacity(mem.n); + AddFromExisting(elem, destructor); +} + +void ChunkList::Cleanup(const SerialArena& arena) { + Chunk* c = head_; + if (c == nullptr) return; + GetDeallocator deallocator(arena.parent_.AllocPolicy()); + + // Iterate backwards in order to destroy in the right order. + CleanupNode* it = next_ - 1; + while (true) { + CleanupNode* first = c->First(); + // A prefetch distance of 8 here was chosen arbitrarily. + constexpr int kPrefetchDistance = 8; + CleanupNode* prefetch = it; + // Prefetch the first kPrefetchDistance nodes. + for (int i = 0; prefetch >= first && i < kPrefetchDistance; + --prefetch, ++i) { + prefetch->Prefetch(); + } + // For the middle nodes, run destructor and prefetch the node + // kPrefetchDistance after the current one. + for (; prefetch >= first; --it, --prefetch) { + it->Destroy(); + prefetch->Prefetch(); + } + // Note: we could consider prefetching `next` chunk earlier. + absl::PrefetchToLocalCacheNta(c->next); + // Destroy the rest without prefetching. + for (; it >= first; --it) { + it->Destroy(); + } + Chunk* next = c->next; + deallocator({c, c->size}); + if (next == nullptr) return; + c = next; + it = c->Last(); + }; +} + +std::vector ChunkList::PeekForTesting() { + std::vector ret; + Chunk* c = head_; + if (c == nullptr) return ret; + // Iterate backwards to match destruction order. + CleanupNode* it = next_ - 1; + while (true) { + CleanupNode* first = c->First(); + for (; it >= first; --it) { + ret.push_back(it->elem); + } + c = c->next; + if (c == nullptr) return ret; + it = c->Last(); + }; +} +} // namespace cleanup + // It is guaranteed that this is constructed in `b`. IOW, this is not the first // arena and `b` cannot be sentry. SerialArena::SerialArena(ArenaBlock* b, ThreadSafeArena& parent) @@ -109,7 +207,6 @@ SerialArena::SerialArena(ArenaBlock* b, ThreadSafeArena& parent) limit_{b->Limit()}, prefetch_ptr_( b->Pointer(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize)), - prefetch_limit_(b->Limit()), head_{b}, space_allocated_{b->size}, parent_{parent} { @@ -130,22 +227,7 @@ SerialArena::SerialArena(FirstSerialArena, ArenaBlock* b, } std::vector SerialArena::PeekCleanupListForTesting() { - std::vector res; - - ArenaBlock* b = head(); - if (b->IsSentry()) return res; - - const auto peek_list = [&](char* pos, char* end) { - for (; pos != end; pos += cleanup::Size()) { - cleanup::PeekNode(pos, res); - } - }; - - peek_list(limit_, b->Limit()); - for (b = b->next; b; b = b->next) { - peek_list(reinterpret_cast(b->cleanup_nodes), b->Limit()); - } - return res; + return cleanup_list_.PeekForTesting(); } std::vector ThreadSafeArena::PeekCleanupListForTesting() { @@ -223,25 +305,16 @@ void* SerialArena::AllocateFromStringBlockFallback() { PROTOBUF_NOINLINE void* SerialArena::AllocateAlignedWithCleanupFallback( size_t n, size_t align, void (*destructor)(void*)) { - size_t required = AlignUpTo(n, align) + cleanup::Size(); + size_t required = AlignUpTo(n, align); AllocateNewBlock(required); return AllocateAlignedWithCleanup(n, align, destructor); } -PROTOBUF_NOINLINE -void SerialArena::AddCleanupFallback(void* elem, void (*destructor)(void*)) { - AllocateNewBlock(cleanup::Size()); - AddCleanupFromExisting(elem, destructor); -} - void SerialArena::AllocateNewBlock(size_t n) { size_t used = 0; size_t wasted = 0; ArenaBlock* old_head = head(); if (!old_head->IsSentry()) { - // Sync limit to block - old_head->cleanup_nodes = limit_; - // Record how much used in this block. used = static_cast(ptr() - old_head->Pointer(kBlockHeaderSize)); wasted = old_head->size - used - kBlockHeaderSize; @@ -253,7 +326,7 @@ void SerialArena::AllocateNewBlock(size_t n) { // but with a CPU regression. The regression might have been an artifact of // the microbenchmark. - auto mem = AllocateMemory(parent_.AllocPolicy(), old_head->size, n); + auto mem = AllocateBlock(parent_.AllocPolicy(), old_head->size, n); AddSpaceAllocated(mem.n); ThreadSafeArenaStats::RecordAllocateStats(parent_.arena_stats_.MutableStats(), /*used=*/used, @@ -314,34 +387,6 @@ size_t SerialArena::FreeStringBlocks(StringBlock* string_block, return deallocated; } -void SerialArena::CleanupList() { - ArenaBlock* b = head(); - if (b->IsSentry()) return; - - b->cleanup_nodes = limit_; - do { - char* limit = b->Limit(); - char* it = reinterpret_cast(b->cleanup_nodes); - ABSL_DCHECK(!b->IsSentry() || it == limit); - // A prefetch distance of 8 here was chosen arbitrarily. - char* prefetch = it; - int prefetch_dist = 8; - for (; prefetch < limit && --prefetch_dist; prefetch += cleanup::Size()) { - cleanup::PrefetchNode(prefetch); - } - for (; prefetch < limit; - it += cleanup::Size(), prefetch += cleanup::Size()) { - cleanup::DestroyNode(it); - cleanup::PrefetchNode(prefetch); - } - absl::PrefetchToLocalCacheNta(b->next); - for (; it < limit; it += cleanup::Size()) { - cleanup::DestroyNode(it); - } - b = b->next; - } while (b); -} - // Stores arrays of void* and SerialArena* instead of linked list of // SerialArena* to speed up traversing all SerialArena. The cost of walk is non // trivial when there are many nodes. Separately storing "ids" minimizes cache @@ -550,7 +595,7 @@ ArenaBlock* ThreadSafeArena::FirstBlock(void* buf, size_t size, SizedPtr mem; if (buf == nullptr || size < kBlockHeaderSize + kAllocPolicySize) { - mem = AllocateMemory(&policy, 0, kAllocPolicySize); + mem = AllocateBlock(&policy, 0, kAllocPolicySize); } else { mem = {buf, size}; // Record user-owned block. @@ -734,6 +779,8 @@ uint64_t ThreadSafeArena::Reset() { // Have to do this in a first pass, because some of the destructors might // refer to memory in other blocks. CleanupList(); + // Reset the first arena's cleanup list. + first_arena_.cleanup_list_ = cleanup::ChunkList(); // Discard all blocks except the first one. Whether it is user-provided or // allocated, always reuse the first block for the first arena. @@ -913,7 +960,7 @@ SerialArena* ThreadSafeArena::GetSerialArenaFallback(size_t n) { // have any blocks yet. So we'll allocate its first block now. It must be // big enough to host SerialArena and the pending request. serial = SerialArena::New( - AllocateMemory(alloc_policy_.get(), 0, n + kSerialArenaSize), *this); + AllocateBlock(alloc_policy_.get(), 0, n + kSerialArenaSize), *this); AddSerialArena(id, serial); } diff --git a/src/google/protobuf/arena_cleanup.h b/src/google/protobuf/arena_cleanup.h index abd2b73349..a153e36667 100644 --- a/src/google/protobuf/arena_cleanup.h +++ b/src/google/protobuf/arena_cleanup.h @@ -21,6 +21,9 @@ namespace google { namespace protobuf { namespace internal { + +class SerialArena; + namespace cleanup { // Helper function invoking the destructor of `object` @@ -33,44 +36,61 @@ void arena_destruct_object(void* object) { // destroyed, and the function to destroy it (`destructor`) // elem must be aligned at minimum on a 4 byte boundary. struct CleanupNode { + // Optimization: performs a prefetch on the elem for the cleanup node. We + // explicitly use NTA prefetch here to avoid polluting remote caches: we are + // destroying these instances, there is no purpose for these cache lines to + // linger around in remote caches. + ABSL_ATTRIBUTE_ALWAYS_INLINE void Prefetch() { + // TODO: we should also prefetch the destructor code once + // processors support code prefetching. + absl::PrefetchToLocalCacheNta(elem); + } + + // Destroys the object referenced by the cleanup node. + ABSL_ATTRIBUTE_ALWAYS_INLINE void Destroy() { destructor(elem); } + void* elem; void (*destructor)(void*); }; -inline ABSL_ATTRIBUTE_ALWAYS_INLINE CleanupNode* ToCleanup(void* pos) { - return reinterpret_cast(pos); -} - -// Adds a cleanup entry at memory location `pos`. -inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CreateNode(void* pos, void* elem, +// Manages the list of cleanup nodes in a chunked linked list. Chunks grow by +// factors of two up to a limit. Trivially destructible, but Cleanup() must be +// called before destruction. +class ChunkList { + public: + PROTOBUF_ALWAYS_INLINE void Add(void* elem, void (*destructor)(void*), + SerialArena& arena) { + if (PROTOBUF_PREDICT_TRUE(next_ < limit_)) { + AddFromExisting(elem, destructor); + return; + } + AddFallback(elem, destructor, arena); + } + + // Runs all inserted cleanups and frees allocated chunks. Must be called + // before destruction. + void Cleanup(const SerialArena& arena); + + private: + struct Chunk; + friend class internal::SerialArena; + + void AddFallback(void* elem, void (*destructor)(void*), SerialArena& arena); + ABSL_ATTRIBUTE_ALWAYS_INLINE void AddFromExisting(void* elem, void (*destructor)(void*)) { - CleanupNode n = {elem, destructor}; - memcpy(pos, &n, sizeof(n)); -} - -// Optimization: performs a prefetch on the elem for the cleanup node at `pos`. -inline ABSL_ATTRIBUTE_ALWAYS_INLINE void PrefetchNode(void* pos) { - // We explicitly use NTA prefetch here to avoid polluting remote caches: we - // are destroying these instances, there is no purpose for these cache lines - // to linger around in remote caches. - absl::PrefetchToLocalCacheNta(ToCleanup(pos)->elem); -} - -// Destroys the object referenced by the cleanup node. -inline ABSL_ATTRIBUTE_ALWAYS_INLINE void DestroyNode(void* pos) { - CleanupNode* cleanup = ToCleanup(pos); - cleanup->destructor(cleanup->elem); -} - -// Append in `out` the pointer to the to-be-cleaned object in `pos`. -inline void PeekNode(void* pos, std::vector& out) { - out.push_back(ToCleanup(pos)->elem); -} - -// Returns the required size for a cleanup node. -constexpr ABSL_ATTRIBUTE_ALWAYS_INLINE size_t Size() { - return sizeof(CleanupNode); -} + *next_++ = CleanupNode{elem, destructor}; + } + + // Returns the pointers to the to-be-cleaned objects. + std::vector PeekForTesting(); + + Chunk* head_ = nullptr; + CleanupNode* next_ = nullptr; + CleanupNode* limit_ = nullptr; + // Current prefetch position. Data from `next_` up to but not including + // `prefetch_ptr_` is software prefetched. Used in SerialArena prefetching. + const char* prefetch_ptr_ = nullptr; +}; } // namespace cleanup } // namespace internal diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index 0752735a48..3cc55c93b0 100644 --- a/src/google/protobuf/arena_unittest.cc +++ b/src/google/protobuf/arena_unittest.cc @@ -31,6 +31,7 @@ #include "absl/strings/string_view.h" #include "absl/synchronization/barrier.h" #include "absl/utility/utility.h" +#include "google/protobuf/arena_cleanup.h" #include "google/protobuf/arena_test_util.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/extension_set.h" @@ -1406,12 +1407,12 @@ TEST(ArenaTest, RepeatedFieldOnArena) { // Preallocate an initial arena block to avoid mallocs during hooked region. std::vector arena_block(1024 * 1024); Arena arena(arena_block.data(), arena_block.size()); + const size_t initial_allocated_size = arena.SpaceAllocated(); { - internal::NoHeapChecker no_heap; - - // Fill some repeated fields on the arena to test for leaks. Also verify no - // memory allocations. + // Fill some repeated fields on the arena to test for leaks. Also that the + // newly allocated memory is approximately the size of the cleanups for the + // repeated messages. RepeatedField repeated_int32(&arena); RepeatedPtrField repeated_message(&arena); for (int i = 0; i < 100; i++) { @@ -1432,10 +1433,14 @@ TEST(ArenaTest, RepeatedFieldOnArena) { repeated_message.UnsafeArenaExtractSubrange(0, 5, extracted_messages); EXPECT_EQ(&arena, repeated_message.Get(0).GetArena()); EXPECT_EQ(5, repeated_message.size()); + // Upper bound of the size of the cleanups of new repeated messages. + const size_t upperbound_cleanup_size = + 2 * 110 * sizeof(internal::cleanup::CleanupNode); + EXPECT_GT(initial_allocated_size + upperbound_cleanup_size, + arena.SpaceAllocated()); } - // Now, outside the scope of the NoHeapChecker, test ExtractSubrange's copying - // semantics. + // Now test ExtractSubrange's copying semantics. { RepeatedPtrField repeated_message(&arena); for (int i = 0; i < 100; i++) { @@ -1577,8 +1582,11 @@ TEST(ArenaTest, NoHeapAllocationsTest) { Arena arena(options); { - + // We need to call Arena::Create before NoHeapChecker because the ArenaDtor + // allocates a new cleanup chunk. TestAllTypes* message = Arena::Create(&arena); + + FillArenaAwareFields(message); } @@ -1610,8 +1618,9 @@ TEST(ArenaTest, MessageLiteOnArena) { initial_message.SerializeToString(&serialized); { - MessageLite* generic_message = prototype->New(&arena); + + EXPECT_TRUE(generic_message != nullptr); EXPECT_EQ(&arena, generic_message->GetArena()); EXPECT_TRUE(generic_message->ParseFromString(serialized)); @@ -1679,6 +1688,23 @@ TEST(ArenaTest, FirstArenaOverhead) { } +TEST(ArenaTest, StartingBlockSize) { + Arena default_arena; + EXPECT_EQ(0, default_arena.SpaceAllocated()); + + // Allocate something to get starting block size. + Arena::CreateArray(&default_arena, 1); + ArenaOptions options; + // First block size should be the default starting block size. + EXPECT_EQ(default_arena.SpaceAllocated(), options.start_block_size); + + // Use a custom starting block size. + options.start_block_size *= 2; + Arena custom_arena(options); + Arena::CreateArray(&custom_arena, 1); + EXPECT_EQ(custom_arena.SpaceAllocated(), options.start_block_size); +} + TEST(ArenaTest, BlockSizeDoubling) { Arena arena; EXPECT_EQ(0, arena.SpaceUsed()); diff --git a/src/google/protobuf/serial_arena.h b/src/google/protobuf/serial_arena.h index 0ccc410a76..84472df189 100644 --- a/src/google/protobuf/serial_arena.h +++ b/src/google/protobuf/serial_arena.h @@ -37,13 +37,10 @@ namespace internal { // Arena blocks are variable length malloc-ed objects. The following structure // describes the common header for all blocks. struct ArenaBlock { - // For the sentry block with zero-size where ptr_, limit_, cleanup_nodes all - // point to "this". - constexpr ArenaBlock() - : next(nullptr), cleanup_nodes(this), size(0) {} + // For the sentry block with zero-size where ptr_/limit_ both point to `this`. + constexpr ArenaBlock() : next(nullptr), size(0) {} - ArenaBlock(ArenaBlock* next, size_t size) - : next(next), cleanup_nodes(nullptr), size(size) { + ArenaBlock(ArenaBlock* next, size_t size) : next(next), size(size) { ABSL_DCHECK_GT(size, sizeof(ArenaBlock)); } @@ -56,7 +53,6 @@ struct ArenaBlock { bool IsSentry() const { return size == 0; } ArenaBlock* const next; - void* cleanup_nodes; const size_t size; // data follows }; @@ -86,7 +82,7 @@ class PROTOBUF_EXPORT SerialArena { static constexpr size_t kBlockHeaderSize = ArenaAlignDefault::Ceil(sizeof(ArenaBlock)); - void CleanupList(); + void CleanupList() { cleanup_list_.Cleanup(*this); } uint64_t SpaceAllocated() const { return space_allocated_.load(std::memory_order_relaxed); } @@ -218,7 +214,7 @@ class PROTOBUF_EXPORT SerialArena { *out = ret; char* next = ret + n; set_ptr(next); - MaybePrefetchForwards(next); + MaybePrefetchData(next); return true; } @@ -236,27 +232,23 @@ class PROTOBUF_EXPORT SerialArena { n = ArenaAlignDefault::Ceil(n); char* ret = ArenaAlignAs(align).CeilDefaultAligned(ptr()); // See the comment in MaybeAllocateAligned re uintptr_t. - if (PROTOBUF_PREDICT_FALSE(reinterpret_cast(ret) + n + - cleanup::Size() > + if (PROTOBUF_PREDICT_FALSE(reinterpret_cast(ret) + n > reinterpret_cast(limit_))) { return AllocateAlignedWithCleanupFallback(n, align, destructor); } PROTOBUF_UNPOISON_MEMORY_REGION(ret, n); char* next = ret + n; set_ptr(next); - AddCleanupFromExisting(ret, destructor); + AddCleanup(ret, destructor); ABSL_DCHECK_GE(limit_, ptr()); - MaybePrefetchForwards(next); + MaybePrefetchData(next); return ret; } PROTOBUF_ALWAYS_INLINE void AddCleanup(void* elem, void (*destructor)(void*)) { - size_t has = static_cast(limit_ - ptr()); - if (PROTOBUF_PREDICT_FALSE(cleanup::Size() > has)) { - return AddCleanupFallback(elem, destructor); - } - AddCleanupFromExisting(elem, destructor); + cleanup_list_.Add(elem, destructor, *this); + MaybePrefetchCleanup(); } ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlock(); @@ -265,6 +257,7 @@ class PROTOBUF_EXPORT SerialArena { private: friend class ThreadSafeArena; + friend class cleanup::ChunkList; // See comments for cached_blocks_. struct CachedBlock { @@ -272,8 +265,8 @@ class PROTOBUF_EXPORT SerialArena { CachedBlock* next; }; - static constexpr ptrdiff_t kPrefetchForwardsDegree = ABSL_CACHELINE_SIZE * 16; - static constexpr ptrdiff_t kPrefetchBackwardsDegree = ABSL_CACHELINE_SIZE * 6; + static constexpr ptrdiff_t kPrefetchDataDegree = ABSL_CACHELINE_SIZE * 16; + static constexpr ptrdiff_t kPrefetchCleanupDegree = ABSL_CACHELINE_SIZE * 6; // Constructor is private as only New() should be used. inline SerialArena(ArenaBlock* b, ThreadSafeArena& parent); @@ -285,59 +278,41 @@ class PROTOBUF_EXPORT SerialArena { bool MaybeAllocateString(void*& p); ABSL_ATTRIBUTE_RETURNS_NONNULL void* AllocateFromStringBlockFallback(); + // Prefetch the next prefetch_degree bytes after `prefetch_ptr` and + // up to `limit`, if `next` is within prefetch_degree bytes of `prefetch_ptr`. PROTOBUF_ALWAYS_INLINE - void AddCleanupFromExisting(void* elem, void (*destructor)(void*)) { - const size_t cleanup_size = cleanup::Size(); - - PROTOBUF_UNPOISON_MEMORY_REGION(limit_ - cleanup_size, cleanup_size); - limit_ -= cleanup_size; - MaybePrefetchBackwards(limit_); - ABSL_DCHECK_GE(limit_, ptr()); - cleanup::CreateNode(limit_, elem, destructor); - } - - // Prefetch the next kPrefetchForwardsDegree bytes after `prefetch_ptr_` and - // up to `prefetch_limit_`, if `next` is within kPrefetchForwardsDegree bytes - // of `prefetch_ptr_`. - PROTOBUF_ALWAYS_INLINE - void MaybePrefetchForwards(const char* next) { - ABSL_DCHECK(static_cast(prefetch_ptr_) == nullptr || - static_cast(prefetch_ptr_) >= head()); - if (PROTOBUF_PREDICT_TRUE(prefetch_ptr_ - next > kPrefetchForwardsDegree)) - return; - if (PROTOBUF_PREDICT_TRUE(prefetch_ptr_ < prefetch_limit_)) { - const char* prefetch_ptr = std::max(next, prefetch_ptr_); + static const char* MaybePrefetchImpl(const ptrdiff_t prefetch_degree, + const char* next, const char* limit, + const char* prefetch_ptr) { + if (PROTOBUF_PREDICT_TRUE(prefetch_ptr - next > prefetch_degree)) + return prefetch_ptr; + if (PROTOBUF_PREDICT_TRUE(prefetch_ptr < limit)) { + prefetch_ptr = std::max(next, prefetch_ptr); ABSL_DCHECK(prefetch_ptr != nullptr); - const char* end = - std::min(prefetch_limit_, prefetch_ptr + ABSL_CACHELINE_SIZE * 16); + const char* end = std::min(limit, prefetch_ptr + prefetch_degree); for (; prefetch_ptr < end; prefetch_ptr += ABSL_CACHELINE_SIZE) { absl::PrefetchToLocalCacheForWrite(prefetch_ptr); } - prefetch_ptr_ = prefetch_ptr; } + return prefetch_ptr; } - PROTOBUF_ALWAYS_INLINE - // Prefetch up to kPrefetchBackwardsDegree before `prefetch_limit_` and after - // `prefetch_ptr_`, if `limit` is within kPrefetchBackwardsDegree of - // `prefetch_limit_`. - void MaybePrefetchBackwards(const char* limit) { - ABSL_DCHECK(prefetch_limit_ == nullptr || - static_cast(prefetch_limit_) <= - static_cast(head()->Limit())); - if (PROTOBUF_PREDICT_TRUE(limit - prefetch_limit_ > - kPrefetchBackwardsDegree)) - return; - if (PROTOBUF_PREDICT_TRUE(prefetch_limit_ > prefetch_ptr_)) { - const char* prefetch_limit = std::min(limit, prefetch_limit_); - ABSL_DCHECK_NE(prefetch_limit, nullptr); - const char* end = - std::max(prefetch_ptr_, prefetch_limit - kPrefetchBackwardsDegree); - for (; prefetch_limit > end; prefetch_limit -= ABSL_CACHELINE_SIZE) { - absl::PrefetchToLocalCacheForWrite(prefetch_limit); - } - prefetch_limit_ = prefetch_limit; - } + void MaybePrefetchData(const char* next) { + ABSL_DCHECK(static_cast(prefetch_ptr_) == nullptr || + static_cast(prefetch_ptr_) >= head()); + prefetch_ptr_ = + MaybePrefetchImpl(kPrefetchDataDegree, next, limit_, prefetch_ptr_); + } + PROTOBUF_ALWAYS_INLINE + void MaybePrefetchCleanup() { + ABSL_DCHECK(static_cast(cleanup_list_.prefetch_ptr_) == + nullptr || + static_cast(cleanup_list_.prefetch_ptr_) >= + cleanup_list_.head_); + cleanup_list_.prefetch_ptr_ = MaybePrefetchImpl( + kPrefetchCleanupDegree, reinterpret_cast(cleanup_list_.next_), + reinterpret_cast(cleanup_list_.limit_), + cleanup_list_.prefetch_ptr_); } // Creates a new SerialArena inside mem using the remaining memory as for @@ -385,7 +360,6 @@ class PROTOBUF_EXPORT SerialArena { set_ptr(ptr); prefetch_ptr_ = ptr; limit_ = limit; - prefetch_limit_ = limit; } void* AllocateAlignedFallback(size_t n); @@ -405,10 +379,11 @@ class PROTOBUF_EXPORT SerialArena { // Limiting address up to which memory can be allocated from the head block. char* limit_ = nullptr; // Current prefetch positions. Data from `ptr_` up to but not including - // `prefetch_ptr_` is software prefetched. Similarly, data from `limit_` down - // to but not including `prefetch_limit_` is software prefetched. + // `prefetch_ptr_` is software prefetched. const char* prefetch_ptr_ = nullptr; - const char* prefetch_limit_ = nullptr; + + // Chunked linked list for managing cleanup for arena elements. + cleanup::ChunkList cleanup_list_; // The active string block. std::atomic string_block_{nullptr}; diff --git a/src/google/protobuf/thread_safe_arena.h b/src/google/protobuf/thread_safe_arena.h index 2332f937a5..2065ee98e8 100644 --- a/src/google/protobuf/thread_safe_arena.h +++ b/src/google/protobuf/thread_safe_arena.h @@ -20,6 +20,7 @@ #include "absl/synchronization/mutex.h" #include "google/protobuf/arena_align.h" #include "google/protobuf/arena_allocation_policy.h" +#include "google/protobuf/arena_cleanup.h" #include "google/protobuf/arenaz_sampler.h" #include "google/protobuf/port.h" #include "google/protobuf/serial_arena.h" @@ -109,6 +110,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { friend class TcParser; friend class SerialArena; friend struct SerialArenaChunkHeader; + friend class cleanup::ChunkList; static uint64_t GetNextLifeCycleId(); class SerialArenaChunk; From 4c8da99c4865bc7e07ec629c2d1ad32ac576554b Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Mon, 6 May 2024 14:33:20 -0700 Subject: [PATCH 110/179] Deprecate building from source with Maven to be removed in 4.28.x. Most users should use the *pre-built* artifacts, or should build from source with Bazel instead. PiperOrigin-RevId: 631187695 --- java/README.md | 53 ++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/java/README.md b/java/README.md index bedd7a580f..a6d9a6316c 100644 --- a/java/README.md +++ b/java/README.md @@ -67,8 +67,36 @@ If you are contributing code to protobuf or want to use a protobuf version that hasn't been officially released yet, you can follow the instructions below to build protobuf from source code. +### Build from Source + +You may follow these instructions to build from source. This does not require +Maven to be installed. Note that these instructions skip running unit tests and +only describes how to install the core protobuf library (without the util +package). + +1) Build the C++ code, or obtain a binary distribution of protoc (see + the toplevel [README.md](../README.md)). If you install a binary + distribution, make sure that it is the same version as this package. + If in doubt, run: + + $ protoc --version + + If you built the C++ code without installing, the compiler binary + should be located in ../src. + +2) Invoke protoc to build DescriptorProtos.java: + + $ protoc --java_out=core/src/main/java -I../src \ + ../src/google/protobuf/descriptor.proto + +3) Compile the code in core/src/main/java using whatever means you prefer. + +4) Install the classes wherever you prefer. + ### Build from Source - With Maven +WARNING: Building from source with Maven is deprecated and will be removed in the 4.28.x release. + 1) Install Apache Maven if you don't have it: http://maven.apache.org/ @@ -109,31 +137,6 @@ The above instructions will install 2 maven artifacts: as well as utilities to work with proto3 well-known types. -### Build from Source - Without Maven - -If you would rather not install Maven to build the library, you may -follow these instructions instead. Note that these instructions skip -running unit tests and only describes how to install the core protobuf -library (without the util package). - -1) Build the C++ code, or obtain a binary distribution of protoc. If - you install a binary distribution, make sure that it is the same - version as this package. If in doubt, run: - - $ protoc --version - - If you built the C++ code without installing, the compiler binary - should be located in ../src. - -2) Invoke protoc to build DescriptorProtos.java: - - $ protoc --java_out=core/src/main/java -I../src \ - ../src/google/protobuf/descriptor.proto - -3) Compile the code in core/src/main/java using whatever means you prefer. - -4) Install the classes wherever you prefer. - ## Compatibility Notice * Protobuf minor version releases are backwards-compatible. If your code From 55592a28e74e0eeabf89c3fd056ec0d81068a9a2 Mon Sep 17 00:00:00 2001 From: Yohei Yukawa Date: Mon, 6 May 2024 14:59:55 -0700 Subject: [PATCH 111/179] Fix #10007: Propagate msvc-style link flags to clang-cl with Bazel The idea here is to set the existing config "config_msvc" not only when "msvc-cl" is specified but also when "clang-cl" is specified. Keep in mind that clang-cl support in protobuf remains to be only best-effort and untested for now. PiperOrigin-RevId: 631195504 --- build_defs/BUILD.bazel | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel index a0e0438d6e..8745e1d618 100644 --- a/build_defs/BUILD.bazel +++ b/build_defs/BUILD.bazel @@ -14,10 +14,24 @@ package( ) create_compiler_config_setting( - name = "config_msvc", + name = "config_msvc_cl", value = "msvc-cl", ) +# Caveat: clang-cl support in protobuf is only best-effort / untested for now. +create_compiler_config_setting( + name = "config_clang_cl", + value = "clang-cl", +) + +selects.config_setting_group( + name = "config_msvc", + match_any = [ + ":config_clang_cl", + ":config_msvc_cl", + ], +) + config_setting( name = "aarch64", values = {"cpu": "linux-aarch_64"}, From f5979918571af01751afa40d9c41cb6070f56f99 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 May 2024 16:03:04 -0700 Subject: [PATCH 112/179] Return the default instance when parsing from an empty byte[]. PiperOrigin-RevId: 631213831 --- .../com/google/protobuf/GeneratedMessageLite.java | 11 +++++++++-- .../src/test/java/com/google/protobuf/LiteTest.java | 8 +++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 115f4288cd..0937114e42 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -1621,10 +1621,17 @@ public abstract class GeneratedMessageLite< /** A static helper method for parsing a partial from byte array. */ private static > T parsePartialFrom( - T instance, byte[] input, int offset, int length, ExtensionRegistryLite extensionRegistry) + T defaultInstance, + byte[] input, + int offset, + int length, + ExtensionRegistryLite extensionRegistry) throws InvalidProtocolBufferException { + if (length == 0) { + return defaultInstance; + } @SuppressWarnings("unchecked") // Guaranteed by protoc - T result = instance.newMutableInstance(); + T result = defaultInstance.newMutableInstance(); try { Schema schema = Protobuf.getInstance().schemaFor(result); schema.mergeFrom( diff --git a/java/lite/src/test/java/com/google/protobuf/LiteTest.java b/java/lite/src/test/java/com/google/protobuf/LiteTest.java index 754ed7d5fc..0c7b8b535b 100644 --- a/java/lite/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/lite/src/test/java/com/google/protobuf/LiteTest.java @@ -2423,6 +2423,12 @@ public class LiteTest { int unused = TestRecursiveOneof.getDefaultInstance().hashCode(); } + @Test + public void testParseFromEmptyBytes() throws Exception { + assertThat(TestAllTypesLite.parseFrom(new byte[] {})) + .isSameInstanceAs(TestAllTypesLite.getDefaultInstance()); + } + @Test public void testParseFromByteBuffer() throws Exception { TestAllTypesLite message = @@ -2718,7 +2724,7 @@ public class LiteTest { @Test public void testNullExtensionRegistry() throws Exception { try { - TestAllTypesLite.parseFrom(new byte[] {}, null); + TestAllTypesLite.parseFrom(TestUtilLite.getAllLiteSetBuilder().build().toByteArray(), null); assertWithMessage("expected exception").fail(); } catch (NullPointerException expected) { } From d0b016fcfcd00a0e48d006f06868b8fb9876974a Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Mon, 6 May 2024 17:26:29 -0700 Subject: [PATCH 113/179] Java full runtime: Avoid allocating when calling asMap on empty UnknownFieldSet PiperOrigin-RevId: 631235442 --- .../src/main/java/com/google/protobuf/UnknownFieldSet.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index 38aecac42f..b93e220333 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -85,6 +85,10 @@ public final class UnknownFieldSet implements MessageLite { /** Get a map of fields in the set by number. */ public Map asMap() { + // Avoid an allocation for the common case of an empty map. + if (fields.isEmpty()) { + return Collections.emptyMap(); + } return (Map) fields.clone(); } From 9de810acdc6e7e2bcac03d207632c0b9443596ea Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Mon, 6 May 2024 17:33:22 -0700 Subject: [PATCH 114/179] use struct instead of ctype. allows the protobuf python library to work on cpython 3.11's new wasm target [1] For https://github.com/protocolbuffers/protobuf/pull/12212 PiperOrigin-RevId: 631237602 --- python/google/protobuf/internal/type_checkers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/google/protobuf/internal/type_checkers.py b/python/google/protobuf/internal/type_checkers.py index e152a43f82..04ccc98500 100755 --- a/python/google/protobuf/internal/type_checkers.py +++ b/python/google/protobuf/internal/type_checkers.py @@ -22,7 +22,7 @@ TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization __author__ = 'robinson@google.com (Will Robinson)' -import ctypes +import struct import numbers from google.protobuf.internal import decoder @@ -34,7 +34,7 @@ _FieldDescriptor = descriptor.FieldDescriptor def TruncateToFourByteFloat(original): - return ctypes.c_float(original).value + return struct.unpack(' Date: Mon, 6 May 2024 17:35:05 -0700 Subject: [PATCH 115/179] Add UnknownFieldSet.isEmpty Allows callers to check if field set is empty without cloning the map PiperOrigin-RevId: 631237925 --- .../src/main/java/com/google/protobuf/UnknownFieldSet.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index b93e220333..aa71004da7 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -83,6 +83,11 @@ public final class UnknownFieldSet implements MessageLite { return fields.hashCode(); } + /** Whether the field set has no fields. */ + public boolean isEmpty() { + return fields.isEmpty(); + } + /** Get a map of fields in the set by number. */ public Map asMap() { // Avoid an allocation for the common case of an empty map. From 5584c934bf6ea873d8d96ce462896a3573401d2a Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 6 May 2024 19:09:05 -0700 Subject: [PATCH 116/179] Update AbstractMessageLite.addAll so that it won't throw when input is a LazyStringArrayList with byte[] elements. Accidentally run into an error when I'm doing protoA.addAllFieldA(protoB.getMutableFieldAList()) PiperOrigin-RevId: 631257710 --- .../src/main/java/com/google/protobuf/AbstractMessageLite.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java index bc643bd15c..dde30c1b7e 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java @@ -397,6 +397,8 @@ public abstract class AbstractMessageLite< } if (value instanceof ByteString) { lazyList.add((ByteString) value); + } else if (value instanceof byte[]) { + lazyList.add(ByteString.copyFrom((byte[]) value)); } else { lazyList.add((String) value); } From d45016a93fcfad975abeefbc457888a60c113d15 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 7 May 2024 08:37:16 -0700 Subject: [PATCH 117/179] Internal changes to versions PiperOrigin-RevId: 631435235 --- editions/golden/compare_cpp_codegen_failure.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editions/golden/compare_cpp_codegen_failure.xml b/editions/golden/compare_cpp_codegen_failure.xml index 33f961215f..af8c9d0432 100644 --- a/editions/golden/compare_cpp_codegen_failure.xml +++ b/editions/golden/compare_cpp_codegen_failure.xml @@ -2,10 +2,10 @@ - + - + From 6ccda4d2b536f9dfa0fa746954bc152f28c29a1a Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Tue, 7 May 2024 09:43:44 -0700 Subject: [PATCH 118/179] Add upb_Message_ClearOneof (and retain upb_Message_WhichOneofFieldNumber) upb users currently need to manually fetch a oneof field in order to clear it. In this CL, we add a convenience method to do that in one fell swoop. PiperOrigin-RevId: 631454136 --- upb/message/accessors.h | 4 ++++ upb/message/accessors_test.cc | 24 ++++++++++++++++++++++++ upb/message/internal/accessors.h | 15 +++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index c2af8df6e9..c50467c73c 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -54,6 +54,10 @@ UPB_API_INLINE void upb_Message_ClearBaseField(upb_Message* msg, UPB_API_INLINE void upb_Message_ClearExtension(upb_Message* msg, const upb_MiniTableExtension* e); +UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f); + UPB_API_INLINE bool upb_Message_HasBaseField(const upb_Message* msg, const upb_MiniTableField* f); diff --git a/upb/message/accessors_test.cc b/upb/message/accessors_test.cc index 5b0dd3638c..b6d8d42667 100644 --- a/upb/message/accessors_test.cc +++ b/upb/message/accessors_test.cc @@ -478,4 +478,28 @@ TEST(GeneratedCode, EnumClosedCheck) { upb_Arena_Free(arena); } +TEST(GeneratedCode, OneofClear) { + upb_Arena* arena = upb_Arena_New(); + + protobuf_test_messages_proto2_TestAllTypesProto2* msg = + protobuf_test_messages_proto2_TestAllTypesProto2_new(arena); + + const upb_MiniTable* table = + &protobuf_0test_0messages__proto2__TestAllTypesProto2_msg_init; + + // oneof_uint32 + const upb_MiniTableField* oneofField = + upb_MiniTable_FindFieldByNumber(table, 111); + EXPECT_TRUE(upb_MiniTableField_IsInOneof(oneofField)); + protobuf_test_messages_proto2_TestAllTypesProto2_set_oneof_uint32(msg, 522); + EXPECT_TRUE( + protobuf_test_messages_proto2_TestAllTypesProto2_has_oneof_uint32(msg)); + + upb_Message_ClearOneof((upb_Message*)msg, table, oneofField); + EXPECT_FALSE( + protobuf_test_messages_proto2_TestAllTypesProto2_has_oneof_uint32(msg)); + + upb_Arena_Free(arena); +} + } // namespace diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index a6c072278a..c7b447ecc8 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -693,6 +693,21 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } +UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); + if (field_number == 0) { + // No field in the oneof is set. + return; + } + + const upb_MiniTableField* field = + upb_MiniTable_FindFieldByNumber(m, field_number); + upb_Message_ClearBaseField(msg, field); +} + UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( struct upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena) { From ae17e81e42b77f9f6757361c3309a41448577fca Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 7 May 2024 16:59:23 +0000 Subject: [PATCH 119/179] Auto-generate files after cl/631454136 --- php/ext/google/protobuf/php-upb.h | 19 +++++++++++++++++++ ruby/ext/google/protobuf_c/ruby-upb.h | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 21448b174c..f9ca750bc9 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -3269,6 +3269,21 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } +UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); + if (field_number == 0) { + // No field in the oneof is set. + return; + } + + const upb_MiniTableField* field = + upb_MiniTable_FindFieldByNumber(m, field_number); + upb_Message_ClearBaseField(msg, field); +} + UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( struct upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena) { @@ -3544,6 +3559,10 @@ UPB_API_INLINE void upb_Message_ClearBaseField(upb_Message* msg, UPB_API_INLINE void upb_Message_ClearExtension(upb_Message* msg, const upb_MiniTableExtension* e); +UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f); + UPB_API_INLINE bool upb_Message_HasBaseField(const upb_Message* msg, const upb_MiniTableField* f); diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index ddb5106d45..0efa1b29ac 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -3271,6 +3271,21 @@ UPB_API_INLINE void upb_Message_ClearExtension( } } +UPB_API_INLINE void upb_Message_ClearOneof(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f) { + UPB_ASSERT(!upb_Message_IsFrozen(msg)); + uint32_t field_number = upb_Message_WhichOneofFieldNumber(msg, f); + if (field_number == 0) { + // No field in the oneof is set. + return; + } + + const upb_MiniTableField* field = + upb_MiniTable_FindFieldByNumber(m, field_number); + upb_Message_ClearBaseField(msg, field); +} + UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( struct upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena) { @@ -3546,6 +3561,10 @@ UPB_API_INLINE void upb_Message_ClearBaseField(upb_Message* msg, UPB_API_INLINE void upb_Message_ClearExtension(upb_Message* msg, const upb_MiniTableExtension* e); +UPB_API_INLINE void upb_Message_ClearOneof(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f); + UPB_API_INLINE bool upb_Message_HasBaseField(const upb_Message* msg, const upb_MiniTableField* f); From 12af9ade1d166683acdb5a07740122564d3a69f6 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 7 May 2024 11:13:14 -0700 Subject: [PATCH 120/179] Add simple conformance test that builds the old gencode against the current runtime. PiperOrigin-RevId: 631486123 --- WORKSPACE | 15 ++++++++ compatibility/BUILD.bazel | 20 ++++++++++ compatibility/runtime_conformance.bzl | 53 +++++++++++++++++++++++++++ src/google/protobuf/BUILD.bazel | 10 +++++ 4 files changed, 98 insertions(+) create mode 100644 compatibility/BUILD.bazel create mode 100644 compatibility/runtime_conformance.bzl diff --git a/WORKSPACE b/WORKSPACE index def9ed993f..ef112bb548 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -226,3 +226,18 @@ crates_repository( load("@crate_index//:defs.bzl", "crate_repositories") crate_repositories() + +# For testing runtime against old gencode from a previous major version. +http_archive( + name = "com_google_protobuf_v25.0", + strip_prefix = "protobuf-25.0", + url = "https://github.com/protocolbuffers/protobuf/releases/download/v25.0/protobuf-25.0.tar.gz", +) + +# Needed as a dependency of @com_google_protobuf_v25.x, which was before +# utf8_range was merged in. +http_archive( + name = "utf8_range", + strip_prefix = "utf8_range-d863bc33e15cba6d873c878dcca9e6fe52b2f8cb", + url = "https://github.com/protocolbuffers/utf8_range/archive/d863bc33e15cba6d873c878dcca9e6fe52b2f8cb.zip", +) diff --git a/compatibility/BUILD.bazel b/compatibility/BUILD.bazel new file mode 100644 index 0000000000..e62cb5b133 --- /dev/null +++ b/compatibility/BUILD.bazel @@ -0,0 +1,20 @@ +# Simple build tests for compatibility of gencode from previous major versions +# with the current runtime. +# +# To add more test cases in Java, use java_runtime_conformance as below, and add +# the corresponding http_archive in the WORKSPACE file for the version. + +load("//compatibility:runtime_conformance.bzl", "java_runtime_conformance") + +# main gencode builds with main runtime as a proof of concept. +java_runtime_conformance( + name = "java_conformance_main", + gencode_version = "main", +) + +# Generates a build_test named "conformance_v3.25.0" +java_runtime_conformance( + name = "java_conformance_v3.25.0", + gencode_version = "3.25.0", + tags = ["manual"], +) diff --git a/compatibility/runtime_conformance.bzl b/compatibility/runtime_conformance.bzl new file mode 100644 index 0000000000..7f1d416412 --- /dev/null +++ b/compatibility/runtime_conformance.bzl @@ -0,0 +1,53 @@ +"""Provides a rule to generate a conformance test for a given version of the gencode.""" + +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +def java_runtime_conformance(name, gencode_version, tags = []): + """Generates a conformance test for the given version of the runtime. + + For example, runtime_conformance("3.19.4") will generate a build test named "conformance_v3.19.4" + that will fail if the runtime at that version fails to compile the unittest proto. + + Args: + name: The name of the test. + gencode_version: The version of the runtime to test. + tags: Any tags to apply to the generated test. + """ + + if gencode_version == "main": + protoc = "//:protoc" + else: + minor = gencode_version[gencode_version.find(".") + 1:] + protoc = Label("@com_google_protobuf_v%s//:protoc" % minor) + + gencode = [ + "v%s/protobuf_unittest/UnittestProto.java" % gencode_version, + "v%s/com/google/protobuf/test/UnittestImport.java" % gencode_version, + "v%s/com/google/protobuf/test/UnittestImportPublic.java" % gencode_version, + ] + native.genrule( + name = "unittest_proto_gencode_v" + gencode_version, + srcs = [ + "//src/google/protobuf:unittest_proto_srcs", + ], + outs = gencode, + cmd = "$(location %s) " % protoc + + "$(locations //src/google/protobuf:unittest_proto_srcs) " + + " -Isrc/ --java_out=$(@D)/v%s" % gencode_version, + tools = [protoc], + ) + + conformance_name = "conformance_v" + gencode_version + conformance_lib_name = conformance_name + "_lib" + native.java_library( + name = conformance_lib_name, + srcs = gencode, + deps = ["//java/core"], + tags = tags, + ) + + build_test( + name = name, + targets = [":" + conformance_lib_name], + tags = tags, + ) diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index e33b4292d5..56d25c309b 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -840,6 +840,16 @@ filegroup( visibility = ["//:__subpackages__"], ) +filegroup( + name = "unittest_proto_srcs", + srcs = [ + "unittest.proto", + "unittest_import.proto", + "unittest_import_public.proto", + ], + visibility = ["//:__subpackages__"], +) + filegroup( name = "test_proto_editions_srcs", srcs = [ From 94cf1e9446d265453c8b72a352c6c926a1eea25f Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Tue, 7 May 2024 13:37:31 -0700 Subject: [PATCH 121/179] Add 27.x to staleness check PiperOrigin-RevId: 631532649 --- .github/workflows/staleness_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 86a9848243..40df4d81e1 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - branch: [main, 22.x, 23.x, 24.x, 25.x, 26.x] + branch: [main, 25.x, 27.x] os: [{ name: Linux, value: ubuntu-latest}] name: Test staleness ${{ matrix.os.name }} ${{ github.head_ref && 'PR' || matrix.branch }} From 5632d8e616bb96cb783868136e3f5f6e352ce8f0 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 7 May 2024 14:14:58 -0700 Subject: [PATCH 122/179] Replace macro with constexpr function to reduce compile time cost of port_def.inc usage. PiperOrigin-RevId: 631545753 --- src/google/protobuf/compiler/command_line_interface.h | 2 +- src/google/protobuf/compiler/cpp/generator.h | 3 ++- src/google/protobuf/compiler/java/generator.h | 3 ++- src/google/protobuf/compiler/java/options.h | 5 ++--- src/google/protobuf/port.h | 2 ++ src/google/protobuf/port_def.inc | 2 -- src/google/protobuf/port_undef.inc | 1 - 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h index 86af621e37..9dec6c998d 100644 --- a/src/google/protobuf/compiler/command_line_interface.h +++ b/src/google/protobuf/compiler/command_line_interface.h @@ -469,7 +469,7 @@ class PROTOC_EXPORT CommandLineInterface { // When using --encode, this will be passed to SetSerializationDeterministic. bool deterministic_output_ = false; - bool opensource_runtime_ = PROTO2_IS_OSS; + bool opensource_runtime_ = google::protobuf::internal::IsOss(); }; diff --git a/src/google/protobuf/compiler/cpp/generator.h b/src/google/protobuf/compiler/cpp/generator.h index 20d037e843..c200b51078 100644 --- a/src/google/protobuf/compiler/cpp/generator.h +++ b/src/google/protobuf/compiler/cpp/generator.h @@ -24,6 +24,7 @@ #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/cpp_features.pb.h" #include "google/protobuf/descriptor.pb.h" +#include "google/protobuf/port.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -85,7 +86,7 @@ class PROTOC_EXPORT CppGenerator : public CodeGenerator { using CodeGenerator::GetResolvedSourceFeatures; private: - bool opensource_runtime_ = PROTO2_IS_OSS; + bool opensource_runtime_ = google::protobuf::internal::IsOss(); std::string runtime_include_base_; absl::Status ValidateFeatures(const FileDescriptor* file) const; diff --git a/src/google/protobuf/compiler/java/generator.h b/src/google/protobuf/compiler/java/generator.h index bd510c6995..53cb616072 100644 --- a/src/google/protobuf/compiler/java/generator.h +++ b/src/google/protobuf/compiler/java/generator.h @@ -21,6 +21,7 @@ #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/java/java_features.pb.h" #include "google/protobuf/descriptor.pb.h" +#include "google/protobuf/port.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -62,7 +63,7 @@ class PROTOC_EXPORT JavaGenerator : public CodeGenerator { using CodeGenerator::GetResolvedSourceFeatures; private: - bool opensource_runtime_ = PROTO2_IS_OSS; + bool opensource_runtime_ = google::protobuf::internal::IsOss(); }; } // namespace java diff --git a/src/google/protobuf/compiler/java/options.h b/src/google/protobuf/compiler/java/options.h index f10b699c68..fbf116bdc6 100644 --- a/src/google/protobuf/compiler/java/options.h +++ b/src/google/protobuf/compiler/java/options.h @@ -10,7 +10,7 @@ #include -#include "google/protobuf/port_def.inc" +#include "google/protobuf/port.h" namespace google { namespace protobuf { @@ -34,7 +34,7 @@ struct Options { // When set, the protoc will generate the current files and all the transitive // dependencies as lite runtime. bool enforce_lite; - bool opensource_runtime = PROTO2_IS_OSS; + bool opensource_runtime = google::protobuf::internal::IsOss(); // If true, we should build .meta files and emit @Generated annotations into // generated code. bool annotate_code; @@ -53,5 +53,4 @@ struct Options { } // namespace protobuf } // namespace google -#include "google/protobuf/port_undef.inc" #endif // GOOGLE_PROTOBUF_COMPILER_JAVA_OPTIONS_H__ diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h index 4fec7dc138..721afc90b1 100644 --- a/src/google/protobuf/port.h +++ b/src/google/protobuf/port.h @@ -314,6 +314,8 @@ inline void PrefetchToLocalCache(const void* ptr) { absl::PrefetchToLocalCache(ptr); } +constexpr bool IsOss() { return true; } + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index bc6e18b039..5867c6ae1e 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -919,8 +919,6 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), #define PROTOBUF_DEBUG false #endif -#define PROTO2_IS_OSS true - #ifdef PROTOBUF_NO_THREADLOCAL #error PROTOBUF_NO_THREADLOCAL was previously defined #endif diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index eb9fa83cb8..4bb52eaca9 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -72,7 +72,6 @@ #undef PROTOBUF_PREFETCH_WITH_OFFSET #undef PROTOBUF_TC_PARAM_DECL #undef PROTOBUF_DEBUG -#undef PROTO2_IS_OSS #undef PROTOBUF_NO_THREADLOCAL #ifdef PROTOBUF_FUTURE_BREAKING_CHANGES From 98d5bdd1117c0bc1541cf6d5de7b28d00a333c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Tue, 7 May 2024 17:20:54 -0700 Subject: [PATCH 123/179] Add "reserved literals" to the list of reserved names for Java. The identifiers `true`, `false`, and `null` are effectively reserved words in Java, although for some reason they are listed separately from the "keywords" in the Java Language Specification. This doesn't matter for regular fields, because a proto field called `true` will be accessed with `getTrue` and `setTrue`. But for extensions, the generated Java code will have a public static field whose name is the same as the name of the extension field, with `_` appended if the name is a reserved word. Previously there was no `_` for `true` etc, so the generated code would not compile. PiperOrigin-RevId: 631599695 --- src/google/protobuf/compiler/java/names.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/google/protobuf/compiler/java/names.cc b/src/google/protobuf/compiler/java/names.cc index 2c47db1942..ae18ee2e4e 100644 --- a/src/google/protobuf/compiler/java/names.cc +++ b/src/google/protobuf/compiler/java/names.cc @@ -39,16 +39,17 @@ const char* DefaultPackage(Options options) { bool IsReservedName(absl::string_view name) { static const auto& kReservedNames = *new absl::flat_hash_set({ - "abstract", "assert", "boolean", "break", "byte", - "case", "catch", "char", "class", "const", - "continue", "default", "do", "double", "else", - "enum", "extends", "final", "finally", "float", - "for", "goto", "if", "implements", "import", - "instanceof", "int", "interface", "long", "native", - "new", "package", "private", "protected", "public", - "return", "short", "static", "strictfp", "super", - "switch", "synchronized", "this", "throw", "throws", - "transient", "try", "void", "volatile", "while", + "abstract", "assert", "boolean", "break", "byte", + "case", "catch", "char", "class", "const", + "continue", "default", "do", "double", "else", + "enum", "extends", "false", "final", "finally", + "float", "for", "goto", "if", "implements", + "import", "instanceof", "int", "interface", "long", + "native", "new", "package", "private", "protected", + "public", "return", "short", "static", "strictfp", + "super", "switch", "synchronized", "this", "throw", + "throws", "transient", "true", "try", "void", + "volatile", "while", }); return kReservedNames.contains(name); } From 65821a67d60c62ea8ec69a5d76d68da3d775b2da Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Tue, 7 May 2024 18:22:51 -0700 Subject: [PATCH 124/179] upb: finish cleaning up message/accessors.h and message/internal/accessors.h PiperOrigin-RevId: 631612923 --- upb/message/accessors.h | 77 +++++++------------------------- upb/message/copy.c | 2 +- upb/message/internal/accessors.h | 46 +++++++++++++++++++ 3 files changed, 62 insertions(+), 63 deletions(-) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index c50467c73c..ebdfa52bef 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -8,30 +8,17 @@ #ifndef UPB_MESSAGE_ACCESSORS_H_ #define UPB_MESSAGE_ACCESSORS_H_ -#include -#include -#include - -#include "upb/base/descriptor_constants.h" #include "upb/base/string_view.h" #include "upb/mem/arena.h" #include "upb/message/array.h" #include "upb/message/internal/accessors.h" -#include "upb/message/internal/array.h" -#include "upb/message/internal/map.h" -#include "upb/message/internal/message.h" -#include "upb/message/internal/tagged_ptr.h" #include "upb/message/map.h" #include "upb/message/message.h" #include "upb/message/tagged_ptr.h" #include "upb/message/value.h" -#include "upb/mini_table/enum.h" #include "upb/mini_table/extension.h" #include "upb/mini_table/field.h" -#include "upb/mini_table/internal/field.h" -#include "upb/mini_table/internal/message.h" #include "upb/mini_table/message.h" -#include "upb/mini_table/sub.h" // Must be last. #include "upb/port/def.inc" @@ -76,6 +63,10 @@ UPB_API_INLINE upb_MessageValue upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, upb_MessageValue default_val); +UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( + const upb_Message* msg, const upb_MiniTableField* field, + upb_Message* default_val); + UPB_API_INLINE const upb_Array* upb_Message_GetArray( const upb_Message* msg, const upb_MiniTableField* f); @@ -102,12 +93,18 @@ UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, UPB_API_INLINE const upb_Map* upb_Message_GetMap(const upb_Message* msg, const upb_MiniTableField* f); +UPB_API_INLINE const upb_Message* upb_Message_GetMessage( + const upb_Message* msg, const upb_MiniTableField* f); + UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( upb_Message* msg, const upb_MiniTableField* f); UPB_API_INLINE upb_Map* upb_Message_GetMutableMap(upb_Message* msg, const upb_MiniTableField* f); +UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( + upb_Message* msg, const upb_MiniTableField* f); + UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena); @@ -155,6 +152,11 @@ UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, const upb_MiniTableField* f, int64_t value, upb_Arena* a); +UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + upb_Message* sub_message); + UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, const upb_MiniTableField* f, upb_StringView value, upb_Arena* a); @@ -167,55 +169,6 @@ UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a); -UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( - const upb_Message* msg, const upb_MiniTableField* field, - upb_Message* default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_TaggedMessagePtr tagged; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); - return tagged; -} - -UPB_API_INLINE const upb_Message* upb_Message_GetMessage( - const upb_Message* msg, const upb_MiniTableField* field) { - upb_TaggedMessagePtr tagged = - upb_Message_GetTaggedMessagePtr(msg, field, NULL); - return upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); -} - -UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( - upb_Message* msg, const upb_MiniTableField* field) { - return (upb_Message*)upb_Message_GetMessage(msg, field); -} - -// For internal use only; users cannot set tagged messages because only the -// parser and the message copier are allowed to directly create an empty -// message. -UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( - struct upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* f, upb_TaggedMessagePtr sub_message) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == - UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_Message_SetBaseField(msg, f, &sub_message); -} - -// Sets the value of a message-typed field. The `mini_table` and `field` -// parameters belong to `msg`, not `sub_message`. The mini_tables of `msg` and -// `sub_message` must have been linked for this to work correctly. -UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, - const upb_MiniTable* mini_table, - const upb_MiniTableField* field, - upb_Message* sub_message) { - UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (msg, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); -} - UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena); diff --git a/upb/message/copy.c b/upb/message/copy.c index dca2107779..67e1b5b503 100644 --- a/upb/message/copy.c +++ b/upb/message/copy.c @@ -215,7 +215,7 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, return NULL; } UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (dst, mini_table, field, + (dst, field, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, is_empty)); } diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index c7b447ecc8..f1c5473939 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -429,6 +429,37 @@ UPB_API_INLINE const struct upb_Map* upb_Message_GetMap( return ret; } +UPB_API_INLINE uintptr_t upb_Message_GetTaggedMessagePtr( + const struct upb_Message* msg, const upb_MiniTableField* f, + struct upb_Message* default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + uintptr_t tagged; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &tagged); + return tagged; +} + +// For internal use only; users cannot set tagged messages because only the +// parser and the message copier are allowed to directly create an empty +// message. +UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( + struct upb_Message* msg, const upb_MiniTableField* f, + uintptr_t sub_message) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_Message_SetBaseField(msg, f, &sub_message); +} + +UPB_API_INLINE const struct upb_Message* upb_Message_GetMessage( + const struct upb_Message* msg, const upb_MiniTableField* f) { + uintptr_t tagged = upb_Message_GetTaggedMessagePtr(msg, f, NULL); + return upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); +} + UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( struct upb_Message* msg, const upb_MiniTableField* f) { UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); @@ -440,6 +471,11 @@ UPB_API_INLINE struct upb_Map* upb_Message_GetMutableMap( return (struct upb_Map*)upb_Message_GetMap(msg, f); } +UPB_API_INLINE struct upb_Message* upb_Message_GetMutableMessage( + struct upb_Message* msg, const upb_MiniTableField* f) { + return (struct upb_Message*)upb_Message_GetMessage(msg, f); +} + UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( struct upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena) { UPB_ASSERT(arena); @@ -613,6 +649,16 @@ UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); } +// Sets the value of a message-typed field. The mini_tables of `msg` and +// `sub_message` must have been linked for this to work correctly. +UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + struct upb_Message* sub_message) { + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); +} + // Sets the value of a `string` or `bytes` field. The bytes of the value are not // copied, so it is the caller's responsibility to ensure that they remain valid // for the lifetime of `msg`. That might be done by copying them into the given From 4d3a98c27ec327cdc94861c52aeea7fa41d1547c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 01:37:14 +0000 Subject: [PATCH 125/179] Auto-generate files after cl/631612923 --- php/ext/google/protobuf/php-upb.c | 2 +- php/ext/google/protobuf/php-upb.h | 390 +++++++++++++------------- ruby/ext/google/protobuf_c/ruby-upb.c | 2 +- ruby/ext/google/protobuf_c/ruby-upb.h | 390 +++++++++++++------------- 4 files changed, 400 insertions(+), 384 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index fa3be023d6..d9d7d41472 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -6158,7 +6158,7 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, return NULL; } UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (dst, mini_table, field, + (dst, field, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, is_empty)); } diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index f9ca750bc9..66ff534cd8 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -413,113 +413,6 @@ void upb_Status_VAppendErrorFormat(upb_Status* status, const char* fmt, #ifndef UPB_MESSAGE_ACCESSORS_H_ #define UPB_MESSAGE_ACCESSORS_H_ -#include -#include -#include - - -#ifndef UPB_BASE_DESCRIPTOR_CONSTANTS_H_ -#define UPB_BASE_DESCRIPTOR_CONSTANTS_H_ - -// Must be last. - -// The types a field can have. Note that this list is not identical to the -// types defined in descriptor.proto, which gives INT32 and SINT32 separate -// types (we distinguish the two with the "integer encoding" enum below). -// This enum is an internal convenience only and has no meaning outside of upb. -typedef enum { - kUpb_CType_Bool = 1, - kUpb_CType_Float = 2, - kUpb_CType_Int32 = 3, - kUpb_CType_UInt32 = 4, - kUpb_CType_Enum = 5, // Enum values are int32. TODO: rename - kUpb_CType_Message = 6, - kUpb_CType_Double = 7, - kUpb_CType_Int64 = 8, - kUpb_CType_UInt64 = 9, - kUpb_CType_String = 10, - kUpb_CType_Bytes = 11 -} upb_CType; - -// The repeated-ness of each field; this matches descriptor.proto. -typedef enum { - kUpb_Label_Optional = 1, - kUpb_Label_Required = 2, - kUpb_Label_Repeated = 3 -} upb_Label; - -// Descriptor types, as defined in descriptor.proto. -typedef enum { - kUpb_FieldType_Double = 1, - kUpb_FieldType_Float = 2, - kUpb_FieldType_Int64 = 3, - kUpb_FieldType_UInt64 = 4, - kUpb_FieldType_Int32 = 5, - kUpb_FieldType_Fixed64 = 6, - kUpb_FieldType_Fixed32 = 7, - kUpb_FieldType_Bool = 8, - kUpb_FieldType_String = 9, - kUpb_FieldType_Group = 10, - kUpb_FieldType_Message = 11, - kUpb_FieldType_Bytes = 12, - kUpb_FieldType_UInt32 = 13, - kUpb_FieldType_Enum = 14, - kUpb_FieldType_SFixed32 = 15, - kUpb_FieldType_SFixed64 = 16, - kUpb_FieldType_SInt32 = 17, - kUpb_FieldType_SInt64 = 18, -} upb_FieldType; - -#define kUpb_FieldType_SizeOf 19 - -#ifdef __cplusplus -extern "C" { -#endif - -// Convert from upb_FieldType to upb_CType -UPB_INLINE upb_CType upb_FieldType_CType(upb_FieldType field_type) { - static const upb_CType c_type[] = { - kUpb_CType_Double, // kUpb_FieldType_Double - kUpb_CType_Float, // kUpb_FieldType_Float - kUpb_CType_Int64, // kUpb_FieldType_Int64 - kUpb_CType_UInt64, // kUpb_FieldType_UInt64 - kUpb_CType_Int32, // kUpb_FieldType_Int32 - kUpb_CType_UInt64, // kUpb_FieldType_Fixed64 - kUpb_CType_UInt32, // kUpb_FieldType_Fixed32 - kUpb_CType_Bool, // kUpb_FieldType_Bool - kUpb_CType_String, // kUpb_FieldType_String - kUpb_CType_Message, // kUpb_FieldType_Group - kUpb_CType_Message, // kUpb_FieldType_Message - kUpb_CType_Bytes, // kUpb_FieldType_Bytes - kUpb_CType_UInt32, // kUpb_FieldType_UInt32 - kUpb_CType_Enum, // kUpb_FieldType_Enum - kUpb_CType_Int32, // kUpb_FieldType_SFixed32 - kUpb_CType_Int64, // kUpb_FieldType_SFixed64 - kUpb_CType_Int32, // kUpb_FieldType_SInt32 - kUpb_CType_Int64, // kUpb_FieldType_SInt64 - }; - - // -1 here because the enum is one-based but the table is zero-based. - return c_type[field_type - 1]; -} - -UPB_INLINE bool upb_FieldType_IsPackable(upb_FieldType field_type) { - // clang-format off - const unsigned kUnpackableTypes = - (1 << kUpb_FieldType_String) | - (1 << kUpb_FieldType_Bytes) | - (1 << kUpb_FieldType_Message) | - (1 << kUpb_FieldType_Group); - // clang-format on - return (1 << field_type) & ~kUnpackableTypes; -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_BASE_DESCRIPTOR_CONSTANTS_H_ */ #ifndef UPB_BASE_STRING_VIEW_H_ #define UPB_BASE_STRING_VIEW_H_ @@ -827,6 +720,109 @@ void upb_Arena_SetTraceHandler(void (*initArenaTraceHandler)(const upb_Arena*, #include +#ifndef UPB_BASE_DESCRIPTOR_CONSTANTS_H_ +#define UPB_BASE_DESCRIPTOR_CONSTANTS_H_ + +// Must be last. + +// The types a field can have. Note that this list is not identical to the +// types defined in descriptor.proto, which gives INT32 and SINT32 separate +// types (we distinguish the two with the "integer encoding" enum below). +// This enum is an internal convenience only and has no meaning outside of upb. +typedef enum { + kUpb_CType_Bool = 1, + kUpb_CType_Float = 2, + kUpb_CType_Int32 = 3, + kUpb_CType_UInt32 = 4, + kUpb_CType_Enum = 5, // Enum values are int32. TODO: rename + kUpb_CType_Message = 6, + kUpb_CType_Double = 7, + kUpb_CType_Int64 = 8, + kUpb_CType_UInt64 = 9, + kUpb_CType_String = 10, + kUpb_CType_Bytes = 11 +} upb_CType; + +// The repeated-ness of each field; this matches descriptor.proto. +typedef enum { + kUpb_Label_Optional = 1, + kUpb_Label_Required = 2, + kUpb_Label_Repeated = 3 +} upb_Label; + +// Descriptor types, as defined in descriptor.proto. +typedef enum { + kUpb_FieldType_Double = 1, + kUpb_FieldType_Float = 2, + kUpb_FieldType_Int64 = 3, + kUpb_FieldType_UInt64 = 4, + kUpb_FieldType_Int32 = 5, + kUpb_FieldType_Fixed64 = 6, + kUpb_FieldType_Fixed32 = 7, + kUpb_FieldType_Bool = 8, + kUpb_FieldType_String = 9, + kUpb_FieldType_Group = 10, + kUpb_FieldType_Message = 11, + kUpb_FieldType_Bytes = 12, + kUpb_FieldType_UInt32 = 13, + kUpb_FieldType_Enum = 14, + kUpb_FieldType_SFixed32 = 15, + kUpb_FieldType_SFixed64 = 16, + kUpb_FieldType_SInt32 = 17, + kUpb_FieldType_SInt64 = 18, +} upb_FieldType; + +#define kUpb_FieldType_SizeOf 19 + +#ifdef __cplusplus +extern "C" { +#endif + +// Convert from upb_FieldType to upb_CType +UPB_INLINE upb_CType upb_FieldType_CType(upb_FieldType field_type) { + static const upb_CType c_type[] = { + kUpb_CType_Double, // kUpb_FieldType_Double + kUpb_CType_Float, // kUpb_FieldType_Float + kUpb_CType_Int64, // kUpb_FieldType_Int64 + kUpb_CType_UInt64, // kUpb_FieldType_UInt64 + kUpb_CType_Int32, // kUpb_FieldType_Int32 + kUpb_CType_UInt64, // kUpb_FieldType_Fixed64 + kUpb_CType_UInt32, // kUpb_FieldType_Fixed32 + kUpb_CType_Bool, // kUpb_FieldType_Bool + kUpb_CType_String, // kUpb_FieldType_String + kUpb_CType_Message, // kUpb_FieldType_Group + kUpb_CType_Message, // kUpb_FieldType_Message + kUpb_CType_Bytes, // kUpb_FieldType_Bytes + kUpb_CType_UInt32, // kUpb_FieldType_UInt32 + kUpb_CType_Enum, // kUpb_FieldType_Enum + kUpb_CType_Int32, // kUpb_FieldType_SFixed32 + kUpb_CType_Int64, // kUpb_FieldType_SFixed64 + kUpb_CType_Int32, // kUpb_FieldType_SInt32 + kUpb_CType_Int64, // kUpb_FieldType_SInt64 + }; + + // -1 here because the enum is one-based but the table is zero-based. + return c_type[field_type - 1]; +} + +UPB_INLINE bool upb_FieldType_IsPackable(upb_FieldType field_type) { + // clang-format off + const unsigned kUnpackableTypes = + (1 << kUpb_FieldType_String) | + (1 << kUpb_FieldType_Bytes) | + (1 << kUpb_FieldType_Message) | + (1 << kUpb_FieldType_Group); + // clang-format on + return (1 << field_type) & ~kUnpackableTypes; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_BASE_DESCRIPTOR_CONSTANTS_H_ */ + #ifndef UPB_MESSAGE_INTERNAL_ARRAY_H_ #define UPB_MESSAGE_INTERNAL_ARRAY_H_ @@ -3005,6 +3001,37 @@ UPB_API_INLINE const struct upb_Map* upb_Message_GetMap( return ret; } +UPB_API_INLINE uintptr_t upb_Message_GetTaggedMessagePtr( + const struct upb_Message* msg, const upb_MiniTableField* f, + struct upb_Message* default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + uintptr_t tagged; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &tagged); + return tagged; +} + +// For internal use only; users cannot set tagged messages because only the +// parser and the message copier are allowed to directly create an empty +// message. +UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( + struct upb_Message* msg, const upb_MiniTableField* f, + uintptr_t sub_message) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_Message_SetBaseField(msg, f, &sub_message); +} + +UPB_API_INLINE const struct upb_Message* upb_Message_GetMessage( + const struct upb_Message* msg, const upb_MiniTableField* f) { + uintptr_t tagged = upb_Message_GetTaggedMessagePtr(msg, f, NULL); + return upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); +} + UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( struct upb_Message* msg, const upb_MiniTableField* f) { UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); @@ -3016,6 +3043,11 @@ UPB_API_INLINE struct upb_Map* upb_Message_GetMutableMap( return (struct upb_Map*)upb_Message_GetMap(msg, f); } +UPB_API_INLINE struct upb_Message* upb_Message_GetMutableMessage( + struct upb_Message* msg, const upb_MiniTableField* f) { + return (struct upb_Message*)upb_Message_GetMessage(msg, f); +} + UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( struct upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena) { UPB_ASSERT(arena); @@ -3189,6 +3221,16 @@ UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); } +// Sets the value of a message-typed field. The mini_tables of `msg` and +// `sub_message` must have been linked for this to work correctly. +UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + struct upb_Message* sub_message) { + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); +} + // Sets the value of a `string` or `bytes` field. The bytes of the value are not // copied, so it is the caller's responsibility to ensure that they remain valid // for the lifetime of `msg`. That might be done by copying them into the given @@ -3504,41 +3546,6 @@ UPB_API_INLINE upb_Message* upb_TaggedMessagePtr_GetNonEmptyMessage( #endif /* UPB_MINI_TABLE_TAGGED_PTR_H_ */ -#ifndef UPB_MINI_TABLE_SUB_H_ -#define UPB_MINI_TABLE_SUB_H_ - - -// Must be last. - -typedef union upb_MiniTableSub upb_MiniTableSub; - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors - -UPB_API_INLINE upb_MiniTableSub -upb_MiniTableSub_FromEnum(const upb_MiniTableEnum* subenum); - -UPB_API_INLINE upb_MiniTableSub -upb_MiniTableSub_FromMessage(const upb_MiniTable* submsg); - -// Getters - -UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTableSub_Enum( - upb_MiniTableSub sub); - -UPB_API_INLINE const upb_MiniTable* upb_MiniTableSub_Message( - upb_MiniTableSub sub); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_MINI_TABLE_SUB_H_ */ - // Must be last. #ifdef __cplusplus @@ -3581,6 +3588,10 @@ UPB_API_INLINE upb_MessageValue upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, upb_MessageValue default_val); +UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( + const upb_Message* msg, const upb_MiniTableField* field, + upb_Message* default_val); + UPB_API_INLINE const upb_Array* upb_Message_GetArray( const upb_Message* msg, const upb_MiniTableField* f); @@ -3607,12 +3618,18 @@ UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, UPB_API_INLINE const upb_Map* upb_Message_GetMap(const upb_Message* msg, const upb_MiniTableField* f); +UPB_API_INLINE const upb_Message* upb_Message_GetMessage( + const upb_Message* msg, const upb_MiniTableField* f); + UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( upb_Message* msg, const upb_MiniTableField* f); UPB_API_INLINE upb_Map* upb_Message_GetMutableMap(upb_Message* msg, const upb_MiniTableField* f); +UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( + upb_Message* msg, const upb_MiniTableField* f); + UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena); @@ -3660,6 +3677,11 @@ UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, const upb_MiniTableField* f, int64_t value, upb_Arena* a); +UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + upb_Message* sub_message); + UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, const upb_MiniTableField* f, upb_StringView value, upb_Arena* a); @@ -3672,55 +3694,6 @@ UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a); -UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( - const upb_Message* msg, const upb_MiniTableField* field, - upb_Message* default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_TaggedMessagePtr tagged; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); - return tagged; -} - -UPB_API_INLINE const upb_Message* upb_Message_GetMessage( - const upb_Message* msg, const upb_MiniTableField* field) { - upb_TaggedMessagePtr tagged = - upb_Message_GetTaggedMessagePtr(msg, field, NULL); - return upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); -} - -UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( - upb_Message* msg, const upb_MiniTableField* field) { - return (upb_Message*)upb_Message_GetMessage(msg, field); -} - -// For internal use only; users cannot set tagged messages because only the -// parser and the message copier are allowed to directly create an empty -// message. -UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( - struct upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* f, upb_TaggedMessagePtr sub_message) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == - UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_Message_SetBaseField(msg, f, &sub_message); -} - -// Sets the value of a message-typed field. The `mini_table` and `field` -// parameters belong to `msg`, not `sub_message`. The mini_tables of `msg` and -// `sub_message` must have been linked for this to work correctly. -UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, - const upb_MiniTable* mini_table, - const upb_MiniTableField* field, - upb_Message* sub_message) { - UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (msg, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); -} - UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena); @@ -3793,6 +3766,41 @@ UPB_INLINE void _upb_msg_map_set_value(void* msg, const void* val, #define UPB_MINI_TABLE_DECODE_H_ +#ifndef UPB_MINI_TABLE_SUB_H_ +#define UPB_MINI_TABLE_SUB_H_ + + +// Must be last. + +typedef union upb_MiniTableSub upb_MiniTableSub; + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors + +UPB_API_INLINE upb_MiniTableSub +upb_MiniTableSub_FromEnum(const upb_MiniTableEnum* subenum); + +UPB_API_INLINE upb_MiniTableSub +upb_MiniTableSub_FromMessage(const upb_MiniTable* submsg); + +// Getters + +UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTableSub_Enum( + upb_MiniTableSub sub); + +UPB_API_INLINE const upb_MiniTable* upb_MiniTableSub_Message( + upb_MiniTableSub sub); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_MINI_TABLE_SUB_H_ */ + // Export the newer headers, for legacy users. New users should include the // more specific headers directly. // IWYU pragma: begin_exports diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 91fada38f9..faa638b8a7 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -5650,7 +5650,7 @@ upb_Message* _upb_Message_Copy(upb_Message* dst, const upb_Message* src, return NULL; } UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (dst, mini_table, field, + (dst, field, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(dst_sub_message, is_empty)); } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 0efa1b29ac..b4345c8423 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -415,113 +415,6 @@ void upb_Status_VAppendErrorFormat(upb_Status* status, const char* fmt, #ifndef UPB_MESSAGE_ACCESSORS_H_ #define UPB_MESSAGE_ACCESSORS_H_ -#include -#include -#include - - -#ifndef UPB_BASE_DESCRIPTOR_CONSTANTS_H_ -#define UPB_BASE_DESCRIPTOR_CONSTANTS_H_ - -// Must be last. - -// The types a field can have. Note that this list is not identical to the -// types defined in descriptor.proto, which gives INT32 and SINT32 separate -// types (we distinguish the two with the "integer encoding" enum below). -// This enum is an internal convenience only and has no meaning outside of upb. -typedef enum { - kUpb_CType_Bool = 1, - kUpb_CType_Float = 2, - kUpb_CType_Int32 = 3, - kUpb_CType_UInt32 = 4, - kUpb_CType_Enum = 5, // Enum values are int32. TODO: rename - kUpb_CType_Message = 6, - kUpb_CType_Double = 7, - kUpb_CType_Int64 = 8, - kUpb_CType_UInt64 = 9, - kUpb_CType_String = 10, - kUpb_CType_Bytes = 11 -} upb_CType; - -// The repeated-ness of each field; this matches descriptor.proto. -typedef enum { - kUpb_Label_Optional = 1, - kUpb_Label_Required = 2, - kUpb_Label_Repeated = 3 -} upb_Label; - -// Descriptor types, as defined in descriptor.proto. -typedef enum { - kUpb_FieldType_Double = 1, - kUpb_FieldType_Float = 2, - kUpb_FieldType_Int64 = 3, - kUpb_FieldType_UInt64 = 4, - kUpb_FieldType_Int32 = 5, - kUpb_FieldType_Fixed64 = 6, - kUpb_FieldType_Fixed32 = 7, - kUpb_FieldType_Bool = 8, - kUpb_FieldType_String = 9, - kUpb_FieldType_Group = 10, - kUpb_FieldType_Message = 11, - kUpb_FieldType_Bytes = 12, - kUpb_FieldType_UInt32 = 13, - kUpb_FieldType_Enum = 14, - kUpb_FieldType_SFixed32 = 15, - kUpb_FieldType_SFixed64 = 16, - kUpb_FieldType_SInt32 = 17, - kUpb_FieldType_SInt64 = 18, -} upb_FieldType; - -#define kUpb_FieldType_SizeOf 19 - -#ifdef __cplusplus -extern "C" { -#endif - -// Convert from upb_FieldType to upb_CType -UPB_INLINE upb_CType upb_FieldType_CType(upb_FieldType field_type) { - static const upb_CType c_type[] = { - kUpb_CType_Double, // kUpb_FieldType_Double - kUpb_CType_Float, // kUpb_FieldType_Float - kUpb_CType_Int64, // kUpb_FieldType_Int64 - kUpb_CType_UInt64, // kUpb_FieldType_UInt64 - kUpb_CType_Int32, // kUpb_FieldType_Int32 - kUpb_CType_UInt64, // kUpb_FieldType_Fixed64 - kUpb_CType_UInt32, // kUpb_FieldType_Fixed32 - kUpb_CType_Bool, // kUpb_FieldType_Bool - kUpb_CType_String, // kUpb_FieldType_String - kUpb_CType_Message, // kUpb_FieldType_Group - kUpb_CType_Message, // kUpb_FieldType_Message - kUpb_CType_Bytes, // kUpb_FieldType_Bytes - kUpb_CType_UInt32, // kUpb_FieldType_UInt32 - kUpb_CType_Enum, // kUpb_FieldType_Enum - kUpb_CType_Int32, // kUpb_FieldType_SFixed32 - kUpb_CType_Int64, // kUpb_FieldType_SFixed64 - kUpb_CType_Int32, // kUpb_FieldType_SInt32 - kUpb_CType_Int64, // kUpb_FieldType_SInt64 - }; - - // -1 here because the enum is one-based but the table is zero-based. - return c_type[field_type - 1]; -} - -UPB_INLINE bool upb_FieldType_IsPackable(upb_FieldType field_type) { - // clang-format off - const unsigned kUnpackableTypes = - (1 << kUpb_FieldType_String) | - (1 << kUpb_FieldType_Bytes) | - (1 << kUpb_FieldType_Message) | - (1 << kUpb_FieldType_Group); - // clang-format on - return (1 << field_type) & ~kUnpackableTypes; -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_BASE_DESCRIPTOR_CONSTANTS_H_ */ #ifndef UPB_BASE_STRING_VIEW_H_ #define UPB_BASE_STRING_VIEW_H_ @@ -829,6 +722,109 @@ void upb_Arena_SetTraceHandler(void (*initArenaTraceHandler)(const upb_Arena*, #include +#ifndef UPB_BASE_DESCRIPTOR_CONSTANTS_H_ +#define UPB_BASE_DESCRIPTOR_CONSTANTS_H_ + +// Must be last. + +// The types a field can have. Note that this list is not identical to the +// types defined in descriptor.proto, which gives INT32 and SINT32 separate +// types (we distinguish the two with the "integer encoding" enum below). +// This enum is an internal convenience only and has no meaning outside of upb. +typedef enum { + kUpb_CType_Bool = 1, + kUpb_CType_Float = 2, + kUpb_CType_Int32 = 3, + kUpb_CType_UInt32 = 4, + kUpb_CType_Enum = 5, // Enum values are int32. TODO: rename + kUpb_CType_Message = 6, + kUpb_CType_Double = 7, + kUpb_CType_Int64 = 8, + kUpb_CType_UInt64 = 9, + kUpb_CType_String = 10, + kUpb_CType_Bytes = 11 +} upb_CType; + +// The repeated-ness of each field; this matches descriptor.proto. +typedef enum { + kUpb_Label_Optional = 1, + kUpb_Label_Required = 2, + kUpb_Label_Repeated = 3 +} upb_Label; + +// Descriptor types, as defined in descriptor.proto. +typedef enum { + kUpb_FieldType_Double = 1, + kUpb_FieldType_Float = 2, + kUpb_FieldType_Int64 = 3, + kUpb_FieldType_UInt64 = 4, + kUpb_FieldType_Int32 = 5, + kUpb_FieldType_Fixed64 = 6, + kUpb_FieldType_Fixed32 = 7, + kUpb_FieldType_Bool = 8, + kUpb_FieldType_String = 9, + kUpb_FieldType_Group = 10, + kUpb_FieldType_Message = 11, + kUpb_FieldType_Bytes = 12, + kUpb_FieldType_UInt32 = 13, + kUpb_FieldType_Enum = 14, + kUpb_FieldType_SFixed32 = 15, + kUpb_FieldType_SFixed64 = 16, + kUpb_FieldType_SInt32 = 17, + kUpb_FieldType_SInt64 = 18, +} upb_FieldType; + +#define kUpb_FieldType_SizeOf 19 + +#ifdef __cplusplus +extern "C" { +#endif + +// Convert from upb_FieldType to upb_CType +UPB_INLINE upb_CType upb_FieldType_CType(upb_FieldType field_type) { + static const upb_CType c_type[] = { + kUpb_CType_Double, // kUpb_FieldType_Double + kUpb_CType_Float, // kUpb_FieldType_Float + kUpb_CType_Int64, // kUpb_FieldType_Int64 + kUpb_CType_UInt64, // kUpb_FieldType_UInt64 + kUpb_CType_Int32, // kUpb_FieldType_Int32 + kUpb_CType_UInt64, // kUpb_FieldType_Fixed64 + kUpb_CType_UInt32, // kUpb_FieldType_Fixed32 + kUpb_CType_Bool, // kUpb_FieldType_Bool + kUpb_CType_String, // kUpb_FieldType_String + kUpb_CType_Message, // kUpb_FieldType_Group + kUpb_CType_Message, // kUpb_FieldType_Message + kUpb_CType_Bytes, // kUpb_FieldType_Bytes + kUpb_CType_UInt32, // kUpb_FieldType_UInt32 + kUpb_CType_Enum, // kUpb_FieldType_Enum + kUpb_CType_Int32, // kUpb_FieldType_SFixed32 + kUpb_CType_Int64, // kUpb_FieldType_SFixed64 + kUpb_CType_Int32, // kUpb_FieldType_SInt32 + kUpb_CType_Int64, // kUpb_FieldType_SInt64 + }; + + // -1 here because the enum is one-based but the table is zero-based. + return c_type[field_type - 1]; +} + +UPB_INLINE bool upb_FieldType_IsPackable(upb_FieldType field_type) { + // clang-format off + const unsigned kUnpackableTypes = + (1 << kUpb_FieldType_String) | + (1 << kUpb_FieldType_Bytes) | + (1 << kUpb_FieldType_Message) | + (1 << kUpb_FieldType_Group); + // clang-format on + return (1 << field_type) & ~kUnpackableTypes; +} + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_BASE_DESCRIPTOR_CONSTANTS_H_ */ + #ifndef UPB_MESSAGE_INTERNAL_ARRAY_H_ #define UPB_MESSAGE_INTERNAL_ARRAY_H_ @@ -3007,6 +3003,37 @@ UPB_API_INLINE const struct upb_Map* upb_Message_GetMap( return ret; } +UPB_API_INLINE uintptr_t upb_Message_GetTaggedMessagePtr( + const struct upb_Message* msg, const upb_MiniTableField* f, + struct upb_Message* default_val) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + uintptr_t tagged; + _upb_Message_GetNonExtensionField(msg, f, &default_val, &tagged); + return tagged; +} + +// For internal use only; users cannot set tagged messages because only the +// parser and the message copier are allowed to directly create an empty +// message. +UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( + struct upb_Message* msg, const upb_MiniTableField* f, + uintptr_t sub_message) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + upb_Message_SetBaseField(msg, f, &sub_message); +} + +UPB_API_INLINE const struct upb_Message* upb_Message_GetMessage( + const struct upb_Message* msg, const upb_MiniTableField* f) { + uintptr_t tagged = upb_Message_GetTaggedMessagePtr(msg, f, NULL); + return upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); +} + UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( struct upb_Message* msg, const upb_MiniTableField* f) { UPB_PRIVATE(_upb_MiniTableField_CheckIsArray)(f); @@ -3018,6 +3045,11 @@ UPB_API_INLINE struct upb_Map* upb_Message_GetMutableMap( return (struct upb_Map*)upb_Message_GetMap(msg, f); } +UPB_API_INLINE struct upb_Message* upb_Message_GetMutableMessage( + struct upb_Message* msg, const upb_MiniTableField* f) { + return (struct upb_Message*)upb_Message_GetMessage(msg, f); +} + UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( struct upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena) { UPB_ASSERT(arena); @@ -3191,6 +3223,16 @@ UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); } +// Sets the value of a message-typed field. The mini_tables of `msg` and +// `sub_message` must have been linked for this to work correctly. +UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + struct upb_Message* sub_message) { + UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) + (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); +} + // Sets the value of a `string` or `bytes` field. The bytes of the value are not // copied, so it is the caller's responsibility to ensure that they remain valid // for the lifetime of `msg`. That might be done by copying them into the given @@ -3506,41 +3548,6 @@ UPB_API_INLINE upb_Message* upb_TaggedMessagePtr_GetNonEmptyMessage( #endif /* UPB_MINI_TABLE_TAGGED_PTR_H_ */ -#ifndef UPB_MINI_TABLE_SUB_H_ -#define UPB_MINI_TABLE_SUB_H_ - - -// Must be last. - -typedef union upb_MiniTableSub upb_MiniTableSub; - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors - -UPB_API_INLINE upb_MiniTableSub -upb_MiniTableSub_FromEnum(const upb_MiniTableEnum* subenum); - -UPB_API_INLINE upb_MiniTableSub -upb_MiniTableSub_FromMessage(const upb_MiniTable* submsg); - -// Getters - -UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTableSub_Enum( - upb_MiniTableSub sub); - -UPB_API_INLINE const upb_MiniTable* upb_MiniTableSub_Message( - upb_MiniTableSub sub); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - - -#endif /* UPB_MINI_TABLE_SUB_H_ */ - // Must be last. #ifdef __cplusplus @@ -3583,6 +3590,10 @@ UPB_API_INLINE upb_MessageValue upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, upb_MessageValue default_val); +UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( + const upb_Message* msg, const upb_MiniTableField* field, + upb_Message* default_val); + UPB_API_INLINE const upb_Array* upb_Message_GetArray( const upb_Message* msg, const upb_MiniTableField* f); @@ -3609,12 +3620,18 @@ UPB_API_INLINE int64_t upb_Message_GetInt64(const upb_Message* msg, UPB_API_INLINE const upb_Map* upb_Message_GetMap(const upb_Message* msg, const upb_MiniTableField* f); +UPB_API_INLINE const upb_Message* upb_Message_GetMessage( + const upb_Message* msg, const upb_MiniTableField* f); + UPB_API_INLINE upb_Array* upb_Message_GetMutableArray( upb_Message* msg, const upb_MiniTableField* f); UPB_API_INLINE upb_Map* upb_Message_GetMutableMap(upb_Message* msg, const upb_MiniTableField* f); +UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( + upb_Message* msg, const upb_MiniTableField* f); + UPB_API_INLINE upb_Array* upb_Message_GetOrCreateMutableArray( upb_Message* msg, const upb_MiniTableField* f, upb_Arena* arena); @@ -3662,6 +3679,11 @@ UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, const upb_MiniTableField* f, int64_t value, upb_Arena* a); +UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + upb_Message* sub_message); + UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, const upb_MiniTableField* f, upb_StringView value, upb_Arena* a); @@ -3674,55 +3696,6 @@ UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a); -UPB_API_INLINE upb_TaggedMessagePtr upb_Message_GetTaggedMessagePtr( - const upb_Message* msg, const upb_MiniTableField* field, - upb_Message* default_val) { - UPB_ASSUME(upb_MiniTableField_CType(field) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == - UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(field)); - upb_TaggedMessagePtr tagged; - _upb_Message_GetNonExtensionField(msg, field, &default_val, &tagged); - return tagged; -} - -UPB_API_INLINE const upb_Message* upb_Message_GetMessage( - const upb_Message* msg, const upb_MiniTableField* field) { - upb_TaggedMessagePtr tagged = - upb_Message_GetTaggedMessagePtr(msg, field, NULL); - return upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); -} - -UPB_API_INLINE upb_Message* upb_Message_GetMutableMessage( - upb_Message* msg, const upb_MiniTableField* field) { - return (upb_Message*)upb_Message_GetMessage(msg, field); -} - -// For internal use only; users cannot set tagged messages because only the -// parser and the message copier are allowed to directly create an empty -// message. -UPB_INLINE void UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr)( - struct upb_Message* msg, const upb_MiniTable* mini_table, - const upb_MiniTableField* f, upb_TaggedMessagePtr sub_message) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Message); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == - UPB_SIZE(kUpb_FieldRep_4Byte, kUpb_FieldRep_8Byte)); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_Message_SetBaseField(msg, f, &sub_message); -} - -// Sets the value of a message-typed field. The `mini_table` and `field` -// parameters belong to `msg`, not `sub_message`. The mini_tables of `msg` and -// `sub_message` must have been linked for this to work correctly. -UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, - const upb_MiniTable* mini_table, - const upb_MiniTableField* field, - upb_Message* sub_message) { - UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (msg, mini_table, field, - UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); -} - UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena); @@ -3795,6 +3768,41 @@ UPB_INLINE void _upb_msg_map_set_value(void* msg, const void* val, #define UPB_MINI_TABLE_DECODE_H_ +#ifndef UPB_MINI_TABLE_SUB_H_ +#define UPB_MINI_TABLE_SUB_H_ + + +// Must be last. + +typedef union upb_MiniTableSub upb_MiniTableSub; + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors + +UPB_API_INLINE upb_MiniTableSub +upb_MiniTableSub_FromEnum(const upb_MiniTableEnum* subenum); + +UPB_API_INLINE upb_MiniTableSub +upb_MiniTableSub_FromMessage(const upb_MiniTable* submsg); + +// Getters + +UPB_API_INLINE const upb_MiniTableEnum* upb_MiniTableSub_Enum( + upb_MiniTableSub sub); + +UPB_API_INLINE const upb_MiniTable* upb_MiniTableSub_Message( + upb_MiniTableSub sub); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + + +#endif /* UPB_MINI_TABLE_SUB_H_ */ + // Export the newer headers, for legacy users. New users should include the // more specific headers directly. // IWYU pragma: begin_exports From d9238962999a66655ea73cd11b7cd4025c051932 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Tue, 7 May 2024 19:03:05 -0700 Subject: [PATCH 126/179] upb: implement most of the non-universal setters PiperOrigin-RevId: 631620526 --- upb/message/internal/accessors.h | 255 +++++++++++++++++++++------- upb/mini_table/internal/extension.h | 5 + 2 files changed, 199 insertions(+), 61 deletions(-) diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index f1c5473939..d81ddd7e17 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -220,7 +220,6 @@ UPB_INLINE bool UPB_PRIVATE(_upb_MiniTableField_DataIsZero)( // const upb_MiniTableField* field, // bool value, upb_Arena* a) { // UPB_ASSUME(field->UPB_PRIVATE(descriptortype) == kUpb_FieldType_Bool); -// UPB_ASSUME(upb_MiniTableField_IsScalar(field)); // UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == // kUpb_FieldRep_1Byte); // upb_Message_SetField(msg, field, &value, a); @@ -351,8 +350,8 @@ UPB_API_INLINE bool upb_Message_GetBool(const struct upb_Message* msg, const upb_MiniTableField* f, bool default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); upb_MessageValue def; def.bool_val = default_val; return upb_Message_GetField(msg, f, def).bool_val; @@ -362,8 +361,8 @@ UPB_API_INLINE double upb_Message_GetDouble(const struct upb_Message* msg, const upb_MiniTableField* f, double default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.double_val = default_val; @@ -374,8 +373,8 @@ UPB_API_INLINE float upb_Message_GetFloat(const struct upb_Message* msg, const upb_MiniTableField* f, float default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.float_val = default_val; @@ -387,8 +386,8 @@ UPB_API_INLINE int32_t upb_Message_GetInt32(const struct upb_Message* msg, int32_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || upb_MiniTableField_CType(f) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.int32_val = default_val; @@ -399,8 +398,8 @@ UPB_API_INLINE int64_t upb_Message_GetInt64(const struct upb_Message* msg, const upb_MiniTableField* f, int64_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.int64_val = default_val; @@ -551,7 +550,6 @@ upb_Message_GetString(const struct upb_Message* msg, upb_MiniTableField_CType(f) == kUpb_CType_Bytes); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); upb_MessageValue def; def.str_val = default_val; @@ -562,8 +560,8 @@ UPB_API_INLINE uint32_t upb_Message_GetUInt32(const struct upb_Message* msg, const upb_MiniTableField* f, uint32_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.uint32_val = default_val; @@ -574,23 +572,89 @@ UPB_API_INLINE uint64_t upb_Message_GetUInt64(const struct upb_Message* msg, const upb_MiniTableField* f, uint64_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.uint64_val = default_val; return upb_Message_GetField(msg, f, def).uint64_val; } -UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, - const upb_MiniTableField* f, bool value, - upb_Arena* a) { +// BaseField Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE void upb_Message_SetBaseFieldBool(struct upb_Message* msg, + const upb_MiniTableField* f, + bool value) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.bool_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); } UPB_API_INLINE void upb_Message_SetClosedEnum( @@ -598,55 +662,132 @@ UPB_API_INLINE void upb_Message_SetClosedEnum( const upb_MiniTableField* f, int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); UPB_ASSERT(upb_MiniTableEnum_CheckValue( upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); upb_Message_SetBaseField(msg, f, &value); } +// Extension Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetExtensionBool( + struct upb_Message* msg, const upb_MiniTableExtension* e, bool value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_1Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionDouble( + struct upb_Message* msg, const upb_MiniTableExtension* e, double value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionFloat( + struct upb_Message* msg, const upb_MiniTableExtension* e, float value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, int32_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Int32 || + upb_MiniTableExtension_CType(e) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, int64_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionString( + struct upb_Message* msg, const upb_MiniTableExtension* e, + upb_StringView value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_String || + upb_MiniTableExtension_CType(e) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_StringView); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionUInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint32_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionUInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint64_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +// Universal Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a) { + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionBool( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldBool(msg, f, value), true); +} + UPB_API_INLINE bool upb_Message_SetDouble(struct upb_Message* msg, const upb_MiniTableField* f, double value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.double_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionDouble( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldDouble(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetFloat(struct upb_Message* msg, const upb_MiniTableField* f, float value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.float_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionFloat( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldFloat(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetInt32(struct upb_Message* msg, const upb_MiniTableField* f, int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || - upb_MiniTableField_CType(f) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.int32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionInt32( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldInt32(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, const upb_MiniTableField* f, int64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.int64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionInt64( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldInt64(msg, f, value), true); } // Sets the value of a message-typed field. The mini_tables of `msg` and @@ -666,36 +807,28 @@ UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, UPB_API_INLINE bool upb_Message_SetString(struct upb_Message* msg, const upb_MiniTableField* f, upb_StringView value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || - upb_MiniTableField_CType(f) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.str_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionString( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldString(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetUInt32(struct upb_Message* msg, const upb_MiniTableField* f, uint32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.uint32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionUInt32( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldUInt32(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetUInt64(struct upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.uint64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionUInt64( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldUInt64(msg, f, value), true); } UPB_API_INLINE void upb_Message_Clear(struct upb_Message* msg, diff --git a/upb/mini_table/internal/extension.h b/upb/mini_table/internal/extension.h index e326f99997..c7b674cd87 100644 --- a/upb/mini_table/internal/extension.h +++ b/upb/mini_table/internal/extension.h @@ -49,6 +49,11 @@ UPB_API_INLINE void upb_MiniTableExtension_SetSubMessage( e->UPB_PRIVATE(sub).UPB_PRIVATE(submsg) = m; } +UPB_INLINE upb_FieldRep UPB_PRIVATE(_upb_MiniTableExtension_GetRep)( + const struct upb_MiniTableExtension* e) { + return UPB_PRIVATE(_upb_MiniTableField_GetRep)(&e->UPB_PRIVATE(field)); +} + #ifdef __cplusplus } /* extern "C" */ #endif From 6afb130dca7e4f26d584c8f17f5b3e99a40f5d7f Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 02:19:22 +0000 Subject: [PATCH 127/179] Auto-generate files after cl/631620526 --- php/ext/google/protobuf/php-upb.h | 260 ++++++++++++++++++++------ ruby/ext/google/protobuf_c/ruby-upb.h | 260 ++++++++++++++++++++------ 2 files changed, 398 insertions(+), 122 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 66ff534cd8..a8e5a0db31 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -1878,6 +1878,11 @@ UPB_API_INLINE void upb_MiniTableExtension_SetSubMessage( e->UPB_PRIVATE(sub).UPB_PRIVATE(submsg) = m; } +UPB_INLINE upb_FieldRep UPB_PRIVATE(_upb_MiniTableExtension_GetRep)( + const struct upb_MiniTableExtension* e) { + return UPB_PRIVATE(_upb_MiniTableField_GetRep)(&e->UPB_PRIVATE(field)); +} + #ifdef __cplusplus } /* extern "C" */ #endif @@ -2792,7 +2797,6 @@ UPB_INLINE bool UPB_PRIVATE(_upb_MiniTableField_DataIsZero)( // const upb_MiniTableField* field, // bool value, upb_Arena* a) { // UPB_ASSUME(field->UPB_PRIVATE(descriptortype) == kUpb_FieldType_Bool); -// UPB_ASSUME(upb_MiniTableField_IsScalar(field)); // UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == // kUpb_FieldRep_1Byte); // upb_Message_SetField(msg, field, &value, a); @@ -2923,8 +2927,8 @@ UPB_API_INLINE bool upb_Message_GetBool(const struct upb_Message* msg, const upb_MiniTableField* f, bool default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); upb_MessageValue def; def.bool_val = default_val; return upb_Message_GetField(msg, f, def).bool_val; @@ -2934,8 +2938,8 @@ UPB_API_INLINE double upb_Message_GetDouble(const struct upb_Message* msg, const upb_MiniTableField* f, double default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.double_val = default_val; @@ -2946,8 +2950,8 @@ UPB_API_INLINE float upb_Message_GetFloat(const struct upb_Message* msg, const upb_MiniTableField* f, float default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.float_val = default_val; @@ -2959,8 +2963,8 @@ UPB_API_INLINE int32_t upb_Message_GetInt32(const struct upb_Message* msg, int32_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || upb_MiniTableField_CType(f) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.int32_val = default_val; @@ -2971,8 +2975,8 @@ UPB_API_INLINE int64_t upb_Message_GetInt64(const struct upb_Message* msg, const upb_MiniTableField* f, int64_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.int64_val = default_val; @@ -3123,7 +3127,6 @@ upb_Message_GetString(const struct upb_Message* msg, upb_MiniTableField_CType(f) == kUpb_CType_Bytes); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); upb_MessageValue def; def.str_val = default_val; @@ -3134,8 +3137,8 @@ UPB_API_INLINE uint32_t upb_Message_GetUInt32(const struct upb_Message* msg, const upb_MiniTableField* f, uint32_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.uint32_val = default_val; @@ -3146,23 +3149,89 @@ UPB_API_INLINE uint64_t upb_Message_GetUInt64(const struct upb_Message* msg, const upb_MiniTableField* f, uint64_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.uint64_val = default_val; return upb_Message_GetField(msg, f, def).uint64_val; } -UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, - const upb_MiniTableField* f, bool value, - upb_Arena* a) { +// BaseField Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE void upb_Message_SetBaseFieldBool(struct upb_Message* msg, + const upb_MiniTableField* f, + bool value) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.bool_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); } UPB_API_INLINE void upb_Message_SetClosedEnum( @@ -3170,55 +3239,132 @@ UPB_API_INLINE void upb_Message_SetClosedEnum( const upb_MiniTableField* f, int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); UPB_ASSERT(upb_MiniTableEnum_CheckValue( upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); upb_Message_SetBaseField(msg, f, &value); } +// Extension Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetExtensionBool( + struct upb_Message* msg, const upb_MiniTableExtension* e, bool value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_1Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionDouble( + struct upb_Message* msg, const upb_MiniTableExtension* e, double value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionFloat( + struct upb_Message* msg, const upb_MiniTableExtension* e, float value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, int32_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Int32 || + upb_MiniTableExtension_CType(e) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, int64_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionString( + struct upb_Message* msg, const upb_MiniTableExtension* e, + upb_StringView value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_String || + upb_MiniTableExtension_CType(e) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_StringView); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionUInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint32_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionUInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint64_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +// Universal Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a) { + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionBool( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldBool(msg, f, value), true); +} + UPB_API_INLINE bool upb_Message_SetDouble(struct upb_Message* msg, const upb_MiniTableField* f, double value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.double_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionDouble( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldDouble(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetFloat(struct upb_Message* msg, const upb_MiniTableField* f, float value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.float_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionFloat( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldFloat(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetInt32(struct upb_Message* msg, const upb_MiniTableField* f, int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || - upb_MiniTableField_CType(f) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.int32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionInt32( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldInt32(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, const upb_MiniTableField* f, int64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.int64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionInt64( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldInt64(msg, f, value), true); } // Sets the value of a message-typed field. The mini_tables of `msg` and @@ -3238,36 +3384,28 @@ UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, UPB_API_INLINE bool upb_Message_SetString(struct upb_Message* msg, const upb_MiniTableField* f, upb_StringView value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || - upb_MiniTableField_CType(f) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.str_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionString( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldString(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetUInt32(struct upb_Message* msg, const upb_MiniTableField* f, uint32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.uint32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionUInt32( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldUInt32(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetUInt64(struct upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.uint64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionUInt64( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldUInt64(msg, f, value), true); } UPB_API_INLINE void upb_Message_Clear(struct upb_Message* msg, diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index b4345c8423..63ff2c9b8a 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -1880,6 +1880,11 @@ UPB_API_INLINE void upb_MiniTableExtension_SetSubMessage( e->UPB_PRIVATE(sub).UPB_PRIVATE(submsg) = m; } +UPB_INLINE upb_FieldRep UPB_PRIVATE(_upb_MiniTableExtension_GetRep)( + const struct upb_MiniTableExtension* e) { + return UPB_PRIVATE(_upb_MiniTableField_GetRep)(&e->UPB_PRIVATE(field)); +} + #ifdef __cplusplus } /* extern "C" */ #endif @@ -2794,7 +2799,6 @@ UPB_INLINE bool UPB_PRIVATE(_upb_MiniTableField_DataIsZero)( // const upb_MiniTableField* field, // bool value, upb_Arena* a) { // UPB_ASSUME(field->UPB_PRIVATE(descriptortype) == kUpb_FieldType_Bool); -// UPB_ASSUME(upb_MiniTableField_IsScalar(field)); // UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(field) == // kUpb_FieldRep_1Byte); // upb_Message_SetField(msg, field, &value, a); @@ -2925,8 +2929,8 @@ UPB_API_INLINE bool upb_Message_GetBool(const struct upb_Message* msg, const upb_MiniTableField* f, bool default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); upb_MessageValue def; def.bool_val = default_val; return upb_Message_GetField(msg, f, def).bool_val; @@ -2936,8 +2940,8 @@ UPB_API_INLINE double upb_Message_GetDouble(const struct upb_Message* msg, const upb_MiniTableField* f, double default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.double_val = default_val; @@ -2948,8 +2952,8 @@ UPB_API_INLINE float upb_Message_GetFloat(const struct upb_Message* msg, const upb_MiniTableField* f, float default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.float_val = default_val; @@ -2961,8 +2965,8 @@ UPB_API_INLINE int32_t upb_Message_GetInt32(const struct upb_Message* msg, int32_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || upb_MiniTableField_CType(f) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.int32_val = default_val; @@ -2973,8 +2977,8 @@ UPB_API_INLINE int64_t upb_Message_GetInt64(const struct upb_Message* msg, const upb_MiniTableField* f, int64_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.int64_val = default_val; @@ -3125,7 +3129,6 @@ upb_Message_GetString(const struct upb_Message* msg, upb_MiniTableField_CType(f) == kUpb_CType_Bytes); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); upb_MessageValue def; def.str_val = default_val; @@ -3136,8 +3139,8 @@ UPB_API_INLINE uint32_t upb_Message_GetUInt32(const struct upb_Message* msg, const upb_MiniTableField* f, uint32_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); upb_MessageValue def; def.uint32_val = default_val; @@ -3148,23 +3151,89 @@ UPB_API_INLINE uint64_t upb_Message_GetUInt64(const struct upb_Message* msg, const upb_MiniTableField* f, uint64_t default_val) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); upb_MessageValue def; def.uint64_val = default_val; return upb_Message_GetField(msg, f, def).uint64_val; } -UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, - const upb_MiniTableField* f, bool value, - upb_Arena* a) { +// BaseField Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE void upb_Message_SetBaseFieldBool(struct upb_Message* msg, + const upb_MiniTableField* f, + bool value) { UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Bool); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_1Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || + upb_MiniTableField_CType(f) == kUpb_CType_Enum); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || + upb_MiniTableField_CType(f) == kUpb_CType_Bytes); UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.bool_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == + kUpb_FieldRep_StringView); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); + upb_Message_SetBaseField(msg, f, &value); +} + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value) { + UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); + UPB_ASSUME(upb_MiniTableField_IsScalar(f)); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); + upb_Message_SetBaseField(msg, f, &value); } UPB_API_INLINE void upb_Message_SetClosedEnum( @@ -3172,55 +3241,132 @@ UPB_API_INLINE void upb_Message_SetClosedEnum( const upb_MiniTableField* f, int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); UPB_ASSERT(upb_MiniTableEnum_CheckValue( upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); upb_Message_SetBaseField(msg, f, &value); } +// Extension Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetExtensionBool( + struct upb_Message* msg, const upb_MiniTableExtension* e, bool value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Bool); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_1Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionDouble( + struct upb_Message* msg, const upb_MiniTableExtension* e, double value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Double); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionFloat( + struct upb_Message* msg, const upb_MiniTableExtension* e, float value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Float); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, int32_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Int32 || + upb_MiniTableExtension_CType(e) == kUpb_CType_Enum); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, int64_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_Int64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionString( + struct upb_Message* msg, const upb_MiniTableExtension* e, + upb_StringView value, upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_String || + upb_MiniTableExtension_CType(e) == kUpb_CType_Bytes); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_StringView); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionUInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint32_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_UInt32); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_4Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +UPB_API_INLINE bool upb_Message_SetExtensionUInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint64_t value, + upb_Arena* a) { + UPB_ASSUME(upb_MiniTableExtension_CType(e) == kUpb_CType_UInt64); + UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableExtension_GetRep)(e) == + kUpb_FieldRep_8Byte); + return upb_Message_SetExtension(msg, e, &value, a); +} + +// Universal Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetBool(struct upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a) { + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionBool( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldBool(msg, f, value), true); +} + UPB_API_INLINE bool upb_Message_SetDouble(struct upb_Message* msg, const upb_MiniTableField* f, double value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Double); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.double_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionDouble( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldDouble(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetFloat(struct upb_Message* msg, const upb_MiniTableField* f, float value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Float); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.float_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionFloat( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldFloat(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetInt32(struct upb_Message* msg, const upb_MiniTableField* f, int32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int32 || - upb_MiniTableField_CType(f) == kUpb_CType_Enum); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.int32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionInt32( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldInt32(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, const upb_MiniTableField* f, int64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_Int64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.int64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionInt64( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldInt64(msg, f, value), true); } // Sets the value of a message-typed field. The mini_tables of `msg` and @@ -3240,36 +3386,28 @@ UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, UPB_API_INLINE bool upb_Message_SetString(struct upb_Message* msg, const upb_MiniTableField* f, upb_StringView value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_String || - upb_MiniTableField_CType(f) == kUpb_CType_Bytes); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == - kUpb_FieldRep_StringView); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.str_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionString( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldString(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetUInt32(struct upb_Message* msg, const upb_MiniTableField* f, uint32_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt32); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.uint32_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionUInt32( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldUInt32(msg, f, value), true); } UPB_API_INLINE bool upb_Message_SetUInt64(struct upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a) { - UPB_ASSUME(upb_MiniTableField_CType(f) == kUpb_CType_UInt64); - UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_8Byte); - UPB_ASSUME(upb_MiniTableField_IsScalar(f)); - upb_MessageValue val; - val.uint64_val = value; - return UPB_PRIVATE(_upb_Message_SetField)(msg, f, val, a); + return upb_MiniTableField_IsExtension(f) + ? upb_Message_SetExtensionUInt64( + msg, (const upb_MiniTableExtension*)f, value, a) + : (upb_Message_SetBaseFieldUInt64(msg, f, value), true); } UPB_API_INLINE void upb_Message_Clear(struct upb_Message* msg, From d8329b68e457f5c2227cbcd05a13aed9f69a1c85 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 00:48:25 -0700 Subject: [PATCH 128/179] Add .to_owned(&self)->Msg functions to MsgMut and MsgView PiperOrigin-RevId: 631686314 --- rust/test/shared/accessors_test.rs | 30 +++++++++++++++++--- src/google/protobuf/compiler/rust/message.cc | 8 ++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index 7a170364e0..e14bcafc4c 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -8,7 +8,7 @@ //! Tests covering accessors for singular bool, int32, int64, and bytes fields. use googletest::prelude::*; -use protobuf::{MutProxy, Optional}; +use protobuf::Optional; use unittest_proto::{test_all_types, TestAllTypes}; #[test] @@ -498,15 +498,13 @@ fn test_nonempty_default_string_accessors() { #[test] fn test_singular_msg_field() { - use test_all_types::*; - let mut msg = TestAllTypes::new(); let msg_view = msg.optional_nested_message(); // testing reading an int inside a view assert_that!(msg_view.bb(), eq(0)); assert_that!(msg.has_optional_nested_message(), eq(false)); - let mut nested_msg_mut = msg.optional_nested_message_mut(); + let nested_msg_mut = msg.optional_nested_message_mut(); // test reading an int inside a mut assert_that!(nested_msg_mut.bb(), eq(0)); @@ -780,3 +778,27 @@ fn test_submsg_setter() { assert_that!(parent.optional_nested_message().bb(), eq(7)); } + +#[test] +fn test_to_owned() { + let mut m = TestAllTypes::new(); + m.set_optional_int32(42); + let clone = m.as_view().to_owned(); + assert_that!(clone.optional_int32(), eq(42)); + + // to_owned should create a new message (modifying the original shouldn't affect + // the to_owned). + m.clear_optional_int32(); + assert_that!(m.has_optional_int32(), eq(false)); + assert_that!(clone.has_optional_int32(), eq(true)); + assert_that!(clone.optional_int32(), eq(42)); + + let mut submsg_mut = m.optional_nested_message_mut(); + submsg_mut.set_bb(7); + let submsg_clone = submsg_mut.to_owned(); + assert_that!(submsg_clone.bb(), eq(7)); + assert_that!(submsg_mut.bb(), eq(7)); + submsg_mut.set_bb(8); + assert_that!(submsg_clone.bb(), eq(7)); + assert_that!(submsg_mut.bb(), eq(8)); +} diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 39236062ef..2b2babd172 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -911,6 +911,10 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $Msg::serialize$ } + pub fn to_owned(&self) -> $Msg$ { + $pb$::IntoProxied::into(*self, $pbi$::Private) + } + $accessor_fns_for_views$ } @@ -986,6 +990,10 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $pb$::ViewProxy::as_view(self).serialize() } + pub fn to_owned(&self) -> $Msg$ { + $pb$::ViewProxy::as_view(self).to_owned() + } + $get_upb_minitable$ $raw_arena_getter_for_msgmut$ From b77343aa2c6e81d81d5c8c627ab8ef8856fac2f5 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Wed, 8 May 2024 09:11:03 -0700 Subject: [PATCH 129/179] Add temporary alias for system_python.bzl at previous location . This was added to protobuf_deps.bzl in cl/629786458 and no longer needs to be specified directly in WORKSPACE. However, an alias is needed to prevent users from breaking when upgrading from 26.x -> 27.x due to the obsolete load pointing to the wrong path. This alias will be removed in a future release, tentatively 4.30.x. PiperOrigin-RevId: 631821637 --- google3/third_party/upb/bazel/system_python.bzl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 google3/third_party/upb/bazel/system_python.bzl diff --git a/google3/third_party/upb/bazel/system_python.bzl b/google3/third_party/upb/bazel/system_python.bzl new file mode 100644 index 0000000000..5126b9bce0 --- /dev/null +++ b/google3/third_party/upb/bazel/system_python.bzl @@ -0,0 +1,17 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Temporary alias to repository rule for using Python 3.x headers from the system.""" + +load( + "//python/dist:system_python.bzl", + _system_python = "system_python", +) + +# TODO: Temporary alias. This is deprecated and to be removed in a future +# release. Users should now get system_python from protobuf_deps.bzl. +system_python = _system_python From 01744cccae57ca934ab925e13d2eaaf93761b1f4 Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Wed, 8 May 2024 09:21:08 -0700 Subject: [PATCH 130/179] chore(php): conformance testing for edition (#16712) COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/16712 from bshaffer:php-editions a1c41add7d7cc6ea716af5f4e83f30cd78b80e22 PiperOrigin-RevId: 631824623 --- conformance/BUILD.bazel | 3 +- conformance/conformance_php.php | 214 +++++++++++------- conformance/text_format_failure_list_php.txt | 8 - editions/BUILD | 27 ++- php/BUILD.bazel | 2 + protobuf.bzl | 8 + .../protobuf/compiler/php/php_generator.h | 8 +- 7 files changed, 179 insertions(+), 91 deletions(-) diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index 1cca2e9323..ae9c18c0ed 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -332,13 +332,14 @@ inline_sh_binary( ], cmd = """ php -dextension=$(rootpath //php:extension) \\ - -d include_path=conformance:src/google/protobuf \\ + -d include_path=conformance:src/google/protobuf:editions/golden \\ $(rootpath conformance_php.php) """, visibility = ["//php:__subpackages__"], deps = [ ":conformance_php_proto", "//:test_messages_proto3_php_proto", + "//editions:test_messages_proto3_editions_php_proto", ], ) diff --git a/conformance/conformance_php.php b/conformance/conformance_php.php index f250ebf138..a753761147 100644 --- a/conformance/conformance_php.php +++ b/conformance/conformance_php.php @@ -1,80 +1,134 @@ getPayload() == "protobuf_payload") { - if ($request->getMessageType() == "conformance.FailureSet") { - $response->setProtobufPayload(""); - return $response; - } elseif ($request->getMessageType() == "protobuf_test_messages.proto3.TestAllTypesProto3") { - try { - $test_message->mergeFromString($request->getProtobufPayload()); - } catch (Exception $e) { - $response->setParseError($e->getMessage()); - return $response; - } - } elseif ($request->getMessageType() == "protobuf_test_messages.proto2.TestAllTypesProto2") { - $response->setSkipped("PHP doesn't support proto2"); - return $response; - } else { - trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR); - } - } elseif ($request->getPayload() == "json_payload") { - $ignore_json_unknown = - ($request->getTestCategory() == - TestCategory::JSON_IGNORE_UNKNOWN_PARSING_TEST); - try { - $test_message->mergeFromJsonString($request->getJsonPayload(), - $ignore_json_unknown); - } catch (Exception $e) { - $response->setParseError($e->getMessage()); - return $response; - } - } elseif ($request->getPayload() == "text_payload") { - $response->setSkipped("PHP doesn't support text format yet"); - return $response; - } else { - trigger_error("Request didn't have payload.", E_USER_ERROR); + $response = new ConformanceResponse(); + + switch ($request->getPayload()) { + case 'protobuf_payload': + switch ($request->getMessageType()) { + case 'protobuf_test_messages.proto3.TestAllTypesProto3': + $test_message = new TestAllTypesProto3(); + break; + case 'protobuf_test_messages.editions.proto3.TestAllTypesProto3': + $test_message = new TestAllTypesProto3Editions(); + break; + case 'conformance.FailureSet': + $response->setProtobufPayload(''); + return $response; + case 'protobuf_test_messages.proto2.TestAllTypesProto2': + case 'protobuf_test_messages.editions.proto2.TestAllTypesProto2': + $response->setSkipped('PHP doesn\'t support proto2'); + return $response; + case 'protobuf_test_messages.editions.TestAllTypesEdition2023': + $response->setSkipped('PHP doesn\'t support editions-specific features yet'); + return $response; + case '': + trigger_error( + 'Protobuf request doesn\'t have specific payload type', + E_USER_ERROR + ); + default: + trigger_error(sprintf( + 'Protobuf request doesn\'t support %s message type', + $request->getMessageType(), + ), E_USER_ERROR); + } + + try { + $test_message->mergeFromString($request->getProtobufPayload()); + } catch (Exception $e) { + $response->setParseError($e->getMessage()); + return $response; + } + break; + + case 'json_payload': + switch ($request->getMessageType()) { + case 'protobuf_test_messages.editions.proto3.TestAllTypesProto3': + $test_message = new TestAllTypesProto3Editions(); + break; + case 'protobuf_test_messages.editions.proto2.TestAllTypesProto2': + $response->setSkipped('PHP doesn\'t support proto2'); + return $response; + default: + $test_message = new TestAllTypesProto3(); + } + + $ignore_json_unknown = + ($request->getTestCategory() == TestCategory::JSON_IGNORE_UNKNOWN_PARSING_TEST); + try { + $test_message->mergeFromJsonString( + $request->getJsonPayload(), + $ignore_json_unknown + ); + } catch (Exception $e) { + $response->setParseError($e->getMessage()); + return $response; + } + break; + + case 'text_payload': + $response->setSkipped('PHP doesn\'t support text format yet'); + return $response; + default: + trigger_error('Request didn\'t have payload.', E_USER_ERROR); } - if ($request->getRequestedOutputFormat() == WireFormat::UNSPECIFIED) { - trigger_error("Unspecified output format.", E_USER_ERROR); - } elseif ($request->getRequestedOutputFormat() == WireFormat::PROTOBUF) { - $response->setProtobufPayload($test_message->serializeToString()); - } elseif ($request->getRequestedOutputFormat() == WireFormat::JSON) { - try { - $response->setJsonPayload($test_message->serializeToJsonString()); - } catch (Exception $e) { - $response->setSerializeError($e->getMessage()); - return $response; - } + switch ($request->getRequestedOutputFormat()) { + case WireFormat::TEXT_FORMAT: + $response->setSkipped('PHP doesn\'t support text format yet'); + return $response; + case WireFormat::UNSPECIFIED: + trigger_error('Unspecified output format.', E_USER_ERROR); + case WireFormat::PROTOBUF: + $response->setProtobufPayload($test_message->serializeToString()); + break; + case WireFormat::JSON: + try { + $response->setJsonPayload($test_message->serializeToJsonString()); + } catch (Exception $e) { + $response->setSerializeError($e->getMessage()); + return $response; + } } return $response; @@ -84,25 +138,25 @@ function doTestIO() { $length_bytes = fread(STDIN, 4); if (strlen($length_bytes) == 0) { - return false; # EOF + return false; # EOF } elseif (strlen($length_bytes) != 4) { - fwrite(STDERR, "I/O error\n"); - return false; + fwrite(STDERR, "I/O error\n"); + return false; } - $length = unpack("V", $length_bytes)[1]; + $length = unpack('V', $length_bytes)[1]; $serialized_request = fread(STDIN, $length); if (strlen($serialized_request) != $length) { - trigger_error("I/O error", E_USER_ERROR); + trigger_error('I/O error', E_USER_ERROR); } - $request = new \Conformance\ConformanceRequest(); + $request = new ConformanceRequest(); $request->mergeFromString($serialized_request); $response = doTest($request); $serialized_response = $response->serializeToString(); - fwrite(STDOUT, pack("V", strlen($serialized_response))); + fwrite(STDOUT, pack('V', strlen($serialized_response))); fwrite(STDOUT, $serialized_response); $GLOBALS['test_count'] += 1; @@ -111,10 +165,10 @@ function doTestIO() } while(true){ - if (!doTestIO()) { - fprintf(STDERR, - "conformance_php: received EOF from test runner " . - "after %d tests, exiting\n", $test_count); - exit; - } + if (!doTestIO()) { + fprintf(STDERR, + 'conformance_php: received EOF from test runner ' . + "after %d tests, exiting\n", $test_count); + exit; + } } diff --git a/conformance/text_format_failure_list_php.txt b/conformance/text_format_failure_list_php.txt index 404b64a584..e69de29bb2 100644 --- a/conformance/text_format_failure_list_php.txt +++ b/conformance/text_format_failure_list_php.txt @@ -1,8 +0,0 @@ -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput -Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput -Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Print.TextFormatOutput diff --git a/editions/BUILD b/editions/BUILD index c78f0af10a..09838afc80 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -1,5 +1,5 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//:protobuf.bzl", "internal_objc_proto_library", "internal_py_proto_library") +load("//:protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library") load("//bazel:cc_proto_library.bzl", "cc_proto_library") load("//bazel:upb_proto_library.bzl", "upb_c_proto_library", "upb_proto_reflection_library") load(":defaults.bzl", "compile_edition_defaults", "embed_edition_defaults") @@ -236,6 +236,31 @@ java_lite_proto_library( deps = [":test_messages_proto3_editions_proto"], ) +internal_php_proto_library( + name = "test_messages_proto3_editions_php_proto", + testonly = 1, + srcs = ["golden/test_messages_proto3_editions.proto"], + outs = [ + "GPBMetadata/TestMessagesProto3Editions.php", + "Protobuf_test_messages/Editions/Proto3/EnumOnlyProto3.php", + "Protobuf_test_messages/Editions/Proto3/EnumOnlyProto3/PBBool.php", + "Protobuf_test_messages/Editions/Proto3/ForeignEnum.php", + "Protobuf_test_messages/Editions/Proto3/ForeignMessage.php", + "Protobuf_test_messages/Editions/Proto3/NullHypothesisProto3.php", + "Protobuf_test_messages/Editions/Proto3/TestAllTypesProto3.php", + "Protobuf_test_messages/Editions/Proto3/TestAllTypesProto3/AliasedEnum.php", + "Protobuf_test_messages/Editions/Proto3/TestAllTypesProto3/NestedEnum.php", + "Protobuf_test_messages/Editions/Proto3/TestAllTypesProto3/NestedMessage.php", + ], + enable_editions = True, + includes = [ + "golden", + "src", + ], + proto_deps = ["//:well_known_protos"], + visibility = ["//conformance:__pkg__"], +) + internal_py_proto_library( name = "test_messages_proto3_editions_py_pb2", testonly = True, diff --git a/php/BUILD.bazel b/php/BUILD.bazel index 09cc76524f..cc0ba5b174 100644 --- a/php/BUILD.bazel +++ b/php/BUILD.bazel @@ -64,6 +64,7 @@ genrule( conformance_test( name = "conformance_test", failure_list = "//conformance:failure_list_php.txt", + maximum_edition = "2023", target_compatible_with = select({ "@platforms//os:osx": ["@platforms//:incompatible"], "//conditions:default": [], @@ -75,6 +76,7 @@ conformance_test( conformance_test( name = "conformance_test_c", failure_list = "//conformance:failure_list_php_c.txt", + maximum_edition = "2023", target_compatible_with = select({ "@platforms//os:osx": [], "//conditions:default": ["@platforms//:incompatible"], diff --git a/protobuf.bzl b/protobuf.bzl index 7db5146a0f..9a05897894 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -80,6 +80,7 @@ def _proto_gen_impl(ctx): srcs = ctx.files.srcs langs = ctx.attr.langs or [] out_type = ctx.attr.out_type + enable_editions = ctx.attr.enable_editions deps = depset(direct = ctx.files.srcs) source_dir = _SourceDir(ctx) gen_dir = _GenDir(ctx).rstrip("/") @@ -130,6 +131,8 @@ def _proto_gen_impl(ctx): generated_files = [] for src in srcs: args = [] + if enable_editions: + args.append("--experimental_editions") in_gen_dir = src.root.path == gen_dir if in_gen_dir: @@ -231,6 +234,7 @@ Args: srcs: Protocol Buffers definition files (.proto) to run the protocol compiler against. deps: a list of dependency labels; must be other proto libraries. + enable_editions: if true, sets the --experimental_editions flag. includes: a list of include paths to .proto files. protoc: the label of the protocol compiler to generate the sources. plugin: the label of the protocol compiler plugin to be passed to the protocol @@ -247,6 +251,7 @@ _proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), "deps": attr.label_list(providers = [ProtoGenInfo]), + "enable_editions": attr.bool(), "includes": attr.string_list(), "protoc": attr.label( cfg = "exec", @@ -655,6 +660,7 @@ def _source_proto_library( protoc = Label("//:protoc"), testonly = None, visibility = ["//visibility:public"], + enable_editions = False, **kwargs): """Bazel rule to create generated protobuf code from proto source files for languages not well supported by Bazel yet. This will output the generated @@ -699,6 +705,7 @@ def _source_proto_library( srcs = proto_deps, protoc = protoc, includes = includes, + enable_editions = enable_editions, ) full_deps.append(":%s_deps_genproto" % name) @@ -712,6 +719,7 @@ def _source_proto_library( protoc = protoc, testonly = testonly, visibility = visibility, + enable_editions = enable_editions, ) native.filegroup( diff --git a/src/google/protobuf/compiler/php/php_generator.h b/src/google/protobuf/compiler/php/php_generator.h index a270b2004f..10d3f0e09c 100644 --- a/src/google/protobuf/compiler/php/php_generator.h +++ b/src/google/protobuf/compiler/php/php_generator.h @@ -37,7 +37,13 @@ class PROTOC_EXPORT Generator : public CodeGenerator { std::string* error) const override; uint64_t GetSupportedFeatures() const override { - return FEATURE_PROTO3_OPTIONAL; + return Feature::FEATURE_PROTO3_OPTIONAL; + } + + Edition GetMinimumEdition() const override { return Edition::EDITION_PROTO2; } + Edition GetMaximumEdition() const override { return Edition::EDITION_2023; } + std::vector GetFeatureExtensions() const override { + return {}; } private: From d5b7d7a87f55b18f1d5d3117bdc1aefc2526508b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 09:28:05 -0700 Subject: [PATCH 131/179] Remove dead feature macro. The opt-out code was removed some time ago. PiperOrigin-RevId: 631826723 --- src/google/protobuf/port_def.inc | 7 ------- src/google/protobuf/port_undef.inc | 1 - 2 files changed, 8 deletions(-) diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 5867c6ae1e..2be3e2073f 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -597,13 +597,6 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), #define PROTOBUF_TSAN_DECLARE_MEMBER #endif -#ifdef PROTOBUF_USE_TABLE_PARSER_ON_REFLECTION -#error PROTOBUF_USE_TABLE_PARSER_ON_REFLECTION was previously defined -#endif -#if !defined(PROTOBUF_TEMPORARY_DISABLE_TABLE_PARSER_ON_REFLECTION) -#define PROTOBUF_USE_TABLE_PARSER_ON_REFLECTION 1 -#endif // PROTOBUF_ENABLE_FORCE_ALLOCATION_ON_CONSTRUCTION - // Note that this is performance sensitive: changing the parameters will change // the registers used by the ABI calling convention, which subsequently affects // register selection logic inside the function. diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 4bb52eaca9..83f8a8c573 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -65,7 +65,6 @@ #undef PROTOBUF_MSAN #undef PROTOBUF_TSAN #undef PROTOBUF_TSAN_DECLARE_MEMBER -#undef PROTOBUF_USE_TABLE_PARSER_ON_REFLECTION #undef PROTOBUF_BUILTIN_CONSTANT_P #undef PROTOBUF_DESCRIPTOR_WEAK_MESSAGES_ALLOWED #undef PROTOBUF_PREFETCH_PARSE_TABLE From 64580b29c1dfdd580052a01a5ccd3fb1c0a22f5c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 11:34:05 -0700 Subject: [PATCH 132/179] Expose JavaPackageDirectory and FileClassName in java/names.h PiperOrigin-RevId: 631870244 --- src/google/protobuf/compiler/java/helpers.h | 3 +-- src/google/protobuf/compiler/java/names.cc | 8 ++++++++ src/google/protobuf/compiler/java/names.h | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index 52b4017cf5..d99e085a65 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -78,8 +78,7 @@ std::string UniqueFileScopeIdentifier(const Descriptor* descriptor); // Gets the unqualified class name for the file. For each .proto file, there // will be one Java class containing all the immutable messages and another // Java class containing all the mutable messages. -// TODO: remove the default value after updating client code. -std::string FileClassName(const FileDescriptor* file, bool immutable = true); +std::string FileClassName(const FileDescriptor* file, bool immutable); // Returns the file's Java package name. std::string FileJavaPackage(const FileDescriptor* file, bool immutable, diff --git a/src/google/protobuf/compiler/java/names.cc b/src/google/protobuf/compiler/java/names.cc index ae18ee2e4e..7f026ecaf6 100644 --- a/src/google/protobuf/compiler/java/names.cc +++ b/src/google/protobuf/compiler/java/names.cc @@ -141,6 +141,14 @@ std::string FileJavaPackage(const FileDescriptor* file, Options options) { return FileJavaPackage(file, true /* immutable */, options); } +std::string JavaPackageDirectory(const FileDescriptor* file) { + return JavaPackageToDir(FileJavaPackage(file)); +} + +std::string FileClassName(const FileDescriptor* file) { + return FileClassName(file, /*immutable=*/true); +} + std::string CapitalizedFieldName(const FieldDescriptor* field) { return UnderscoresToCamelCase(FieldName(field), true); } diff --git a/src/google/protobuf/compiler/java/names.h b/src/google/protobuf/compiler/java/names.h index d6402fc5d3..c2eb34cb37 100644 --- a/src/google/protobuf/compiler/java/names.h +++ b/src/google/protobuf/compiler/java/names.h @@ -72,6 +72,20 @@ std::string ClassName(const ServiceDescriptor* descriptor); std::string FileJavaPackage(const FileDescriptor* descriptor, Options options = {}); +// Requires: +// descriptor != NULL +// +// Returns: +// Java package directory. +std::string JavaPackageDirectory(const FileDescriptor* file); + +// Requires: +// descriptor != NULL +// +// Returns: +// The unqualified Java class name. +std::string FileClassName(const FileDescriptor* file); + // Requires: // descriptor != NULL // Returns: From ec126a5069fbb81a857e41c8b2cfbe71ace35993 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 12:23:44 -0700 Subject: [PATCH 133/179] Replace the single user of `PROTOBUF_ALIGNAS` with `alignas` and remove the now dead macro. This reduces the compile time cost of port_def.inc PiperOrigin-RevId: 631886181 --- src/google/protobuf/arena.h | 2 +- src/google/protobuf/port_def.inc | 33 ------------------------------ src/google/protobuf/port_undef.inc | 1 - 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index c8ccfe24a1..f32f6d8c18 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -142,7 +142,7 @@ struct ArenaOptions { // This is a thread-safe implementation: multiple threads may allocate from the // arena concurrently. Destruction is not thread-safe and the destructing // thread must synchronize with users of the arena first. -class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { +class PROTOBUF_EXPORT alignas(8) Arena final { public: // Default constructor with sensible default options, tuned for average // use-cases. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 2be3e2073f..1e05aad179 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -403,39 +403,6 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), #error PROTOBUF_FORCE_ALLOCATION_ON_CONSTRUCTION was previously defined #endif -// Specify memory alignment for structs, classes, etc. -// Use like: -// class PROTOBUF_ALIGNAS(16) MyClass { ... } -// PROTOBUF_ALIGNAS(16) int array[4]; -// -// In most places you can use the C++11 keyword "alignas", which is preferred. -// -// But compilers have trouble mixing __attribute__((...)) syntax with -// alignas(...) syntax. -// -// Doesn't work in clang or gcc: -// struct alignas(16) __attribute__((packed)) S { char c; }; -// Works in clang but not gcc: -// struct __attribute__((packed)) alignas(16) S2 { char c; }; -// Works in clang and gcc: -// struct alignas(16) S3 { char c; } __attribute__((packed)); -// -// There are also some attributes that must be specified *before* a class -// definition: visibility (used for exporting functions/classes) is one of -// these attributes. This means that it is not possible to use alignas() with a -// class that is marked as exported. -#ifdef PROTOBUF_ALIGNAS -#error PROTOBUF_ALIGNAS was previously defined -#endif -#if defined(_MSC_VER) -#define PROTOBUF_ALIGNAS(byte_alignment) __declspec(align(byte_alignment)) -#elif defined(__GNUC__) -#define PROTOBUF_ALIGNAS(byte_alignment) \ - __attribute__((aligned(byte_alignment))) -#else -#define PROTOBUF_ALIGNAS(byte_alignment) alignas(byte_alignment) -#endif - #ifdef PROTOBUF_THREAD_LOCAL #error PROTOBUF_THREAD_LOCAL was previously defined #endif diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 83f8a8c573..9d72db2f52 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -50,7 +50,6 @@ #undef PROTOBUF_ASSUME #undef PROTOBUF_EXPORT_TEMPLATE_DECLARE #undef PROTOBUF_EXPORT_TEMPLATE_DEFINE -#undef PROTOBUF_ALIGNAS #undef PROTOBUF_THREAD_LOCAL #undef PROTOBUF_CONSTINIT #undef PROTOBUF_CONSTEXPR From b694cfa4414c817ab12b7e593044ff92572c2e63 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Wed, 8 May 2024 15:10:51 -0700 Subject: [PATCH 134/179] Clarify that the input to proto2::DynamicMessageFactory::GetPrototype should be non-null PiperOrigin-RevId: 631938646 --- src/google/protobuf/dynamic_message.cc | 1 + src/google/protobuf/dynamic_message.h | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 7927107815..c4448d51a0 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -620,6 +620,7 @@ DynamicMessageFactory::~DynamicMessageFactory() { } const Message* DynamicMessageFactory::GetPrototype(const Descriptor* type) { + ABSL_CHECK(type != nullptr); absl::MutexLock lock(&prototypes_mutex_); return GetPrototypeNoLock(type); } diff --git a/src/google/protobuf/dynamic_message.h b/src/google/protobuf/dynamic_message.h index 8d53e28cbc..0af9d441b5 100644 --- a/src/google/protobuf/dynamic_message.h +++ b/src/google/protobuf/dynamic_message.h @@ -108,8 +108,8 @@ class PROTOBUF_EXPORT DynamicMessageFactory : public MessageFactory { // prototype, so these must be destroyed before the DynamicMessageFactory // is destroyed. // - // The given descriptor must outlive the returned message, and hence must - // outlive the DynamicMessageFactory. + // The given descriptor must be non-null and outlive the returned message, and + // hence must outlive the DynamicMessageFactory. // // The method is thread-safe. const Message* GetPrototype(const Descriptor* type) override; From 7febb4c48ff4c22e0b78b84706bf325f4e92838b Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Wed, 8 May 2024 16:20:14 -0700 Subject: [PATCH 135/179] Move upb/bazel/system_python.bzl (new alias file) to correct open source directory. PiperOrigin-RevId: 631958564 --- google3/third_party/upb/bazel/system_python.bzl | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 google3/third_party/upb/bazel/system_python.bzl diff --git a/google3/third_party/upb/bazel/system_python.bzl b/google3/third_party/upb/bazel/system_python.bzl deleted file mode 100644 index 5126b9bce0..0000000000 --- a/google3/third_party/upb/bazel/system_python.bzl +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2009-2021, Google LLC -# All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -"""Temporary alias to repository rule for using Python 3.x headers from the system.""" - -load( - "//python/dist:system_python.bzl", - _system_python = "system_python", -) - -# TODO: Temporary alias. This is deprecated and to be removed in a future -# release. Users should now get system_python from protobuf_deps.bzl. -system_python = _system_python From 1b06cefe337f73ca8c78c855c02f15caf6210c9b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 06:55:30 -0700 Subject: [PATCH 136/179] Move casting functions to MessageLite and use ClassData as the uniqueness instead of Reflection. This allows using these functions instead of `dynamic_cast` for all generated types including LITE. PiperOrigin-RevId: 632135009 --- src/google/protobuf/lite_unittest.cc | 87 +++++++++++++++++ src/google/protobuf/message.h | 84 ---------------- src/google/protobuf/message_lite.h | 118 +++++++++++++++++++++++ src/google/protobuf/message_unittest.inc | 26 ++--- 4 files changed, 218 insertions(+), 97 deletions(-) diff --git a/src/google/protobuf/lite_unittest.cc b/src/google/protobuf/lite_unittest.cc index 978307528a..f9f0135501 100644 --- a/src/google/protobuf/lite_unittest.cc +++ b/src/google/protobuf/lite_unittest.cc @@ -27,6 +27,7 @@ #include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/map_lite_test_util.h" #include "google/protobuf/map_lite_unittest.pb.h" +#include "google/protobuf/message_lite.h" #include "google/protobuf/parse_context.h" #include "google/protobuf/test_util_lite.h" #include "google/protobuf/unittest_lite.pb.h" @@ -1325,6 +1326,92 @@ TEST(LiteBasicTest, CodedInputStreamRollback) { } } +// Two arbitary types +using CastType1 = protobuf_unittest::TestAllTypesLite; +using CastType2 = protobuf_unittest::TestPackedTypesLite; + +TEST(LiteTest, DynamicCastToGenerated) { + CastType1 test_type_1; + + MessageLite* test_type_1_pointer = &test_type_1; + EXPECT_EQ(&test_type_1, + DynamicCastToGenerated(test_type_1_pointer)); + EXPECT_EQ(nullptr, DynamicCastToGenerated(test_type_1_pointer)); + + const MessageLite* test_type_1_pointer_const = &test_type_1; + EXPECT_EQ(&test_type_1, + DynamicCastToGenerated(test_type_1_pointer_const)); + EXPECT_EQ(nullptr, + DynamicCastToGenerated(test_type_1_pointer_const)); + + MessageLite* test_type_1_pointer_nullptr = nullptr; + EXPECT_EQ(nullptr, + DynamicCastToGenerated(test_type_1_pointer_nullptr)); + + MessageLite& test_type_1_pointer_ref = test_type_1; + EXPECT_EQ(&test_type_1, + &DynamicCastToGenerated(test_type_1_pointer_ref)); + + const MessageLite& test_type_1_pointer_const_ref = test_type_1; + EXPECT_EQ(&test_type_1, + &DynamicCastToGenerated(test_type_1_pointer_const_ref)); +} + +#if GTEST_HAS_DEATH_TEST +TEST(LiteTest, DynamicCastToGeneratedInvalidReferenceType) { + CastType1 test_type_1; + const MessageLite& test_type_1_pointer_const_ref = test_type_1; + ASSERT_DEATH(DynamicCastToGenerated(test_type_1_pointer_const_ref), + "Cannot downcast " + test_type_1.GetTypeName() + " to " + + CastType2::default_instance().GetTypeName()); +} +#endif // GTEST_HAS_DEATH_TEST + +TEST(LiteTest, DownCastToGeneratedValidType) { + CastType1 test_type_1; + + MessageLite* test_type_1_pointer = &test_type_1; + EXPECT_EQ(&test_type_1, DownCastToGenerated(test_type_1_pointer)); + + const MessageLite* test_type_1_pointer_const = &test_type_1; + EXPECT_EQ(&test_type_1, + DownCastToGenerated(test_type_1_pointer_const)); + + MessageLite* test_type_1_pointer_nullptr = nullptr; + EXPECT_EQ(nullptr, + DownCastToGenerated(test_type_1_pointer_nullptr)); + + MessageLite& test_type_1_pointer_ref = test_type_1; + EXPECT_EQ(&test_type_1, + &DownCastToGenerated(test_type_1_pointer_ref)); + + const MessageLite& test_type_1_pointer_const_ref = test_type_1; + EXPECT_EQ(&test_type_1, + &DownCastToGenerated(test_type_1_pointer_const_ref)); +} + +#if GTEST_HAS_DEATH_TEST +TEST(LiteTest, DownCastToGeneratedInvalidPointerType) { + CastType1 test_type_1; + + MessageLite* test_type_1_pointer = &test_type_1; + + ASSERT_DEBUG_DEATH(DownCastToGenerated(test_type_1_pointer), + "Cannot downcast " + test_type_1.GetTypeName() + " to " + + CastType2::default_instance().GetTypeName()); +} + +TEST(LiteTest, DownCastToGeneratedInvalidReferenceType) { + CastType1 test_type_1; + + MessageLite& test_type_1_pointer = test_type_1; + + ASSERT_DEBUG_DEATH(DownCastToGenerated(test_type_1_pointer), + "Cannot downcast " + test_type_1.GetTypeName() + " to " + + CastType2::default_instance().GetTypeName()); +} +#endif // GTEST_HAS_DEATH_TEST + } // namespace } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 0354c9e9d7..88666e62e1 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -1424,90 +1424,6 @@ DECLARE_GET_REPEATED_FIELD(bool) #undef DECLARE_GET_REPEATED_FIELD -// Tries to downcast this message to a generated message type. Returns nullptr -// if this class is not an instance of T. This works even if RTTI is disabled. -// -// This also has the effect of creating a strong reference to T that will -// prevent the linker from stripping it out at link time. This can be important -// if you are using a DynamicMessageFactory that delegates to the generated -// factory. -template -const T* DynamicCastToGenerated(const Message* from) { - // Compile-time assert that T is a generated type that has a - // default_instance() accessor, but avoid actually calling it. - const T& (*get_default_instance)() = &T::default_instance; - (void)get_default_instance; - - // Compile-time assert that T is a subclass of google::protobuf::Message. - const Message* unused = static_cast(nullptr); - (void)unused; - -#if PROTOBUF_RTTI - internal::StrongReferenceToType(); - return dynamic_cast(from); -#else - bool ok = from != nullptr && - T::default_instance().GetReflection() == from->GetReflection(); - return ok ? internal::DownCast(from) : nullptr; -#endif -} - -template -T* DynamicCastToGenerated(Message* from) { - const Message* message_const = from; - return const_cast(DynamicCastToGenerated(message_const)); -} - -// An overloaded version of DynamicCastToGenerated for downcasting references to -// base Message class. If the destination type T if the argument is not an -// instance of T and dynamic_cast returns nullptr, it terminates with an error. -template -const T& DynamicCastToGenerated(const Message& from) { - const T* destination_message = DynamicCastToGenerated(&from); - ABSL_CHECK(destination_message != nullptr) - << "Cannot downcast " << from.GetTypeName() << " to " - << T::default_instance().GetTypeName(); - return *destination_message; -} - -template -T& DynamicCastToGenerated(Message& from) { - const Message& message_const = from; - const T& destination_message = DynamicCastToGenerated(message_const); - return const_cast(destination_message); -} - -// A lightweight function for downcasting base Message pointer to derived type. -// It should only be used when the caller is certain that the argument is of -// instance T and T is a type derived from base Message class. -template -const T* DownCastToGenerated(const Message* from) { - internal::StrongReferenceToType(); - ABSL_DCHECK(DynamicCastToGenerated(from) == from) - << "Cannot downcast " << from->GetTypeName() << " to " - << T::default_instance().GetTypeName(); - - return static_cast(from); -} - -template -T* DownCastToGenerated(Message* from) { - const Message* message_const = from; - return const_cast(DownCastToGenerated(message_const)); -} - -template -const T& DownCastToGenerated(const Message& from) { - return *DownCastToGenerated(&from); -} - -template -T& DownCastToGenerated(Message& from) { - const Message& message_const = from; - const T& destination_message = DownCastToGenerated(message_const); - return const_cast(destination_message); -} - // Call this function to ensure that this message's reflection is linked into // the binary: // diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index 1b85aa0c65..abb869e817 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "absl/base/attributes.h" #include "absl/log/absl_check.h" @@ -53,6 +54,7 @@ class FastReflectionStringSetter; class Reflection; class Descriptor; class AssignDescriptorsHelper; +class MessageLite; namespace io { @@ -120,6 +122,9 @@ class PROTOBUF_EXPORT CachedSize { #endif }; +// For MessageLite to friend. +class TypeId; + class SwapFieldHelper; // See parse_context.h for explanation @@ -638,6 +643,15 @@ class PROTOBUF_EXPORT MessageLite { // return a default table instead of a unique one. virtual const ClassData* GetClassData() const = 0; + template + static auto GetClassDataGenerated() { + // We could speed this up if needed by avoiding the function call. + // In LTO this is likely inlined, so it might not matter. + static_assert( + std::is_same::value, ""); + return T::default_instance().T::GetClassData(); + } + internal::InternalMetadata _internal_metadata_; // Return the cached size object as described by @@ -682,6 +696,7 @@ class PROTOBUF_EXPORT MessageLite { friend class internal::LazyField; friend class internal::SwapFieldHelper; friend class internal::TcParser; + friend class internal::TypeId; friend class internal::WeakFieldMap; friend class internal::WireFormatLite; @@ -717,6 +732,32 @@ class PROTOBUF_EXPORT MessageLite { namespace internal { +// A typeinfo equivalent for protobuf message types. Used for +// DynamicCastToGenerated. +// We might make this class public later on to have an alternative to +// `std::type_info` that works when RTTI is disabled. +class TypeId { + public: + constexpr explicit TypeId(const MessageLite::ClassData* data) : data_(data) {} + + friend constexpr bool operator==(TypeId a, TypeId b) { + return a.data_ == b.data_; + } + friend constexpr bool operator!=(TypeId a, TypeId b) { return !(a == b); } + + static TypeId Get(const MessageLite& msg) { + return TypeId(msg.GetClassData()); + } + + template + static TypeId Get() { + return TypeId(MessageLite::GetClassDataGenerated()); + } + + private: + const MessageLite::ClassData* data_; +}; + template bool MergeFromImpl(absl::string_view input, MessageLite* msg, const internal::TcParseTableBase* tc_table, @@ -820,6 +861,83 @@ T* OnShutdownDelete(T* p) { std::string ShortFormat(const MessageLite& message_lite); std::string Utf8Format(const MessageLite& message_lite); +// Tries to downcast this message to a generated message type. Returns nullptr +// if this class is not an instance of T. This works even if RTTI is disabled. +// +// This also has the effect of creating a strong reference to T that will +// prevent the linker from stripping it out at link time. This can be important +// if you are using a DynamicMessageFactory that delegates to the generated +// factory. +template +const T* DynamicCastToGenerated(const MessageLite* from) { + static_assert(std::is_base_of::value, ""); + + internal::StrongReferenceToType(); + // We might avoid the call to T::GetClassData() altogether if T were to + // expose the class data pointer. + if (from == nullptr || + internal::TypeId::Get() != internal::TypeId::Get(*from)) { + return nullptr; + } + + return static_cast(from); +} + +template +const T* DynamicCastToGenerated(const MessageLite* from); + +template +T* DynamicCastToGenerated(MessageLite* from) { + return const_cast( + DynamicCastToGenerated(static_cast(from))); +} + +// An overloaded version of DynamicCastToGenerated for downcasting references to +// base Message class. If the argument is not an instance of T, it terminates +// with an error. +template +const T& DynamicCastToGenerated(const MessageLite& from) { + const T* destination_message = DynamicCastToGenerated(&from); + ABSL_CHECK(destination_message != nullptr) + << "Cannot downcast " << from.GetTypeName() << " to " + << T::default_instance().GetTypeName(); + return *destination_message; +} + +template +T& DynamicCastToGenerated(MessageLite& from) { + return const_cast( + DynamicCastToGenerated(static_cast(from))); +} + +// A lightweight function for downcasting base MessageLite pointer to derived +// type. It should only be used when the caller is certain that the argument is +// of instance T and T is a generated message type. +template +const T* DownCastToGenerated(const MessageLite* from) { + internal::StrongReferenceToType(); + ABSL_DCHECK(DynamicCastToGenerated(from) == from) + << "Cannot downcast " << from->GetTypeName() << " to " + << T::default_instance().GetTypeName(); + return static_cast(from); +} + +template +T* DownCastToGenerated(MessageLite* from) { + return const_cast( + DownCastToGenerated(static_cast(from))); +} + +template +const T& DownCastToGenerated(const MessageLite& from) { + return *DownCastToGenerated(&from); +} + +template +T& DownCastToGenerated(MessageLite& from) { + return *DownCastToGenerated(&from); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index 08b00a482f..2f647fbf4a 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -753,35 +753,35 @@ TEST(MESSAGE_TEST_NAME, InitializationErrorString) { TEST(MESSAGE_TEST_NAME, DynamicCastToGenerated) { UNITTEST::TestAllTypes test_all_types; - Message* test_all_types_pointer = &test_all_types; + MessageLite* test_all_types_pointer = &test_all_types; EXPECT_EQ(&test_all_types, DynamicCastToGenerated( test_all_types_pointer)); EXPECT_EQ(nullptr, DynamicCastToGenerated( test_all_types_pointer)); - const Message* test_all_types_pointer_const = &test_all_types; + const MessageLite* test_all_types_pointer_const = &test_all_types; EXPECT_EQ(&test_all_types, DynamicCastToGenerated( test_all_types_pointer_const)); EXPECT_EQ(nullptr, DynamicCastToGenerated( test_all_types_pointer_const)); - Message* test_all_types_pointer_nullptr = nullptr; + MessageLite* test_all_types_pointer_nullptr = nullptr; EXPECT_EQ(nullptr, DynamicCastToGenerated( test_all_types_pointer_nullptr)); - Message& test_all_types_pointer_ref = test_all_types; + MessageLite& test_all_types_pointer_ref = test_all_types; EXPECT_EQ(&test_all_types, &DynamicCastToGenerated( test_all_types_pointer_ref)); - const Message& test_all_types_pointer_const_ref = test_all_types; + const MessageLite& test_all_types_pointer_const_ref = test_all_types; EXPECT_EQ(&test_all_types, &DynamicCastToGenerated( test_all_types_pointer_const_ref)); } TEST(MESSAGE_TEST_NAME, DynamicCastToGeneratedInvalidReferenceType) { UNITTEST::TestAllTypes test_all_types; - const Message& test_all_types_pointer_const_ref = test_all_types; + const MessageLite& test_all_types_pointer_const_ref = test_all_types; ASSERT_DEATH(DynamicCastToGenerated( test_all_types_pointer_const_ref), "Cannot downcast " + test_all_types.GetTypeName() + " to " + @@ -791,23 +791,23 @@ TEST(MESSAGE_TEST_NAME, DynamicCastToGeneratedInvalidReferenceType) { TEST(MESSAGE_TEST_NAME, DownCastToGeneratedValidType) { UNITTEST::TestAllTypes test_all_types; - Message* test_all_types_pointer = &test_all_types; + MessageLite* test_all_types_pointer = &test_all_types; EXPECT_EQ(&test_all_types, DownCastToGenerated( test_all_types_pointer)); - const Message* test_all_types_pointer_const = &test_all_types; + const MessageLite* test_all_types_pointer_const = &test_all_types; EXPECT_EQ(&test_all_types, DownCastToGenerated( test_all_types_pointer_const)); - Message* test_all_types_pointer_nullptr = nullptr; + MessageLite* test_all_types_pointer_nullptr = nullptr; EXPECT_EQ(nullptr, DownCastToGenerated( test_all_types_pointer_nullptr)); - Message& test_all_types_pointer_ref = test_all_types; + MessageLite& test_all_types_pointer_ref = test_all_types; EXPECT_EQ(&test_all_types, &DownCastToGenerated( test_all_types_pointer_ref)); - const Message& test_all_types_pointer_const_ref = test_all_types; + const MessageLite& test_all_types_pointer_const_ref = test_all_types; EXPECT_EQ(&test_all_types, &DownCastToGenerated( test_all_types_pointer_const_ref)); } @@ -815,7 +815,7 @@ TEST(MESSAGE_TEST_NAME, DownCastToGeneratedValidType) { TEST(MESSAGE_TEST_NAME, DownCastToGeneratedInvalidPointerType) { UNITTEST::TestAllTypes test_all_types; - Message* test_all_types_pointer = &test_all_types; + MessageLite* test_all_types_pointer = &test_all_types; ASSERT_DEBUG_DEATH( DownCastToGenerated(test_all_types_pointer), @@ -826,7 +826,7 @@ TEST(MESSAGE_TEST_NAME, DownCastToGeneratedInvalidPointerType) { TEST(MESSAGE_TEST_NAME, DownCastToGeneratedInvalidReferenceType) { UNITTEST::TestAllTypes test_all_types; - Message& test_all_types_pointer = test_all_types; + MessageLite& test_all_types_pointer = test_all_types; ASSERT_DEBUG_DEATH( DownCastToGenerated(test_all_types_pointer), From 13caa08692d522df8812b1c6f6a88c70105d3941 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 08:44:27 -0700 Subject: [PATCH 137/179] Enable textproto conformance tests for "-0" and "-0F". -0 should be treated as distinct from 0 when deciding it is default valued for implicit presence fields. PiperOrigin-RevId: 632162352 --- conformance/text_format_conformance_suite.cc | 25 +++++++++++++------ .../text_format_failure_list_python.txt | 20 +++++++++++++++ .../text_format_failure_list_python_cpp.txt | 4 +++ .../text_format_failure_list_python_upb.txt | 4 +++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index 6025199991..b86566fe33 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -440,13 +440,11 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { absl::StrCat("FloatFieldLargerThanUint64", name_suffix), REQUIRED, absl::StrCat("optional_float: 18446744073709551616", suffix)); // https://protobuf.dev/reference/protobuf/textformat-spec/#literals says - // "-0" is a valid float literal. - // TODO: Figure out if this should count as not setting - // presence or if -0 should be reflected back. - // RunValidTextFormatTestWithExpected( - // absl::StrCat("FloatFieldNegativeZero", name_suffix), REQUIRED, - // absl::StrCat("optional_float: -0", suffix), - // "" /* implicit presence, so zero means unset*/); + // "-0" is a valid float literal. -0 should be considered not the same as 0 + // when considering implicit presence, and so should round trip. + RunValidTextFormatTest(absl::StrCat("FloatFieldNegativeZero", name_suffix), + REQUIRED, + absl::StrCat("optional_float: -0", suffix)); // https://protobuf.dev/reference/protobuf/textformat-spec/#literals says // ".123", "-.123", ".123e2" are a valid float literal. RunValidTextFormatTest(absl::StrCat("FloatFieldNoLeadingZero", name_suffix), @@ -502,6 +500,19 @@ void TextFormatConformanceTestSuiteImpl::RunAllTests() { RunValidTextFormatTestWithExpected( "DoubleFieldOverflowInfinityHugeExponent", REQUIRED, "optional_double: 1e18446744073709551616", "optional_double: inf"); + RunValidTextFormatTestWithExpected( + "DoubleFieldLargeNegativeExponentParsesAsZero", REQUIRED, + "optional_double: 1e-18446744073709551616", ""); + RunValidTextFormatTestWithExpected( + "NegDoubleFieldLargeNegativeExponentParsesAsNegZero", REQUIRED, + "optional_double: -1e-18446744073709551616", "optional_double: -0"); + + RunValidTextFormatTestWithExpected( + "FloatFieldLargeNegativeExponentParsesAsZero", REQUIRED, + "optional_float: 1e-50", ""); + RunValidTextFormatTestWithExpected( + "NegFloatFieldLargeNegativeExponentParsesAsNegZero", REQUIRED, + "optional_float: -1e-50", "optional_float: -0"); // String literals x {Strings, Bytes} for (const auto& field_type : std::vector{"String", "Bytes"}) { diff --git a/conformance/text_format_failure_list_python.txt b/conformance/text_format_failure_list_python.txt index 903160103f..48abf4e55b 100644 --- a/conformance/text_format_failure_list_python.txt +++ b/conformance/text_format_failure_list_python.txt @@ -77,3 +77,23 @@ Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput +Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput +Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput +Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput +Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput diff --git a/conformance/text_format_failure_list_python_cpp.txt b/conformance/text_format_failure_list_python_cpp.txt index 870f686fcb..fc233ed286 100644 --- a/conformance/text_format_failure_list_python_cpp.txt +++ b/conformance/text_format_failure_list_python_cpp.txt @@ -28,6 +28,8 @@ Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.Proto Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput @@ -62,6 +64,8 @@ Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput diff --git a/conformance/text_format_failure_list_python_upb.txt b/conformance/text_format_failure_list_python_upb.txt index a4a6fc69c0..85da8a1660 100644 --- a/conformance/text_format_failure_list_python_upb.txt +++ b/conformance/text_format_failure_list_python_upb.txt @@ -15,6 +15,8 @@ Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput @@ -49,6 +51,8 @@ Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput From dbc2baf5bdba18854fe2dacd63fcfe519c169b9b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 09:24:33 -0700 Subject: [PATCH 138/179] Generate extension and k[FieldName]FieldNumber as constexpr int(s). PiperOrigin-RevId: 632173946 --- protos_generator/gen_messages.cc | 53 ++++++++++++++++++++++++ protos_generator/tests/test_generated.cc | 6 +++ 2 files changed, 59 insertions(+) diff --git a/protos_generator/gen_messages.cc b/protos_generator/gen_messages.cc index 06a0d8e4b0..c8277a57c5 100644 --- a/protos_generator/gen_messages.cc +++ b/protos_generator/gen_messages.cc @@ -7,11 +7,14 @@ #include "protos_generator/gen_messages.h" +#include #include #include #include "google/protobuf/descriptor.pb.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "google/protobuf/descriptor.h" #include "protos_generator/gen_accessors.h" #include "protos_generator/gen_enums.h" @@ -120,6 +123,55 @@ void WriteModelAccessDeclaration(const protobuf::Descriptor* descriptor, output("};\n"); } +std::string UnderscoresToCamelCase(absl::string_view input, + bool cap_next_letter) { + std::string result; + + for (size_t i = 0; i < input.size(); i++) { + if (absl::ascii_islower(input[i])) { + if (cap_next_letter) { + result += absl::ascii_toupper(input[i]); + } else { + result += input[i]; + } + cap_next_letter = false; + } else if (absl::ascii_isupper(input[i])) { + // Capital letters are left as-is. + result += input[i]; + cap_next_letter = false; + } else if (absl::ascii_isdigit(input[i])) { + result += input[i]; + cap_next_letter = true; + } else { + cap_next_letter = true; + } + } + return result; +} + +std::string FieldConstantName(const protobuf::FieldDescriptor* field) { + std::string field_name = UnderscoresToCamelCase(field->name(), true); + std::string result = absl::StrCat("k", field_name, "FieldNumber"); + + if (!field->is_extension() && + field->containing_type()->FindFieldByCamelcaseName( + field->camelcase_name()) != field) { + // This field's camelcase name is not unique, add field number to make it + // unique. + absl::StrAppend(&result, "_", field->number()); + } + return result; +} + +void WriteConstFieldNumbers(Output& output, + const protobuf::Descriptor* descriptor) { + for (auto field : FieldRange(descriptor)) { + output("static constexpr ::uint32_t $0 = $1;\n", FieldConstantName(field), + field->number()); + } + output("\n\n"); +} + void WriteModelPublicDeclaration( const protobuf::Descriptor* descriptor, const std::vector& file_exts, @@ -178,6 +230,7 @@ void WriteModelPublicDeclaration( )cc", ClassName(descriptor)); output("\n"); + WriteConstFieldNumbers(output, descriptor); output( R"cc( private: diff --git a/protos_generator/tests/test_generated.cc b/protos_generator/tests/test_generated.cc index ffa3d5cb8d..d745eb01cf 100644 --- a/protos_generator/tests/test_generated.cc +++ b/protos_generator/tests/test_generated.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -1134,6 +1135,11 @@ TEST(CppGeneratedCode, HasExtensionAndRegistry) { EXPECT_TRUE(::protos::HasExtension(&parsed_model, theme)); } +TEST(CppGeneratedCode, FieldNumberConstants) { + static_assert(TestModel::kChildMapFieldNumber == 225); + EXPECT_EQ(225, TestModel::kChildMapFieldNumber); +} + // TODO : Add BUILD rule to test failures below. #ifdef TEST_CLEAR_MESSAGE_FAILURE TEST(CppGeneratedCode, ClearConstMessageShouldFail) { From 510c1434ed16dadeeb64a3b841c9ada96e273573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Thu, 9 May 2024 09:25:30 -0700 Subject: [PATCH 139/179] Add `null` to the list of reserved words. The previous change claimed to do this in addition to `true` and `false`, but it was not actually in the code. To reiterate the text from the earlier change, now actually reflected entirely in the code: > The identifiers `true`, `false`, and `null` are effectively reserved words in Java, although for some reason they are listed separately from the "keywords" in the Java Language Specification. > > This doesn't matter for regular fields, because a proto field called `true` will be accessed with `getTrue` and `setTrue`. But for extensions, the generated Java code will have a public static field whose name is the same as the name of the extension field, with `_` appended if the name is a reserved word. Previously there was no `_` for `true` etc, so the generated code would not compile. This change should not affect any existing client code in Java. If someone had tried to use an extension called `true` (etc), they would have found that the generated proto code did not compile. Now it is possible to reference such an extension as `true_`. PiperOrigin-RevId: 632174190 --- src/google/protobuf/compiler/java/names.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/google/protobuf/compiler/java/names.cc b/src/google/protobuf/compiler/java/names.cc index 7f026ecaf6..6df146353c 100644 --- a/src/google/protobuf/compiler/java/names.cc +++ b/src/google/protobuf/compiler/java/names.cc @@ -39,17 +39,17 @@ const char* DefaultPackage(Options options) { bool IsReservedName(absl::string_view name) { static const auto& kReservedNames = *new absl::flat_hash_set({ - "abstract", "assert", "boolean", "break", "byte", - "case", "catch", "char", "class", "const", - "continue", "default", "do", "double", "else", - "enum", "extends", "false", "final", "finally", - "float", "for", "goto", "if", "implements", - "import", "instanceof", "int", "interface", "long", - "native", "new", "package", "private", "protected", - "public", "return", "short", "static", "strictfp", - "super", "switch", "synchronized", "this", "throw", - "throws", "transient", "true", "try", "void", - "volatile", "while", + "abstract", "assert", "boolean", "break", "byte", + "case", "catch", "char", "class", "const", + "continue", "default", "do", "double", "else", + "enum", "extends", "false", "final", "finally", + "float", "for", "goto", "if", "implements", + "import", "instanceof", "int", "interface", "long", + "native", "new", "null", "package", "private", + "protected", "public", "return", "short", "static", + "strictfp", "super", "switch", "synchronized", "this", + "throw", "throws", "transient", "true", "try", + "void", "volatile", "while", }); return kReservedNames.contains(name); } From 82e83ddc95330bbd988ba874ea5aa37d343d4490 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Thu, 9 May 2024 10:19:09 -0700 Subject: [PATCH 140/179] wire_format.cc: Refactor Message and Group handling into a helper func PiperOrigin-RevId: 632191495 --- src/google/protobuf/wire_format.cc | 81 +++++++++++++++--------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index 72e13c4ba7..99db63f76e 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -766,6 +766,44 @@ struct WireFormat::MessageSetParser { const Reflection* reflection; }; +static const char* HandleMessage(Message* msg, const char* ptr, + internal::ParseContext* ctx, uint64_t tag, + const Reflection* reflection, + const FieldDescriptor* field) { + Message* sub_message; + if (field->is_repeated()) { + sub_message = reflection->AddMessage(msg, field, ctx->data().factory); + } else { + sub_message = reflection->MutableMessage(msg, field, ctx->data().factory); + } + + if (WireFormatLite::GetTagWireType(tag) == + WireFormatLite::WIRETYPE_START_GROUP) { + return ctx->ParseGroup(sub_message, ptr, tag); + } else { + ABSL_DCHECK(WireFormatLite::GetTagWireType(tag) == + WireFormatLite::WIRETYPE_LENGTH_DELIMITED); + } + + ptr = ctx->ParseMessage(sub_message, ptr); + + // For map entries, if the value is an unknown enum we have to push it + // into the unknown field set and remove it from the list. + if (ptr != nullptr && field->is_map()) { + auto* value_field = field->message_type()->map_value(); + auto* enum_type = value_field->enum_type(); + if (enum_type != nullptr && + !internal::cpp::HasPreservingUnknownEnumSemantics(value_field) && + enum_type->FindValueByNumber(sub_message->GetReflection()->GetEnumValue( + *sub_message, value_field)) == nullptr) { + reflection->MutableUnknownFields(msg)->AddLengthDelimited( + field->number(), sub_message->SerializeAsString()); + reflection->RemoveLast(msg, field); + } + } + return ptr; +} + const char* WireFormat::_InternalParse(Message* msg, const char* ptr, internal::ParseContext* ctx) { const Descriptor* descriptor = msg->GetDescriptor(); @@ -995,46 +1033,9 @@ const char* WireFormat::_InternalParseAndMergeField( return ptr; } - case FieldDescriptor::TYPE_GROUP: { - Message* sub_message; - if (field->is_repeated()) { - sub_message = reflection->AddMessage(msg, field, ctx->data().factory); - } else { - sub_message = - reflection->MutableMessage(msg, field, ctx->data().factory); - } - - return ctx->ParseGroup(sub_message, ptr, tag); - } - - case FieldDescriptor::TYPE_MESSAGE: { - Message* sub_message; - if (field->is_repeated()) { - sub_message = reflection->AddMessage(msg, field, ctx->data().factory); - } else { - sub_message = - reflection->MutableMessage(msg, field, ctx->data().factory); - } - ptr = ctx->ParseMessage(sub_message, ptr); - - // For map entries, if the value is an unknown enum we have to push it - // into the unknown field set and remove it from the list. - if (ptr != nullptr && field->is_map()) { - auto* value_field = field->message_type()->map_value(); - auto* enum_type = value_field->enum_type(); - if (enum_type != nullptr && - !internal::cpp::HasPreservingUnknownEnumSemantics(value_field) && - enum_type->FindValueByNumber( - sub_message->GetReflection()->GetEnumValue( - *sub_message, value_field)) == nullptr) { - reflection->MutableUnknownFields(msg)->AddLengthDelimited( - field->number(), sub_message->SerializeAsString()); - reflection->RemoveLast(msg, field); - } - } - - return ptr; - } + case FieldDescriptor::TYPE_MESSAGE: + case FieldDescriptor::TYPE_GROUP: + return HandleMessage(msg, ptr, ctx, tag, reflection, field); } // GCC 8 complains about control reaching end of non-void function here. From 495ba7bcdbd806306f0dfab128b2081c17fd66fd Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Thu, 9 May 2024 11:51:48 -0700 Subject: [PATCH 141/179] =?UTF-8?q?Nextgen=20Proto=20Pythonic=20API:=20=20?= =?UTF-8?q?=E2=80=9CAdd-on=E2=80=9D=20proto=20for=20serialize/parse=20-=20?= =?UTF-8?q?add=20google.protobuf.proto=20module=20-=20wrap=20generated=20S?= =?UTF-8?q?erializeToString=20and=20ParseFromString=20to=20the=20new=20mod?= =?UTF-8?q?ule:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit def serialize(message: Message, deterministic: bool=None) -> bytes: """Return the serialized proto.""" def parse(message_class: typing.Type[Message], payload: bytes) -> Message: """Given a serialized proto, deserialize it into a Message.""" PiperOrigin-RevId: 632223409 --- python/google/protobuf/internal/proto_test.py | 34 ++++++++++++++++ python/google/protobuf/proto.py | 39 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 python/google/protobuf/internal/proto_test.py create mode 100644 python/google/protobuf/proto.py diff --git a/python/google/protobuf/internal/proto_test.py b/python/google/protobuf/internal/proto_test.py new file mode 100644 index 0000000000..6bf5b54471 --- /dev/null +++ b/python/google/protobuf/internal/proto_test.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Tests Nextgen Pythonic protobuf APIs.""" + +import unittest + +from google.protobuf import proto + +from google.protobuf.internal import test_util +from google.protobuf.internal import testing_refleaks +from google.protobuf.internal import _parameterized +from google.protobuf import unittest_pb2 +from google.protobuf import unittest_proto3_arena_pb2 + +@_parameterized.named_parameters(('_proto2', unittest_pb2), + ('_proto3', unittest_proto3_arena_pb2)) +@testing_refleaks.TestCase +class ProtoTest(unittest.TestCase): + + def testSerializeParse(self, message_module): + msg = message_module.TestAllTypes() + test_util.SetAllFields(msg) + serialized_data = proto.serialize(msg) + parsed_msg = proto.parse(message_module.TestAllTypes, serialized_data) + self.assertEqual(msg, parsed_msg) + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/proto.py b/python/google/protobuf/proto.py new file mode 100644 index 0000000000..722bbb23fc --- /dev/null +++ b/python/google/protobuf/proto.py @@ -0,0 +1,39 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Nextgen Pythonic protobuf APIs.""" + +import typing + +from google.protobuf.message import Message + +def serialize(message: Message, deterministic: bool=None) -> bytes: + """Return the serialized proto. + + Args: + message: The proto message to be serialized. + deterministic: If true, requests deterministic serialization + of the protobuf, with predictable ordering of map keys. + + Returns: + A binary bytes representation of the message. + """ + return message.SerializeToString(deterministic=deterministic) + +def parse(message_class: typing.Type[Message], payload: bytes) -> Message: + """Given a serialized data in binary form, deserialize it into a Message. + + Args: + message_class: The message meta class. + payload: A serialized bytes in binary form. + + Returns: + A new message deserialized from payload. + """ + new_message = message_class() + new_message.ParseFromString(payload) + return new_message From f68350225f52b81a0511e7ed114894c8dd31e9a5 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Thu, 9 May 2024 14:10:55 -0700 Subject: [PATCH 142/179] Move system_python.bzl to the correct open source folder PiperOrigin-RevId: 632265705 --- bazel/system_python.bzl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 bazel/system_python.bzl diff --git a/bazel/system_python.bzl b/bazel/system_python.bzl new file mode 100644 index 0000000000..5126b9bce0 --- /dev/null +++ b/bazel/system_python.bzl @@ -0,0 +1,17 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Temporary alias to repository rule for using Python 3.x headers from the system.""" + +load( + "//python/dist:system_python.bzl", + _system_python = "system_python", +) + +# TODO: Temporary alias. This is deprecated and to be removed in a future +# release. Users should now get system_python from protobuf_deps.bzl. +system_python = _system_python From 62457336552669c6a4f1403112f4d58c45998cb3 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Thu, 9 May 2024 14:16:20 -0700 Subject: [PATCH 143/179] upb: remove the unused (upb_MiniTable*) arg from upb_Message_SetMessage() PiperOrigin-RevId: 632267379 --- upb/message/accessors.h | 3 +-- upb/message/accessors_test.cc | 6 ++---- upb/message/copy_test.cc | 6 ++---- upb/message/internal/accessors.h | 7 +++---- upb/message/promote.c | 4 ++-- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index ebdfa52bef..b0425dbc9c 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -153,9 +153,8 @@ UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, int64_t value, upb_Arena* a); UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, - const upb_MiniTable* m, const upb_MiniTableField* f, - upb_Message* sub_message); + upb_Message* value); UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, const upb_MiniTableField* f, diff --git a/upb/message/accessors_test.cc b/upb/message/accessors_test.cc index b6d8d42667..d14daf2312 100644 --- a/upb/message/accessors_test.cc +++ b/upb/message/accessors_test.cc @@ -329,10 +329,8 @@ TEST(GeneratedCode, SubMessage) { arena)); upb_Message_SetInt32(new_nested_message, nested_message_a_field, 123, nullptr); - upb_Message_SetMessage( - UPB_UPCAST(msg), - &protobuf_0test_0messages__proto2__TestAllTypesProto2_msg_init, - optional_message_field, new_nested_message); + upb_Message_SetMessage(UPB_UPCAST(msg), optional_message_field, + new_nested_message); upb_Message* mutable_message = upb_Message_GetOrCreateMutableMessage( UPB_UPCAST(msg), diff --git a/upb/message/copy_test.cc b/upb/message/copy_test.cc index 8434258cb2..1bdb3a3099 100644 --- a/upb/message/copy_test.cc +++ b/upb/message/copy_test.cc @@ -108,10 +108,8 @@ TEST(GeneratedCode, DeepCloneMessageSubMessage) { source_arena); protobuf_test_messages_proto2_TestAllTypesProto2_NestedMessage_set_a( nested, kTestNestedInt32); - upb_Message_SetMessage( - UPB_UPCAST(msg), - &protobuf_0test_0messages__proto2__TestAllTypesProto2_msg_init, - nested_message_field, UPB_UPCAST(nested)); + upb_Message_SetMessage(UPB_UPCAST(msg), nested_message_field, + UPB_UPCAST(nested)); upb_Arena* arena = upb_Arena_New(); protobuf_test_messages_proto2_TestAllTypesProto2* clone = (protobuf_test_messages_proto2_TestAllTypesProto2*)upb_Message_DeepClone( diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index d81ddd7e17..31d3ff8dd3 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -791,13 +791,12 @@ UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, } // Sets the value of a message-typed field. The mini_tables of `msg` and -// `sub_message` must have been linked for this to work correctly. +// `value` must have been linked for this to work correctly. UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, - const upb_MiniTable* m, const upb_MiniTableField* f, - struct upb_Message* sub_message) { + struct upb_Message* value) { UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); + (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(value, false)); } // Sets the value of a `string` or `bytes` field. The bytes of the value are not diff --git a/upb/message/promote.c b/upb/message/promote.c index 6496e23a68..566049245a 100644 --- a/upb/message/promote.c +++ b/upb/message/promote.c @@ -188,7 +188,7 @@ upb_DecodeStatus upb_Message_PromoteMessage(upb_Message* parent, upb_Message_PromoteOne(&tagged, sub_table, decode_options, arena); if (ret == kUpb_DecodeStatus_Ok) { *promoted = upb_TaggedMessagePtr_GetNonEmptyMessage(tagged); - upb_Message_SetMessage(parent, mini_table, field, *promoted); + upb_Message_SetMessage(parent, field, *promoted); } return ret; } @@ -280,7 +280,7 @@ upb_UnknownToMessageRet upb_MiniTable_PromoteUnknownToMessage( if (is_oneof) { UPB_PRIVATE(_upb_Message_SetOneofCase)(msg, field); } - upb_Message_SetMessage(msg, mini_table, field, message); + upb_Message_SetMessage(msg, field, message); ret.message = message; } return ret; From b51dc1b438485e6b147c6b874d1859f1e9ea7da3 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 21:34:06 +0000 Subject: [PATCH 144/179] Auto-generate files after cl/632267379 --- php/ext/google/protobuf/php-upb.h | 10 ++++------ ruby/ext/google/protobuf_c/ruby-upb.h | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index a8e5a0db31..83e492914f 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -3368,13 +3368,12 @@ UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, } // Sets the value of a message-typed field. The mini_tables of `msg` and -// `sub_message` must have been linked for this to work correctly. +// `value` must have been linked for this to work correctly. UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, - const upb_MiniTable* m, const upb_MiniTableField* f, - struct upb_Message* sub_message) { + struct upb_Message* value) { UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); + (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(value, false)); } // Sets the value of a `string` or `bytes` field. The bytes of the value are not @@ -3816,9 +3815,8 @@ UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, int64_t value, upb_Arena* a); UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, - const upb_MiniTable* m, const upb_MiniTableField* f, - upb_Message* sub_message); + upb_Message* value); UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, const upb_MiniTableField* f, diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 63ff2c9b8a..bb4c847a1a 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -3370,13 +3370,12 @@ UPB_API_INLINE bool upb_Message_SetInt64(struct upb_Message* msg, } // Sets the value of a message-typed field. The mini_tables of `msg` and -// `sub_message` must have been linked for this to work correctly. +// `value` must have been linked for this to work correctly. UPB_API_INLINE void upb_Message_SetMessage(struct upb_Message* msg, - const upb_MiniTable* m, const upb_MiniTableField* f, - struct upb_Message* sub_message) { + struct upb_Message* value) { UPB_PRIVATE(_upb_Message_SetTaggedMessagePtr) - (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(sub_message, false)); + (msg, f, UPB_PRIVATE(_upb_TaggedMessagePtr_Pack)(value, false)); } // Sets the value of a `string` or `bytes` field. The bytes of the value are not @@ -3818,9 +3817,8 @@ UPB_API_INLINE bool upb_Message_SetInt64(upb_Message* msg, int64_t value, upb_Arena* a); UPB_API_INLINE void upb_Message_SetMessage(upb_Message* msg, - const upb_MiniTable* m, const upb_MiniTableField* f, - upb_Message* sub_message); + upb_Message* value); UPB_API_INLINE bool upb_Message_SetString(upb_Message* msg, const upb_MiniTableField* f, From 5a91d6fe5e3fbe667eed9b27ed4d8293efb0cf56 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Thu, 9 May 2024 14:39:58 -0700 Subject: [PATCH 145/179] [php] Added a unit test that bad UTF-8 is rejected in the parser. PiperOrigin-RevId: 632274113 --- php/src/Google/Protobuf/Internal/Message.php | 3 ++- php/tests/EncodeDecodeTest.php | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/php/src/Google/Protobuf/Internal/Message.php b/php/src/Google/Protobuf/Internal/Message.php index 21d9c980e0..31e2f29d30 100644 --- a/php/src/Google/Protobuf/Internal/Message.php +++ b/php/src/Google/Protobuf/Internal/Message.php @@ -394,7 +394,8 @@ class Message } break; case GPBType::STRING: - // TODO: Add utf-8 check. + // We don't check UTF-8 here; that will be validated by the + // setter later. if (!GPBWire::readString($input, $value)) { throw new GPBDecodeException( "Unexpected EOF inside string field."); diff --git a/php/tests/EncodeDecodeTest.php b/php/tests/EncodeDecodeTest.php index 276528d26e..abf95cca00 100644 --- a/php/tests/EncodeDecodeTest.php +++ b/php/tests/EncodeDecodeTest.php @@ -683,6 +683,21 @@ class EncodeDecodeTest extends TestBase $m->mergeFromString(hex2bin('7201')); } + public function testDecodeInvalidStringDataBadUtf8() + { + $this->expectException(Exception::class); + + $m = new TestMessage(); + $m->mergeFromString(hex2bin('720180')); + } + + public function testDecodeValidStringData() + { + $m = new TestMessage(); + $m->mergeFromString(hex2bin('720161')); + $this->assertSame('a', $m->getOptionalString()); + } + public function testDecodeInvalidBytesLengthMiss() { $this->expectException(Exception::class); From b5fc2735e956be3dfb03de55fd25cd98b398ba51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89amonn=20McManus?= Date: Thu, 9 May 2024 14:53:06 -0700 Subject: [PATCH 146/179] In `upb_MiniTableExtension_GetSubMessage`, return NULL for non-messages. This parallels the logic in `upb_MiniTable_SubMessage`. PiperOrigin-RevId: 632277936 --- upb/mini_table/internal/extension.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/upb/mini_table/internal/extension.h b/upb/mini_table/internal/extension.h index c7b674cd87..7220c5c6e0 100644 --- a/upb/mini_table/internal/extension.h +++ b/upb/mini_table/internal/extension.h @@ -8,6 +8,7 @@ #ifndef UPB_MINI_TABLE_INTERNAL_EXTENSION_H_ #define UPB_MINI_TABLE_INTERNAL_EXTENSION_H_ +#include #include #include "upb/base/descriptor_constants.h" @@ -41,6 +42,9 @@ upb_MiniTableExtension_Number(const struct upb_MiniTableExtension* e) { UPB_API_INLINE const struct upb_MiniTable* upb_MiniTableExtension_GetSubMessage( const struct upb_MiniTableExtension* e) { + if (upb_MiniTableExtension_CType(e) != kUpb_CType_Message) { + return NULL; + } return upb_MiniTableSub_Message(e->UPB_PRIVATE(sub)); } From eb31de309782c94ac887aa7325630b6d5da1a623 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 22:09:21 +0000 Subject: [PATCH 147/179] Auto-generate files after cl/632277936 --- php/ext/google/protobuf/php-upb.h | 4 ++++ ruby/ext/google/protobuf_c/ruby-upb.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 83e492914f..0ae74fb214 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -1841,6 +1841,7 @@ UPB_INLINE uint64_t upb_BigEndian64(uint64_t val) { #ifndef UPB_MINI_TABLE_INTERNAL_EXTENSION_H_ #define UPB_MINI_TABLE_INTERNAL_EXTENSION_H_ +#include #include @@ -1870,6 +1871,9 @@ upb_MiniTableExtension_Number(const struct upb_MiniTableExtension* e) { UPB_API_INLINE const struct upb_MiniTable* upb_MiniTableExtension_GetSubMessage( const struct upb_MiniTableExtension* e) { + if (upb_MiniTableExtension_CType(e) != kUpb_CType_Message) { + return NULL; + } return upb_MiniTableSub_Message(e->UPB_PRIVATE(sub)); } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index bb4c847a1a..9d6cdcffe6 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -1843,6 +1843,7 @@ UPB_INLINE uint64_t upb_BigEndian64(uint64_t val) { #ifndef UPB_MINI_TABLE_INTERNAL_EXTENSION_H_ #define UPB_MINI_TABLE_INTERNAL_EXTENSION_H_ +#include #include @@ -1872,6 +1873,9 @@ upb_MiniTableExtension_Number(const struct upb_MiniTableExtension* e) { UPB_API_INLINE const struct upb_MiniTable* upb_MiniTableExtension_GetSubMessage( const struct upb_MiniTableExtension* e) { + if (upb_MiniTableExtension_CType(e) != kUpb_CType_Message) { + return NULL; + } return upb_MiniTableSub_Message(e->UPB_PRIVATE(sub)); } From af1ecc03531461900c1c9b8499aa64f5900bca27 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 15:16:16 -0700 Subject: [PATCH 148/179] Add a check to ensure that the byte limit is not negative. PiperOrigin-RevId: 632284538 --- .../src/main/java/com/google/protobuf/CodedInputStream.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index 81da417783..6fb96afa4b 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -2690,6 +2690,9 @@ public abstract class CodedInputStream { throw InvalidProtocolBufferException.negativeSize(); } byteLimit += totalBytesRetired + pos; + if (byteLimit < 0) { + throw InvalidProtocolBufferException.parseFailure(); + } final int oldLimit = currentLimit; if (byteLimit > oldLimit) { throw InvalidProtocolBufferException.truncatedMessage(); From fefec687536234a46f96d0c6e209dabceaf18466 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Thu, 9 May 2024 15:52:11 -0700 Subject: [PATCH 149/179] Internal changes/optimizations PiperOrigin-RevId: 632294209 --- java/core/BUILD.bazel | 2 + .../com/google/protobuf/ExtensionSchemas.java | 2 + .../com/google/protobuf/ListFieldSchema.java | 153 +----------------- .../google/protobuf/ListFieldSchemaFull.java | 99 ++++++++++++ .../google/protobuf/ListFieldSchemaLite.java | 59 +++++++ .../com/google/protobuf/ListFieldSchemas.java | 33 ++++ .../protobuf/ManifestSchemaFactory.java | 58 +++---- .../google/protobuf/MapFieldSchemaFull.java | 2 +- .../google/protobuf/MapFieldSchemaLite.java | 2 +- .../com/google/protobuf/MapFieldSchemas.java | 2 + .../google/protobuf/NewInstanceSchemas.java | 2 + java/lite/pom.xml | 2 + 12 files changed, 228 insertions(+), 188 deletions(-) create mode 100644 java/core/src/main/java/com/google/protobuf/ListFieldSchemaFull.java create mode 100644 java/core/src/main/java/com/google/protobuf/ListFieldSchemaLite.java create mode 100644 java/core/src/main/java/com/google/protobuf/ListFieldSchemas.java diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 683b0654ac..6127b4c5ca 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -60,6 +60,8 @@ LITE_SRCS = [ "src/main/java/com/google/protobuf/LazyStringArrayList.java", "src/main/java/com/google/protobuf/LazyStringList.java", "src/main/java/com/google/protobuf/ListFieldSchema.java", + "src/main/java/com/google/protobuf/ListFieldSchemaLite.java", + "src/main/java/com/google/protobuf/ListFieldSchemas.java", "src/main/java/com/google/protobuf/LongArrayList.java", "src/main/java/com/google/protobuf/ManifestSchemaFactory.java", "src/main/java/com/google/protobuf/MapEntryLite.java", diff --git a/java/core/src/main/java/com/google/protobuf/ExtensionSchemas.java b/java/core/src/main/java/com/google/protobuf/ExtensionSchemas.java index cd8c852e9e..5f0ce06685 100644 --- a/java/core/src/main/java/com/google/protobuf/ExtensionSchemas.java +++ b/java/core/src/main/java/com/google/protobuf/ExtensionSchemas.java @@ -31,4 +31,6 @@ final class ExtensionSchemas { } return FULL_SCHEMA; } + + private ExtensionSchemas() {} } diff --git a/java/core/src/main/java/com/google/protobuf/ListFieldSchema.java b/java/core/src/main/java/com/google/protobuf/ListFieldSchema.java index bc55eb23c7..eaab68b8d3 100644 --- a/java/core/src/main/java/com/google/protobuf/ListFieldSchema.java +++ b/java/core/src/main/java/com/google/protobuf/ListFieldSchema.java @@ -7,162 +7,17 @@ package com.google.protobuf; -import com.google.protobuf.Internal.ProtobufList; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** * Utility class that aids in properly manipulating list fields for either the lite or full runtime. */ @CheckReturnValue -abstract class ListFieldSchema { - // Disallow construction. - private ListFieldSchema() {} +interface ListFieldSchema { - private static final ListFieldSchema FULL_INSTANCE = new ListFieldSchemaFull(); - private static final ListFieldSchema LITE_INSTANCE = new ListFieldSchemaLite(); + List mutableListAt(Object msg, long offset); - abstract List mutableListAt(Object msg, long offset); + void makeImmutableListAt(Object msg, long offset); - abstract void makeImmutableListAt(Object msg, long offset); - - abstract void mergeListsAt(Object msg, Object otherMsg, long offset); - - static ListFieldSchema full() { - return FULL_INSTANCE; - } - - static ListFieldSchema lite() { - return LITE_INSTANCE; - } - - /** Implementation for the full runtime. */ - private static final class ListFieldSchemaFull extends ListFieldSchema { - - private static final Class UNMODIFIABLE_LIST_CLASS = - Collections.unmodifiableList(Collections.emptyList()).getClass(); - - @Override - List mutableListAt(Object message, long offset) { - return mutableListAt(message, offset, AbstractProtobufList.DEFAULT_CAPACITY); - } - - @Override - void makeImmutableListAt(Object message, long offset) { - List list = (List) UnsafeUtil.getObject(message, offset); - Object immutable = null; - if (list instanceof LazyStringList) { - immutable = ((LazyStringList) list).getUnmodifiableView(); - } else if (UNMODIFIABLE_LIST_CLASS.isAssignableFrom(list.getClass())) { - // already immutable - return; - } else if (list instanceof PrimitiveNonBoxingCollection && list instanceof ProtobufList) { - if (((ProtobufList) list).isModifiable()) { - ((ProtobufList) list).makeImmutable(); - } - return; - } else { - immutable = Collections.unmodifiableList((List) list); - } - UnsafeUtil.putObject(message, offset, immutable); - } - - @SuppressWarnings("unchecked") - private static List mutableListAt(Object message, long offset, int additionalCapacity) { - List list = getList(message, offset); - if (list.isEmpty()) { - if (list instanceof LazyStringList) { - list = (List) new LazyStringArrayList(additionalCapacity); - } else if (list instanceof PrimitiveNonBoxingCollection && list instanceof ProtobufList) { - list = ((ProtobufList) list).mutableCopyWithCapacity(additionalCapacity); - } else { - list = new ArrayList(additionalCapacity); - } - UnsafeUtil.putObject(message, offset, list); - } else if (UNMODIFIABLE_LIST_CLASS.isAssignableFrom(list.getClass())) { - ArrayList newList = new ArrayList(list.size() + additionalCapacity); - newList.addAll(list); - list = newList; - UnsafeUtil.putObject(message, offset, list); - } else if (list instanceof UnmodifiableLazyStringList) { - LazyStringArrayList newList = new LazyStringArrayList(list.size() + additionalCapacity); - newList.addAll((UnmodifiableLazyStringList) list); - list = (List) newList; - UnsafeUtil.putObject(message, offset, list); - } else if (list instanceof PrimitiveNonBoxingCollection - && list instanceof ProtobufList - && !((ProtobufList) list).isModifiable()) { - list = ((ProtobufList) list).mutableCopyWithCapacity(list.size() + additionalCapacity); - UnsafeUtil.putObject(message, offset, list); - } - return list; - } - - @Override - void mergeListsAt(Object msg, Object otherMsg, long offset) { - List other = getList(otherMsg, offset); - List mine = mutableListAt(msg, offset, other.size()); - - int size = mine.size(); - int otherSize = other.size(); - if (size > 0 && otherSize > 0) { - mine.addAll(other); - } - - List merged = size > 0 ? mine : other; - UnsafeUtil.putObject(msg, offset, merged); - } - - @SuppressWarnings("unchecked") - static List getList(Object message, long offset) { - return (List) UnsafeUtil.getObject(message, offset); - } - } - - /** Implementation for the lite runtime. */ - private static final class ListFieldSchemaLite extends ListFieldSchema { - - @Override - List mutableListAt(Object message, long offset) { - ProtobufList list = getProtobufList(message, offset); - if (!list.isModifiable()) { - int size = list.size(); - list = - list.mutableCopyWithCapacity( - size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2); - UnsafeUtil.putObject(message, offset, list); - } - return list; - } - - @Override - void makeImmutableListAt(Object message, long offset) { - ProtobufList list = getProtobufList(message, offset); - list.makeImmutable(); - } - - @Override - void mergeListsAt(Object msg, Object otherMsg, long offset) { - ProtobufList mine = getProtobufList(msg, offset); - ProtobufList other = getProtobufList(otherMsg, offset); - - int size = mine.size(); - int otherSize = other.size(); - if (size > 0 && otherSize > 0) { - if (!mine.isModifiable()) { - mine = mine.mutableCopyWithCapacity(size + otherSize); - } - mine.addAll(other); - } - - ProtobufList merged = size > 0 ? mine : other; - UnsafeUtil.putObject(msg, offset, merged); - } - - @SuppressWarnings("unchecked") - static ProtobufList getProtobufList(Object message, long offset) { - return (ProtobufList) UnsafeUtil.getObject(message, offset); - } - } + void mergeListsAt(Object msg, Object otherMsg, long offset); } diff --git a/java/core/src/main/java/com/google/protobuf/ListFieldSchemaFull.java b/java/core/src/main/java/com/google/protobuf/ListFieldSchemaFull.java new file mode 100644 index 0000000000..8f0308ab63 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/ListFieldSchemaFull.java @@ -0,0 +1,99 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +package com.google.protobuf; + +import com.google.protobuf.Internal.ProtobufList; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Utility class that aids in properly manipulating list fields for either the lite or full runtime. + */ +@CheckReturnValue +final class ListFieldSchemaFull implements ListFieldSchema { + + private static final Class UNMODIFIABLE_LIST_CLASS = + Collections.unmodifiableList(Collections.emptyList()).getClass(); + + @Override + public List mutableListAt(Object message, long offset) { + return mutableListAt(message, offset, AbstractProtobufList.DEFAULT_CAPACITY); + } + + @SuppressWarnings("unchecked") + private static List mutableListAt(Object message, long offset, int additionalCapacity) { + List list = getList(message, offset); + if (list.isEmpty()) { + if (list instanceof LazyStringList) { + list = (List) new LazyStringArrayList(additionalCapacity); + } else if (list instanceof PrimitiveNonBoxingCollection && list instanceof ProtobufList) { + list = ((ProtobufList) list).mutableCopyWithCapacity(additionalCapacity); + } else { + list = new ArrayList(additionalCapacity); + } + UnsafeUtil.putObject(message, offset, list); + } else if (UNMODIFIABLE_LIST_CLASS.isAssignableFrom(list.getClass())) { + ArrayList newList = new ArrayList(list.size() + additionalCapacity); + newList.addAll(list); + list = newList; + UnsafeUtil.putObject(message, offset, list); + } else if (list instanceof UnmodifiableLazyStringList) { + LazyStringArrayList newList = new LazyStringArrayList(list.size() + additionalCapacity); + newList.addAll((UnmodifiableLazyStringList) list); + list = (List) newList; + UnsafeUtil.putObject(message, offset, list); + } else if (list instanceof PrimitiveNonBoxingCollection + && list instanceof ProtobufList + && !((ProtobufList) list).isModifiable()) { + list = ((ProtobufList) list).mutableCopyWithCapacity(list.size() + additionalCapacity); + UnsafeUtil.putObject(message, offset, list); + } + return list; + } + + @Override + public void makeImmutableListAt(Object message, long offset) { + List list = (List) UnsafeUtil.getObject(message, offset); + Object immutable = null; + if (list instanceof LazyStringList) { + immutable = ((LazyStringList) list).getUnmodifiableView(); + } else if (UNMODIFIABLE_LIST_CLASS.isAssignableFrom(list.getClass())) { + // already immutable + return; + } else if (list instanceof PrimitiveNonBoxingCollection && list instanceof ProtobufList) { + if (((ProtobufList) list).isModifiable()) { + ((ProtobufList) list).makeImmutable(); + } + return; + } else { + immutable = Collections.unmodifiableList((List) list); + } + UnsafeUtil.putObject(message, offset, immutable); + } + + @Override + public void mergeListsAt(Object msg, Object otherMsg, long offset) { + List other = getList(otherMsg, offset); + List mine = mutableListAt(msg, offset, other.size()); + + int size = mine.size(); + int otherSize = other.size(); + if (size > 0 && otherSize > 0) { + mine.addAll(other); + } + + List merged = size > 0 ? mine : other; + UnsafeUtil.putObject(msg, offset, merged); + } + + @SuppressWarnings("unchecked") + static List getList(Object message, long offset) { + return (List) UnsafeUtil.getObject(message, offset); + } +} diff --git a/java/core/src/main/java/com/google/protobuf/ListFieldSchemaLite.java b/java/core/src/main/java/com/google/protobuf/ListFieldSchemaLite.java new file mode 100644 index 0000000000..f0cf71ecc0 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/ListFieldSchemaLite.java @@ -0,0 +1,59 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +package com.google.protobuf; + +import com.google.protobuf.Internal.ProtobufList; +import java.util.List; + +/** + * Utility class that aids in properly manipulating list fields for either the lite or full runtime. + */ +final class ListFieldSchemaLite implements ListFieldSchema { + + @Override + public List mutableListAt(Object message, long offset) { + ProtobufList list = getProtobufList(message, offset); + if (!list.isModifiable()) { + int size = list.size(); + list = + list.mutableCopyWithCapacity( + size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2); + UnsafeUtil.putObject(message, offset, list); + } + return list; + } + + @Override + public void makeImmutableListAt(Object message, long offset) { + ProtobufList list = getProtobufList(message, offset); + list.makeImmutable(); + } + + @Override + public void mergeListsAt(Object msg, Object otherMsg, long offset) { + ProtobufList mine = getProtobufList(msg, offset); + ProtobufList other = getProtobufList(otherMsg, offset); + + int size = mine.size(); + int otherSize = other.size(); + if (size > 0 && otherSize > 0) { + if (!mine.isModifiable()) { + mine = mine.mutableCopyWithCapacity(size + otherSize); + } + mine.addAll(other); + } + + ProtobufList merged = size > 0 ? mine : other; + UnsafeUtil.putObject(msg, offset, merged); + } + + @SuppressWarnings("unchecked") + static ProtobufList getProtobufList(Object message, long offset) { + return (ProtobufList) UnsafeUtil.getObject(message, offset); + } +} diff --git a/java/core/src/main/java/com/google/protobuf/ListFieldSchemas.java b/java/core/src/main/java/com/google/protobuf/ListFieldSchemas.java new file mode 100644 index 0000000000..6250b7a2d7 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/ListFieldSchemas.java @@ -0,0 +1,33 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +package com.google.protobuf; + +@CheckReturnValue +final class ListFieldSchemas { + private static final ListFieldSchema FULL_SCHEMA = loadSchemaForFullRuntime(); + private static final ListFieldSchema LITE_SCHEMA = new ListFieldSchemaLite(); + + static ListFieldSchema full() { + return FULL_SCHEMA; + } + + static ListFieldSchema lite() { + return LITE_SCHEMA; + } + + private static ListFieldSchema loadSchemaForFullRuntime() { + try { + Class clazz = Class.forName("com.google.protobuf.ListFieldSchemaFull"); + return (ListFieldSchema) clazz.getDeclaredConstructor().newInstance(); + } catch (Exception e) { + return null; + } + } + + private ListFieldSchemas() {} +} diff --git a/java/core/src/main/java/com/google/protobuf/ManifestSchemaFactory.java b/java/core/src/main/java/com/google/protobuf/ManifestSchemaFactory.java index c923b129db..9b4fd82d47 100644 --- a/java/core/src/main/java/com/google/protobuf/ManifestSchemaFactory.java +++ b/java/core/src/main/java/com/google/protobuf/ManifestSchemaFactory.java @@ -34,57 +34,37 @@ final class ManifestSchemaFactory implements SchemaFactory { // MessageSet has a special schema. if (messageInfo.isMessageSetWireFormat()) { - if (GeneratedMessageLite.class.isAssignableFrom(messageType)) { - return MessageSetSchema.newSchema( - SchemaUtil.unknownFieldSetLiteSchema(), - ExtensionSchemas.lite(), - messageInfo.getDefaultInstance()); - } - return MessageSetSchema.newSchema( - SchemaUtil.unknownFieldSetFullSchema(), - ExtensionSchemas.full(), - messageInfo.getDefaultInstance()); + return useLiteRuntime(messageType) + ? MessageSetSchema.newSchema( + SchemaUtil.unknownFieldSetLiteSchema(), + ExtensionSchemas.lite(), + messageInfo.getDefaultInstance()) + : MessageSetSchema.newSchema( + SchemaUtil.unknownFieldSetFullSchema(), + ExtensionSchemas.full(), + messageInfo.getDefaultInstance()); } return newSchema(messageType, messageInfo); } private static Schema newSchema(Class messageType, MessageInfo messageInfo) { - if (GeneratedMessageLite.class.isAssignableFrom(messageType)) { - return allowExtensions(messageInfo) - ? MessageSchema.newSchema( - messageType, - messageInfo, - NewInstanceSchemas.lite(), - ListFieldSchema.lite(), - SchemaUtil.unknownFieldSetLiteSchema(), - ExtensionSchemas.lite(), - MapFieldSchemas.lite()) - : MessageSchema.newSchema( - messageType, - messageInfo, - NewInstanceSchemas.lite(), - ListFieldSchema.lite(), - SchemaUtil.unknownFieldSetLiteSchema(), - /* extensionSchema= */ null, - MapFieldSchemas.lite()); - } - return allowExtensions(messageInfo) + return useLiteRuntime(messageType) ? MessageSchema.newSchema( messageType, messageInfo, - NewInstanceSchemas.full(), - ListFieldSchema.full(), - SchemaUtil.unknownFieldSetFullSchema(), - ExtensionSchemas.full(), - MapFieldSchemas.full()) + NewInstanceSchemas.lite(), + ListFieldSchemas.lite(), + SchemaUtil.unknownFieldSetLiteSchema(), + allowExtensions(messageInfo) ? ExtensionSchemas.lite() : null, + MapFieldSchemas.lite()) : MessageSchema.newSchema( messageType, messageInfo, NewInstanceSchemas.full(), - ListFieldSchema.full(), + ListFieldSchemas.full(), SchemaUtil.unknownFieldSetFullSchema(), - /* extensionSchema= */ null, + allowExtensions(messageInfo) ? ExtensionSchemas.full() : null, MapFieldSchemas.full()); } @@ -152,4 +132,8 @@ final class ManifestSchemaFactory implements SchemaFactory { return EMPTY_FACTORY; } } + + private static boolean useLiteRuntime(Class messageType) { + return GeneratedMessageLite.class.isAssignableFrom(messageType); + } } diff --git a/java/core/src/main/java/com/google/protobuf/MapFieldSchemaFull.java b/java/core/src/main/java/com/google/protobuf/MapFieldSchemaFull.java index d411449172..624c258d41 100644 --- a/java/core/src/main/java/com/google/protobuf/MapFieldSchemaFull.java +++ b/java/core/src/main/java/com/google/protobuf/MapFieldSchemaFull.java @@ -10,7 +10,7 @@ package com.google.protobuf; import com.google.protobuf.MapEntryLite.Metadata; import java.util.Map; -class MapFieldSchemaFull implements MapFieldSchema { +final class MapFieldSchemaFull implements MapFieldSchema { @Override public Map forMutableMapData(Object mapField) { return ((MapField) mapField).getMutableMap(); diff --git a/java/core/src/main/java/com/google/protobuf/MapFieldSchemaLite.java b/java/core/src/main/java/com/google/protobuf/MapFieldSchemaLite.java index f7c5d85f31..c21678d287 100644 --- a/java/core/src/main/java/com/google/protobuf/MapFieldSchemaLite.java +++ b/java/core/src/main/java/com/google/protobuf/MapFieldSchemaLite.java @@ -11,7 +11,7 @@ import com.google.protobuf.MapEntryLite.Metadata; import java.util.Map; @CheckReturnValue -class MapFieldSchemaLite implements MapFieldSchema { +final class MapFieldSchemaLite implements MapFieldSchema { @Override public Map forMutableMapData(Object mapField) { diff --git a/java/core/src/main/java/com/google/protobuf/MapFieldSchemas.java b/java/core/src/main/java/com/google/protobuf/MapFieldSchemas.java index d7eeccd8c4..ea419a56c6 100644 --- a/java/core/src/main/java/com/google/protobuf/MapFieldSchemas.java +++ b/java/core/src/main/java/com/google/protobuf/MapFieldSchemas.java @@ -28,4 +28,6 @@ final class MapFieldSchemas { return null; } } + + private MapFieldSchemas() {} } diff --git a/java/core/src/main/java/com/google/protobuf/NewInstanceSchemas.java b/java/core/src/main/java/com/google/protobuf/NewInstanceSchemas.java index 800f44891c..6fbdd47a38 100644 --- a/java/core/src/main/java/com/google/protobuf/NewInstanceSchemas.java +++ b/java/core/src/main/java/com/google/protobuf/NewInstanceSchemas.java @@ -28,4 +28,6 @@ final class NewInstanceSchemas { return null; } } + + private NewInstanceSchemas() {} } diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 64a69085f3..d2da130846 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -139,6 +139,8 @@ LazyStringArrayList.java LazyStringList.java ListFieldSchema.java + ListFieldSchemaLite.java + ListFieldSchemas.java LongArrayList.java ManifestSchemaFactory.java MapEntryLite.java From d071c608780d3fa890b191aafb8aa2d4f8634973 Mon Sep 17 00:00:00 2001 From: Hong Shin Date: Thu, 9 May 2024 17:16:10 -0700 Subject: [PATCH 150/179] Break out idx in upb_MiniTableSub computation for clarity PiperOrigin-RevId: 632314763 --- upb/mini_descriptor/link.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/upb/mini_descriptor/link.c b/upb/mini_descriptor/link.c index bc5fa2f65b..093150b623 100644 --- a/upb/mini_descriptor/link.c +++ b/upb/mini_descriptor/link.c @@ -50,12 +50,12 @@ bool upb_MiniTable_SetSubMessage(upb_MiniTable* table, return false; } - upb_MiniTableSub* table_sub = - (void*)&table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]; + int idx = field->UPB_PRIVATE(submsg_index); + upb_MiniTableSub* table_subs = (void*)table->UPB_PRIVATE(subs); // TODO: Add this assert back once YouTube is updated to not call // this function repeatedly. // UPB_ASSERT(UPB_PRIVATE(_upb_MiniTable_IsEmpty)(table_sub->submsg)); - *table_sub = upb_MiniTableSub_FromMessage(sub); + table_subs[idx] = upb_MiniTableSub_FromMessage(sub); return true; } From bc80135f10ca2ad6f307d4a2165cdaa112d68555 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 May 2024 00:30:13 +0000 Subject: [PATCH 151/179] Auto-generate files after cl/632314763 --- php/ext/google/protobuf/php-upb.c | 6 +++--- ruby/ext/google/protobuf_c/ruby-upb.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index d9d7d41472..03a3298878 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -7305,12 +7305,12 @@ bool upb_MiniTable_SetSubMessage(upb_MiniTable* table, return false; } - upb_MiniTableSub* table_sub = - (void*)&table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]; + int idx = field->UPB_PRIVATE(submsg_index); + upb_MiniTableSub* table_subs = (void*)table->UPB_PRIVATE(subs); // TODO: Add this assert back once YouTube is updated to not call // this function repeatedly. // UPB_ASSERT(UPB_PRIVATE(_upb_MiniTable_IsEmpty)(table_sub->submsg)); - *table_sub = upb_MiniTableSub_FromMessage(sub); + table_subs[idx] = upb_MiniTableSub_FromMessage(sub); return true; } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index faa638b8a7..c4cfa69ca9 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -6797,12 +6797,12 @@ bool upb_MiniTable_SetSubMessage(upb_MiniTable* table, return false; } - upb_MiniTableSub* table_sub = - (void*)&table->UPB_PRIVATE(subs)[field->UPB_PRIVATE(submsg_index)]; + int idx = field->UPB_PRIVATE(submsg_index); + upb_MiniTableSub* table_subs = (void*)table->UPB_PRIVATE(subs); // TODO: Add this assert back once YouTube is updated to not call // this function repeatedly. // UPB_ASSERT(UPB_PRIVATE(_upb_MiniTable_IsEmpty)(table_sub->submsg)); - *table_sub = upb_MiniTableSub_FromMessage(sub); + table_subs[idx] = upb_MiniTableSub_FromMessage(sub); return true; } From 8c6e5b79636673435c9a8afe51dd2ffe4a2c7f4b Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Fri, 10 May 2024 08:29:52 -0700 Subject: [PATCH 152/179] Avoid allocating iterator in TextFormat printing UnknownFieldSet PiperOrigin-RevId: 632503842 --- java/core/src/main/java/com/google/protobuf/TextFormat.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index eb05fb2f41..71b2b33f67 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -581,6 +581,9 @@ public final class TextFormat { private static void printUnknownFields( final UnknownFieldSet unknownFields, final TextGenerator generator) throws IOException { + if (unknownFields.isEmpty()) { + return; + } for (Map.Entry entry : unknownFields.asMap().entrySet()) { final int number = entry.getKey(); final UnknownFieldSet.Field field = entry.getValue(); From 4b7c2d43818a71e0c6975fcf3a1f1e0f1533ce3c Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Fri, 10 May 2024 08:33:09 -0700 Subject: [PATCH 153/179] Avoid allocating TreeMap in UnknownFieldSet.Builder.asMap If UnknownFieldSet is empty PiperOrigin-RevId: 632504647 --- .../src/main/java/com/google/protobuf/UnknownFieldSet.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index aa71004da7..2c21c54794 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -483,6 +483,11 @@ public final class UnknownFieldSet implements MessageLite { * changes may or may not be reflected in this map. */ public Map asMap() { + // Avoid an allocation for the common case of an empty map. + if (fieldBuilders.isEmpty()) { + return Collections.emptyMap(); + } + TreeMap fields = new TreeMap<>(); for (Map.Entry entry : fieldBuilders.entrySet()) { fields.put(entry.getKey(), entry.getValue().build()); From 91b7cf3325d6832409f17be86bb2ce282af95293 Mon Sep 17 00:00:00 2001 From: Jaap Taal Date: Fri, 10 May 2024 08:39:18 -0700 Subject: [PATCH 154/179] Fix indention formatter for google.protobuf.Any (#16785) `google.protobuf.Any` formatting with indentation was somewhat off. Formatting an `Any` as root object: ```json {"@type": "type.googleapis.com/protobuf_unittest3.ForeignMessage", "c": 1 } ``` changes to ```json { "@type": "type.googleapis.com/protobuf_unittest3.ForeignMessage", "c": 1 } ``` For messages were `Any` is in a nested field the change makes more of a visual impact. The `c` field seems to be at the same level as the `anyField`, but it's nested so it should be indented: ```json { "anyField": {"@type": "type.googleapis.com/protobuf_unittest3.ForeignMessage", "c": 1 } } ``` changes to: ```json { "anyField": { "@type": "type.googleapis.com/protobuf_unittest3.ForeignMessage", "c": 1 } } ``` Closes #16785 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/16785 from q42jaap:main 72deed6ed774c3206b5a4e925d7c58c114cb2537 PiperOrigin-RevId: 632506226 --- .../Google.Protobuf.Test/JsonFormatterTest.cs | 36 +++++++++++++++++++ csharp/src/Google.Protobuf/JsonFormatter.cs | 5 +-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs index 77da66351e..61adc1da78 100644 --- a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs @@ -900,6 +900,42 @@ namespace Google.Protobuf AssertWriteValue(value, "[\n 1,\n 2,\n 3\n]", JsonFormatter.Settings.Default.WithIndentation()); } + [Test] + public void WriteValueWithIndentation_Any() + { + var registry = TypeRegistry.FromMessages(ForeignMessage.Descriptor); + var formatter = JsonFormatter.Settings.Default.WithIndentation().WithTypeRegistry(registry); + + var nestedMessage = new ForeignMessage { C = 1 }; + var value = Any.Pack(nestedMessage); + const string expectedJson = @" +{ + '@type': 'type.googleapis.com/protobuf_unittest3.ForeignMessage', + 'c': 1 +}"; + + AssertWriteValue(value, expectedJson, formatter); + } + + [Test] + public void WriteValueWithIndentation_NestedAny() + { + var registry = TypeRegistry.FromMessages(ForeignMessage.Descriptor); + var formatter = JsonFormatter.Settings.Default.WithIndentation().WithTypeRegistry(registry); + + var nestedMessage = new ForeignMessage { C = 1 }; + var value = new TestWellKnownTypes { AnyField = Any.Pack(nestedMessage) }; + const string expectedJson = @" +{ + 'anyField': { + '@type': 'type.googleapis.com/protobuf_unittest3.ForeignMessage', + 'c': 1 + } +}"; + + AssertWriteValue(value, expectedJson, formatter); + } + [Test] public void WriteValueWithIndentation_CustomIndentation() { diff --git a/csharp/src/Google.Protobuf/JsonFormatter.cs b/csharp/src/Google.Protobuf/JsonFormatter.cs index 77ca26e7cc..fbcc839707 100644 --- a/csharp/src/Google.Protobuf/JsonFormatter.cs +++ b/csharp/src/Google.Protobuf/JsonFormatter.cs @@ -481,6 +481,7 @@ namespace Google.Protobuf { } IMessage message = descriptor.Parser.ParseFrom(data); WriteBracketOpen(writer, ObjectOpenBracket); + MaybeWriteValueWhitespace(writer, indentationLevel + 1); WriteString(writer, AnyTypeUrlField); writer.Write(NameValueSeparator); WriteString(writer, typeUrl); @@ -489,9 +490,9 @@ namespace Google.Protobuf { writer.Write(ValueSeparator); WriteString(writer, AnyWellKnownTypeValueField); writer.Write(NameValueSeparator); - WriteWellKnownTypeValue(writer, descriptor, message, indentationLevel); + WriteWellKnownTypeValue(writer, descriptor, message, indentationLevel + 1); } else { - WriteMessageFields(writer, message, true, indentationLevel); + WriteMessageFields(writer, message, true, indentationLevel + 1); } WriteBracketClose(writer, ObjectCloseBracket, true, indentationLevel); } From c99cf4bbfaf71f735cd0dea7d6fc0b2c1a589ed2 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Fri, 10 May 2024 08:48:01 -0700 Subject: [PATCH 155/179] Add `java` to reserved names to escape extensions to `java_`. This may break existing references to gencode for extensions named `java`. This prevents shadowing of `java.lang` package commonly used in protobuf gencode. Existing extensions named `java` may or may not previously fail to compile depending on if the contents of their .proto result in gencode using `java.lang`. This is needed to fix `java_features.proto` lite gencode since enum gencode uses `java.lang`. Fields named `java` should already be escaped. *Warning: This may break user code for existing protos with extensions named `java`. References to the extension should be renamed to use `java_` e.g. registry.add(GeneratedClassName.java_)* PiperOrigin-RevId: 632508249 --- .../golden/editions_transform_proto2.proto | 2 +- java/core/BUILD.bazel | 1 + .../java/com/google/protobuf/Descriptors.java | 8 +++---- .../com/google/protobuf/DescriptorsTest.java | 10 ++++----- .../com/google/protobuf/test_check_utf8.proto | 5 ++++- objectivec/Tests/unittest.proto | 4 ++-- src/google/protobuf/compiler/java/names.cc | 22 +++++++++---------- src/google/protobuf/edition_unittest.proto | 4 ++-- src/google/protobuf/unittest.proto | 4 ++-- 9 files changed, 32 insertions(+), 28 deletions(-) diff --git a/editions/golden/editions_transform_proto2.proto b/editions/golden/editions_transform_proto2.proto index 260a8d26ae..84807b6d59 100644 --- a/editions/golden/editions_transform_proto2.proto +++ b/editions/golden/editions_transform_proto2.proto @@ -15,7 +15,7 @@ edition = "2023"; package protobuf_editions_test; import "net/proto/proto1_features.proto"; -import "third_party/java/protobuf/java_features.proto"; +import "google/protobuf/java_features.proto"; import "google/protobuf/cpp_features.proto"; import "google/protobuf/editions/proto/editions_transform_proto3.proto"; diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 6127b4c5ca..41802133ed 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -336,6 +336,7 @@ proto_library( deps = [ "//:any_proto", "//:descriptor_proto", + "//:java_features_proto", "//:lite_test_protos", "//:wrappers_proto", "//src/google/protobuf:generic_test_protos", diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index f4aa9ca3ba..bba3178614 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -94,7 +94,7 @@ public final class Descriptors { if (javaEditionDefaults == null) { try { ExtensionRegistry registry = ExtensionRegistry.newInstance(); - registry.add(JavaFeaturesProto.java); + registry.add(JavaFeaturesProto.java_); setTestJavaEditionDefaults( FeatureSetDefaults.parseFrom( JavaEditionDefaults.PROTOBUF_INTERNAL_JAVA_EDITION_DEFAULTS.getBytes( @@ -679,7 +679,7 @@ public final class Descriptors { if (getEdition() == Edition.EDITION_PROTO2) { if (proto.getOptions().getJavaStringCheckUtf8()) { features.setExtension( - JavaFeaturesProto.java, + JavaFeaturesProto.java_, JavaFeatures.newBuilder() .setUtf8Validation(JavaFeatures.Utf8Validation.VERIFY) .build()); @@ -1320,7 +1320,7 @@ public final class Descriptors { return true; } if (this.features - .getExtension(JavaFeaturesProto.java) + .getExtension(JavaFeaturesProto.java_) .getUtf8Validation() .equals(JavaFeatures.Utf8Validation.VERIFY)) { return true; @@ -1577,7 +1577,7 @@ public final class Descriptors { } return getType() == Type.ENUM - && (this.features.getExtension(JavaFeaturesProto.java).getLegacyClosedEnum() + && (this.features.getExtension(JavaFeaturesProto.java_).getLegacyClosedEnum() || enumType.isClosed()); } diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index d64186ad7a..2ebdf71e75 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -1158,7 +1158,7 @@ public class DescriptorsTest { .setOptions(FileOptions.newBuilder().setJavaStringCheckUtf8(true)) .build(), new FileDescriptor[0]); - assertThat(file.features.getExtension(JavaFeaturesProto.java).getUtf8Validation()) + assertThat(file.features.getExtension(JavaFeaturesProto.java_).getUtf8Validation()) .isEqualTo(JavaFeaturesProto.JavaFeatures.Utf8Validation.VERIFY); } @@ -1178,8 +1178,8 @@ public class DescriptorsTest { assertThat(features.getJsonFormat()) .isEqualTo(DescriptorProtos.FeatureSet.JsonFormat.LEGACY_BEST_EFFORT); - assertThat(features.getExtension(JavaFeaturesProto.java).getLegacyClosedEnum()).isTrue(); - assertThat(features.getExtension(JavaFeaturesProto.java).getUtf8Validation()) + assertThat(features.getExtension(JavaFeaturesProto.java_).getLegacyClosedEnum()).isTrue(); + assertThat(features.getExtension(JavaFeaturesProto.java_).getUtf8Validation()) .isEqualTo(JavaFeaturesProto.JavaFeatures.Utf8Validation.DEFAULT); } @@ -1198,8 +1198,8 @@ public class DescriptorsTest { assertThat(features.getMessageEncoding()) .isEqualTo(DescriptorProtos.FeatureSet.MessageEncoding.LENGTH_PREFIXED); - assertThat(features.getExtension(JavaFeaturesProto.java).getLegacyClosedEnum()).isFalse(); - assertThat(features.getExtension(JavaFeaturesProto.java).getUtf8Validation()) + assertThat(features.getExtension(JavaFeaturesProto.java_).getLegacyClosedEnum()).isFalse(); + assertThat(features.getExtension(JavaFeaturesProto.java_).getUtf8Validation()) .isEqualTo(JavaFeaturesProto.JavaFeatures.Utf8Validation.DEFAULT); } diff --git a/java/core/src/test/proto/com/google/protobuf/test_check_utf8.proto b/java/core/src/test/proto/com/google/protobuf/test_check_utf8.proto index 3af9831d29..fff2805e78 100644 --- a/java/core/src/test/proto/com/google/protobuf/test_check_utf8.proto +++ b/java/core/src/test/proto/com/google/protobuf/test_check_utf8.proto @@ -12,7 +12,10 @@ edition = "2023"; package proto2_test_check_utf8; -option features.utf8_validation = VERIFY; +import "google/protobuf/java_features.proto"; + +option features.utf8_validation = NONE; +option features.(pb.java).utf8_validation = VERIFY; option java_outer_classname = "TestCheckUtf8"; message StringWrapper { diff --git a/objectivec/Tests/unittest.proto b/objectivec/Tests/unittest.proto index 2239fe6928..dc9546db76 100644 --- a/objectivec/Tests/unittest.proto +++ b/objectivec/Tests/unittest.proto @@ -1109,11 +1109,11 @@ message TestNestedGroupExtensionOuter { repeated group Layer2RepeatedGroup = 2 { extensions 3 // NOTE: extension metadata is not supported due to targets such as - // `//third_party/protobuf_legacy_opensource/src:shell_scripts_test`, + // `//google/protobuf_legacy_opensource/src:shell_scripts_test`, // eee https://screenshot.googleplex.com/Axz2QD8nxjdpyFF //[metadata = { // NOTE: can't write type there due to some clever build gen code at - // http://google3/net/proto2/internal/BUILD;l=1247;rcl=411090862 + // http://google3/google/protobuf/BUILD;l=1247;rcl=411090862 // type: "objc.protobuf.tests.TestNestedGroupExtensionInnerExtension", // name: "inner", // }] diff --git a/src/google/protobuf/compiler/java/names.cc b/src/google/protobuf/compiler/java/names.cc index 6df146353c..e1381c78ca 100644 --- a/src/google/protobuf/compiler/java/names.cc +++ b/src/google/protobuf/compiler/java/names.cc @@ -39,17 +39,17 @@ const char* DefaultPackage(Options options) { bool IsReservedName(absl::string_view name) { static const auto& kReservedNames = *new absl::flat_hash_set({ - "abstract", "assert", "boolean", "break", "byte", - "case", "catch", "char", "class", "const", - "continue", "default", "do", "double", "else", - "enum", "extends", "false", "final", "finally", - "float", "for", "goto", "if", "implements", - "import", "instanceof", "int", "interface", "long", - "native", "new", "null", "package", "private", - "protected", "public", "return", "short", "static", - "strictfp", "super", "switch", "synchronized", "this", - "throw", "throws", "transient", "true", "try", - "void", "volatile", "while", + "abstract", "assert", "boolean", "break", "byte", + "case", "catch", "char", "class", "const", + "continue", "default", "do", "double", "else", + "enum", "extends", "false", "final", "finally", + "float", "for", "goto", "if", "implements", + "import", "instanceof", "int", "interface", "java", + "long", "native", "new", "null", "package", + "private", "protected", "public", "return", "short", + "static", "strictfp", "super", "switch", "synchronized", + "this", "throw", "throws", "transient", "true", + "try", "void", "volatile", "while", }); return kReservedNames.contains(name); } diff --git a/src/google/protobuf/edition_unittest.proto b/src/google/protobuf/edition_unittest.proto index fadde47d29..81848b87e4 100644 --- a/src/google/protobuf/edition_unittest.proto +++ b/src/google/protobuf/edition_unittest.proto @@ -1254,11 +1254,11 @@ message TestNestedGroupExtensionOuter { message Layer2RepeatedGroup { extensions 3 // NOTE: extension metadata is not supported due to targets such as - // `//third_party/protobuf_legacy_opensource/src:shell_scripts_test`, + // `//google/protobuf_legacy_opensource/src:shell_scripts_test`, // eee https://screenshot.googleplex.com/Axz2QD8nxjdpyFF //[metadata = { // NOTE: can't write type there due to some clever build gen code at - // http://google3/net/proto2/internal/BUILD;l=1247;rcl=411090862 + // http://google3/google/protobuf/BUILD;l=1247;rcl=411090862 // type: "edition_unittest.TestNestedGroupExtensionInnerExtension", // name: "inner", // }] diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 3d9dc3a775..7c65fb594f 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -1226,11 +1226,11 @@ message TestNestedGroupExtensionOuter { repeated group Layer2RepeatedGroup = 2 { extensions 3 // NOTE: extension metadata is not supported due to targets such as - // `//third_party/protobuf_legacy_opensource/src:shell_scripts_test`, + // `//google/protobuf_legacy_opensource/src:shell_scripts_test`, // eee https://screenshot.googleplex.com/Axz2QD8nxjdpyFF //[metadata = { // NOTE: can't write type there due to some clever build gen code at - // http://google3/net/proto2/internal/BUILD;l=1247;rcl=411090862 + // http://google3/google/protobuf/BUILD;l=1247;rcl=411090862 // type: "protobuf_unittest.TestNestedGroupExtensionInnerExtension", // name: "inner", // }] From 24d42eacec4088319c9261e58fb1e7b04b5d0634 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 May 2024 10:24:50 -0700 Subject: [PATCH 156/179] Add a BUILD file under rust/accessors and rename helpers.h to default_value.h PiperOrigin-RevId: 632535021 --- src/google/protobuf/compiler/rust/BUILD.bazel | 65 ++++--------------- .../compiler/rust/accessors/BUILD.bazel | 45 +++++++++++++ .../{helpers.cc => default_value.cc} | 2 +- .../accessors/{helpers.h => default_value.h} | 0 .../rust/accessors/singular_scalar.cc | 2 +- 5 files changed, 58 insertions(+), 56 deletions(-) create mode 100644 src/google/protobuf/compiler/rust/accessors/BUILD.bazel rename src/google/protobuf/compiler/rust/accessors/{helpers.cc => default_value.cc} (98%) rename src/google/protobuf/compiler/rust/accessors/{helpers.h => default_value.h} (100%) diff --git a/src/google/protobuf/compiler/rust/BUILD.bazel b/src/google/protobuf/compiler/rust/BUILD.bazel index 6b6f98754c..5ff8f510ac 100644 --- a/src/google/protobuf/compiler/rust/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/BUILD.bazel @@ -82,7 +82,6 @@ cc_library( copts = COPTS, strip_include_prefix = "/src", deps = [ - ":accessors", ":context", ":enum", ":naming", @@ -90,6 +89,7 @@ cc_library( "//src/google/protobuf", "//src/google/protobuf/compiler/cpp:names", "//src/google/protobuf/compiler/cpp:names_internal", + "//src/google/protobuf/compiler/rust/accessors", "//upb_generator:mangle", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", @@ -98,46 +98,15 @@ cc_library( ], ) -cc_library( - name = "accessors", - srcs = [ - "accessors/accessor_case.cc", - "accessors/accessors.cc", - "accessors/map.cc", - "accessors/repeated_field.cc", - "accessors/singular_message.cc", - "accessors/singular_scalar.cc", - "accessors/singular_string.cc", - "accessors/unsupported_field.cc", - ], - hdrs = [ - "accessors/accessor_case.h", - "accessors/accessor_generator.h", - "accessors/accessors.h", - ], - copts = COPTS, - strip_include_prefix = "/src", - deps = [ - ":context", - ":helpers", - ":naming", - ":rust_field_type", - "//src/google/protobuf", - "//src/google/protobuf/compiler/cpp:names_internal", - "//src/google/protobuf/io:tokenizer", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], -) - cc_library( name = "context", srcs = ["context.cc"], hdrs = ["context.h"], copts = COPTS, strip_include_prefix = "/src", + visibility = [ + "//src/google/protobuf/compiler/rust:__subpackages__", + ], deps = [ "//src/google/protobuf", "//src/google/protobuf/compiler:code_generator", @@ -195,6 +164,9 @@ cc_library( ], copts = COPTS, strip_include_prefix = "/src", + visibility = [ + "//src/google/protobuf/compiler/rust:__subpackages__", + ], deps = [ ":context", ":rust_field_type", @@ -216,13 +188,13 @@ cc_library( copts = COPTS, strip_include_prefix = "/src", deps = [ - ":accessors", ":context", ":naming", ":rust_field_type", "//src/google/protobuf", "//src/google/protobuf/compiler/cpp:names", "//src/google/protobuf/compiler/cpp:names_internal", + "//src/google/protobuf/compiler/rust/accessors", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], @@ -250,29 +222,14 @@ cc_test( ], ) -cc_library( - name = "helpers", - srcs = ["accessors/helpers.cc"], - hdrs = ["accessors/helpers.h"], - strip_include_prefix = "/src", - deps = [ - ":context", - ":naming", - ":rust_field_type", - "//src/google/protobuf", - "//src/google/protobuf/io:tokenizer", - "@com_google_absl//absl/log:absl_check", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], -) - cc_library( name = "rust_field_type", srcs = ["rust_field_type.cc"], hdrs = ["rust_field_type.h"], strip_include_prefix = "/src", + visibility = [ + "//src/google/protobuf/compiler/rust:__subpackages__", + ], deps = [ "//src/google/protobuf", "@com_google_absl//absl/log:absl_log", diff --git a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel new file mode 100644 index 0000000000..d864d62587 --- /dev/null +++ b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel @@ -0,0 +1,45 @@ +################################################################################ +# Protocol Buffers Compiler - Generation of accessors for individual fields. +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "accessors", + srcs = [ + "accessor_case.cc", + "accessors.cc", + "default_value.cc", + "map.cc", + "repeated_field.cc", + "singular_message.cc", + "singular_scalar.cc", + "singular_string.cc", + "unsupported_field.cc", + ], + hdrs = [ + "accessor_case.h", + "accessor_generator.h", + "accessors.h", + "default_value.h", + ], + copts = COPTS, + strip_include_prefix = "/src", + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__subpackages__", + ], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/compiler/cpp:names_internal", + "//src/google/protobuf/compiler/rust:context", + "//src/google/protobuf/compiler/rust:naming", + "//src/google/protobuf/compiler/rust:rust_field_type", + "//src/google/protobuf/io:tokenizer", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + ], +) diff --git a/src/google/protobuf/compiler/rust/accessors/helpers.cc b/src/google/protobuf/compiler/rust/accessors/default_value.cc similarity index 98% rename from src/google/protobuf/compiler/rust/accessors/helpers.cc rename to src/google/protobuf/compiler/rust/accessors/default_value.cc index 71c6b280f6..27594b3c1f 100644 --- a/src/google/protobuf/compiler/rust/accessors/helpers.cc +++ b/src/google/protobuf/compiler/rust/accessors/default_value.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/compiler/rust/accessors/helpers.h" +#include "google/protobuf/compiler/rust/accessors/default_value.h" #include #include diff --git a/src/google/protobuf/compiler/rust/accessors/helpers.h b/src/google/protobuf/compiler/rust/accessors/default_value.h similarity index 100% rename from src/google/protobuf/compiler/rust/accessors/helpers.h rename to src/google/protobuf/compiler/rust/accessors/default_value.h diff --git a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc index 2c3546cbd9..9131b361a7 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc @@ -11,7 +11,7 @@ #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" #include "google/protobuf/compiler/rust/accessors/accessor_generator.h" -#include "google/protobuf/compiler/rust/accessors/helpers.h" +#include "google/protobuf/compiler/rust/accessors/default_value.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" From 39986c099b2b28da9f407bdb70e76f24bb07c97b Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 10 May 2024 17:38:46 +0000 Subject: [PATCH 157/179] Auto-generate files after cl/632535021 --- src/file_lists.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 318d0763e8..e22a26cf24 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -391,7 +391,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_case.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessors.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/helpers.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/default_value.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/map.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/repeated_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_message.cc @@ -525,7 +525,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_case.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessors.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/helpers.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/default_value.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/context.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/crate_mapping.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/enum.h From e7fdab4a2c822481c04d470593f0bf193f701649 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Fri, 10 May 2024 13:55:08 -0700 Subject: [PATCH 158/179] Add Nextgen python proto_test to github PiperOrigin-RevId: 632595087 --- python/build_targets.bzl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/build_targets.bzl b/python/build_targets.bzl index 0b18fe19b9..1014514a4b 100644 --- a/python/build_targets.bzl +++ b/python/build_targets.bzl @@ -416,6 +416,11 @@ def build_targets(name): srcs = ["google/protobuf/internal/wire_format_test.py"], ) + internal_py_test( + name = "proto_test", + srcs = ["google/protobuf/internal/proto_test.py"], + ) + native.cc_library( name = "proto_api", hdrs = ["google/protobuf/proto_api.h"], From be99767a69e3754c38852a19b611715dac28f85b Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 May 2024 15:35:38 -0700 Subject: [PATCH 159/179] Clear repository cache between bazel runs PiperOrigin-RevId: 632621361 --- .github/workflows/test_csharp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index f816f1f9bb..8b04ff4d91 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -36,7 +36,7 @@ jobs: dotnet test -c Release -f net6.0 src/Google.Protobuf.Test/Google.Protobuf.Test.csproj" - name: Clear bazel between docker instances - run: sudo rm -rf _build + run: sudo rm -rf _build .repository-cache - name: Run conformance tests uses: protocolbuffers/protobuf-ci/bazel-docker@v3 From f93565c20cefead85c782f3ad8c51dd0bfc482a3 Mon Sep 17 00:00:00 2001 From: Eric Salo Date: Fri, 10 May 2024 19:07:05 -0700 Subject: [PATCH 160/179] upb: publish and start using the new non-universal setters out in the wild PiperOrigin-RevId: 632663127 --- upb/message/accessors.h | 96 ++++++++++++++++++++++++++++---- upb/message/accessors_test.cc | 22 ++++---- upb/message/internal/accessors.h | 11 ++-- 3 files changed, 101 insertions(+), 28 deletions(-) diff --git a/upb/message/accessors.h b/upb/message/accessors.h index b0425dbc9c..9e9b54b127 100644 --- a/upb/message/accessors.h +++ b/upb/message/accessors.h @@ -51,14 +51,6 @@ UPB_API_INLINE bool upb_Message_HasBaseField(const upb_Message* msg, UPB_API_INLINE bool upb_Message_HasExtension(const upb_Message* msg, const upb_MiniTableExtension* e); -UPB_API_INLINE void upb_Message_SetBaseField(upb_Message* msg, - const upb_MiniTableField* f, - const void* val); - -UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, - const upb_MiniTableExtension* e, - const void* val, upb_Arena* a); - UPB_API_INLINE upb_MessageValue upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, upb_MessageValue default_val); @@ -128,14 +120,92 @@ UPB_API_INLINE uint64_t upb_Message_GetUInt64(const upb_Message* msg, const upb_MiniTableField* f, uint64_t default_val); -UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, - const upb_MiniTableField* f, bool value, - upb_Arena* a); - UPB_API_INLINE void upb_Message_SetClosedEnum( upb_Message* msg, const upb_MiniTable* msg_mini_table, const upb_MiniTableField* f, int32_t value); +// BaseField Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE void upb_Message_SetBaseField(upb_Message* msg, + const upb_MiniTableField* f, + const void* val); + +UPB_API_INLINE void upb_Message_SetBaseFieldBool(struct upb_Message* msg, + const upb_MiniTableField* f, + bool value); + +UPB_API_INLINE void upb_Message_SetBaseFieldDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value); + +UPB_API_INLINE void upb_Message_SetBaseFieldFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value); + +UPB_API_INLINE void upb_Message_SetBaseFieldInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value); + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value); + +// Extension Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, + const upb_MiniTableExtension* e, + const void* value, upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionBool( + struct upb_Message* msg, const upb_MiniTableExtension* e, bool value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionDouble( + struct upb_Message* msg, const upb_MiniTableExtension* e, double value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionFloat( + struct upb_Message* msg, const upb_MiniTableExtension* e, float value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, int32_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, int64_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionString( + struct upb_Message* msg, const upb_MiniTableExtension* e, + upb_StringView value, upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionUInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint32_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionUInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint64_t value, + upb_Arena* a); + +// Universal Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a); + UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, const upb_MiniTableField* f, double value, upb_Arena* a); @@ -168,6 +238,8 @@ UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a); +//////////////////////////////////////////////////////////////////////////////// + UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena); diff --git a/upb/message/accessors_test.cc b/upb/message/accessors_test.cc index d14daf2312..76248578b3 100644 --- a/upb/message/accessors_test.cc +++ b/upb/message/accessors_test.cc @@ -179,8 +179,8 @@ TEST(GeneratedCode, ScalarsProto2) { 0, protobuf_test_messages_proto2_TestAllTypesProto2_optional_int32(msg)); EXPECT_EQ(0, upb_Message_GetInt32(UPB_UPCAST(msg), optional_int32_field, 0)); - upb_Message_SetInt32(UPB_UPCAST(msg), optional_int32_field, kTestInt32, - nullptr); + upb_Message_SetBaseFieldInt32(UPB_UPCAST(msg), optional_int32_field, + kTestInt32); EXPECT_EQ(true, upb_Message_HasBaseField(UPB_UPCAST(msg), optional_int32_field)); EXPECT_EQ(kTestInt32, @@ -196,8 +196,8 @@ TEST(GeneratedCode, ScalarsProto2) { 0, protobuf_test_messages_proto2_TestAllTypesProto2_optional_uint32(msg)); EXPECT_EQ(0, upb_Message_GetUInt32(UPB_UPCAST(msg), optional_uint32_field, 0)); - upb_Message_SetUInt32(UPB_UPCAST(msg), optional_uint32_field, kTestUInt32, - nullptr); + upb_Message_SetBaseFieldUInt32(UPB_UPCAST(msg), optional_uint32_field, + kTestUInt32); EXPECT_EQ(kTestUInt32, upb_Message_GetUInt32(UPB_UPCAST(msg), optional_uint32_field, 0)); EXPECT_EQ( @@ -219,15 +219,15 @@ TEST(GeneratedCode, ScalarProto3) { EXPECT_EQ( 0, protobuf_test_messages_proto3_TestAllTypesProto3_optional_int64(msg)); - upb_Message_SetInt64(UPB_UPCAST(msg), optional_int64_field, -1, nullptr); + upb_Message_SetBaseFieldInt64(UPB_UPCAST(msg), optional_int64_field, -1); EXPECT_EQ( -1, protobuf_test_messages_proto3_TestAllTypesProto3_optional_int64(msg)); EXPECT_EQ(-1, upb_Message_GetInt64(UPB_UPCAST(msg), optional_int64_field, 0)); EXPECT_EQ( 0, protobuf_test_messages_proto3_TestAllTypesProto3_optional_uint64(msg)); - upb_Message_SetUInt64(UPB_UPCAST(msg), optional_uint64_field, kTestUInt64, - nullptr); + upb_Message_SetBaseFieldUInt64(UPB_UPCAST(msg), optional_uint64_field, + kTestUInt64); EXPECT_EQ( kTestUInt64, protobuf_test_messages_proto3_TestAllTypesProto3_optional_uint64(msg)); @@ -265,8 +265,8 @@ TEST(GeneratedCode, Strings) { false, protobuf_test_messages_proto2_TestAllTypesProto2_has_optional_string( msg)); - upb_Message_SetString(UPB_UPCAST(msg), optional_string_field, - upb_StringView_FromString(kTestStr2), nullptr); + upb_Message_SetBaseFieldString(UPB_UPCAST(msg), optional_string_field, + upb_StringView_FromString(kTestStr2)); EXPECT_EQ(true, upb_Message_HasBaseField(UPB_UPCAST(msg), optional_string_field)); EXPECT_EQ( @@ -327,8 +327,8 @@ TEST(GeneratedCode, SubMessage) { upb_Message* new_nested_message = UPB_UPCAST( protobuf_test_messages_proto2_TestAllTypesProto2_NestedMessage_new( arena)); - upb_Message_SetInt32(new_nested_message, nested_message_a_field, 123, - nullptr); + upb_Message_SetBaseFieldInt32(new_nested_message, nested_message_a_field, + 123); upb_Message_SetMessage(UPB_UPCAST(msg), optional_message_field, new_nested_message); diff --git a/upb/message/internal/accessors.h b/upb/message/internal/accessors.h index 31d3ff8dd3..6e5e43ec62 100644 --- a/upb/message/internal/accessors.h +++ b/upb/message/internal/accessors.h @@ -657,13 +657,14 @@ UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, upb_Message_SetBaseField(msg, f, &value); } -UPB_API_INLINE void upb_Message_SetClosedEnum( - struct upb_Message* msg, const upb_MiniTable* msg_mini_table, - const upb_MiniTableField* f, int32_t value) { +UPB_API_INLINE void upb_Message_SetClosedEnum(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSERT(upb_MiniTableEnum_CheckValue( - upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); + UPB_ASSERT( + upb_MiniTableEnum_CheckValue(upb_MiniTable_GetSubEnumTable(m, f), value)); upb_Message_SetBaseField(msg, f, &value); } From d2aa5fb483c53c18c00501a0e960ed4d14a90e44 Mon Sep 17 00:00:00 2001 From: MiguelBarro <45819833+MiguelBarro@users.noreply.github.com> Date: Fri, 10 May 2024 19:15:40 -0700 Subject: [PATCH 161/179] Prevent find_package failure if multiple versions are installed (#16654) The generated `protobuf-config-version.cmake` sets the `Protobuf_WITH_ZLIB`, `Protobuf_MSVC_STATIC_RUNTIME` and `Protobuf_BUILD_SHARED_LIBS` unconditionally while executing the `_check_and_save_build_option(...)` macro. If there are several **Protobuf** installations available the original code may lead to failure. An example with two installations: + A system deployed installation with version `3.12.4`. + A repo's thirdparty with version `3.19.4` The user wants it's project to use the thirdparty one by doing: ```cmake find_package(Protobuf 3.19.4 CONFIG REQUIRED PATHS path/to/thirdparty) ``` The user is not setting the `Protobuf_XXX` variables in its `CMakeLitsts.txt`. If the first config processed is the system deployed it sets the variables to its desired variables, for example, `Protobuf_WITH_ZLIB=OFF` but is discarded due to the non-matched version. Then the thirdparty's config is processed but is discarded because requires `Protobuf_WITH_ZLIB=ON` which is not a real requirement and makes the find operation fail (despite of matching version). Closes #16654 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/16654 from MiguelBarro:fix/cmake-config 119dd25ecb09946afbdb33519b22d2454e196251 PiperOrigin-RevId: 632664341 --- cmake/protobuf-config-version.cmake.in | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/protobuf-config-version.cmake.in b/cmake/protobuf-config-version.cmake.in index 3fa01763ee..269236afdc 100644 --- a/cmake/protobuf-config-version.cmake.in +++ b/cmake/protobuf-config-version.cmake.in @@ -44,9 +44,12 @@ macro(_check_and_save_build_option OPTION VALUE) endif() set(${PACKAGE_FIND_NAME}_${OPTION} ${VALUE} PARENT_SCOPE) endmacro() -_check_and_save_build_option(WITH_ZLIB @protobuf_WITH_ZLIB@) -_check_and_save_build_option(MSVC_STATIC_RUNTIME @protobuf_MSVC_STATIC_RUNTIME@) -_check_and_save_build_option(BUILD_SHARED_LIBS @protobuf_BUILD_SHARED_LIBS@) + +if(PACKAGE_VERSION_COMPATIBLE) + _check_and_save_build_option(WITH_ZLIB @protobuf_WITH_ZLIB@) + _check_and_save_build_option(MSVC_STATIC_RUNTIME @protobuf_MSVC_STATIC_RUNTIME@) + _check_and_save_build_option(BUILD_SHARED_LIBS @protobuf_BUILD_SHARED_LIBS@) +endif() # if the installed or the using project don't have CMAKE_SIZEOF_VOID_P set, ignore it: if(CMAKE_SIZEOF_VOID_P AND "@CMAKE_SIZEOF_VOID_P@") From ddc70b0ebfedca40f5ff8f287074ce9a8db27c72 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Sat, 11 May 2024 02:21:06 +0000 Subject: [PATCH 162/179] Auto-generate files after cl/632663127 --- php/ext/google/protobuf/php-upb.h | 107 ++++++++++++++++++++++---- ruby/ext/google/protobuf_c/ruby-upb.h | 107 ++++++++++++++++++++++---- 2 files changed, 180 insertions(+), 34 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index 0ae74fb214..26cf6d7616 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -3238,13 +3238,14 @@ UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, upb_Message_SetBaseField(msg, f, &value); } -UPB_API_INLINE void upb_Message_SetClosedEnum( - struct upb_Message* msg, const upb_MiniTable* msg_mini_table, - const upb_MiniTableField* f, int32_t value) { +UPB_API_INLINE void upb_Message_SetClosedEnum(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSERT(upb_MiniTableEnum_CheckValue( - upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); + UPB_ASSERT( + upb_MiniTableEnum_CheckValue(upb_MiniTable_GetSubEnumTable(m, f), value)); upb_Message_SetBaseField(msg, f, &value); } @@ -3717,14 +3718,6 @@ UPB_API_INLINE bool upb_Message_HasBaseField(const upb_Message* msg, UPB_API_INLINE bool upb_Message_HasExtension(const upb_Message* msg, const upb_MiniTableExtension* e); -UPB_API_INLINE void upb_Message_SetBaseField(upb_Message* msg, - const upb_MiniTableField* f, - const void* val); - -UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, - const upb_MiniTableExtension* e, - const void* val, upb_Arena* a); - UPB_API_INLINE upb_MessageValue upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, upb_MessageValue default_val); @@ -3794,14 +3787,92 @@ UPB_API_INLINE uint64_t upb_Message_GetUInt64(const upb_Message* msg, const upb_MiniTableField* f, uint64_t default_val); -UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, - const upb_MiniTableField* f, bool value, - upb_Arena* a); - UPB_API_INLINE void upb_Message_SetClosedEnum( upb_Message* msg, const upb_MiniTable* msg_mini_table, const upb_MiniTableField* f, int32_t value); +// BaseField Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE void upb_Message_SetBaseField(upb_Message* msg, + const upb_MiniTableField* f, + const void* val); + +UPB_API_INLINE void upb_Message_SetBaseFieldBool(struct upb_Message* msg, + const upb_MiniTableField* f, + bool value); + +UPB_API_INLINE void upb_Message_SetBaseFieldDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value); + +UPB_API_INLINE void upb_Message_SetBaseFieldFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value); + +UPB_API_INLINE void upb_Message_SetBaseFieldInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value); + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value); + +// Extension Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, + const upb_MiniTableExtension* e, + const void* value, upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionBool( + struct upb_Message* msg, const upb_MiniTableExtension* e, bool value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionDouble( + struct upb_Message* msg, const upb_MiniTableExtension* e, double value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionFloat( + struct upb_Message* msg, const upb_MiniTableExtension* e, float value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, int32_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, int64_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionString( + struct upb_Message* msg, const upb_MiniTableExtension* e, + upb_StringView value, upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionUInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint32_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionUInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint64_t value, + upb_Arena* a); + +// Universal Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a); + UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, const upb_MiniTableField* f, double value, upb_Arena* a); @@ -3834,6 +3905,8 @@ UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a); +//////////////////////////////////////////////////////////////////////////////// + UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena); diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 9d6cdcffe6..3d9024c6a9 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -3240,13 +3240,14 @@ UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, upb_Message_SetBaseField(msg, f, &value); } -UPB_API_INLINE void upb_Message_SetClosedEnum( - struct upb_Message* msg, const upb_MiniTable* msg_mini_table, - const upb_MiniTableField* f, int32_t value) { +UPB_API_INLINE void upb_Message_SetClosedEnum(struct upb_Message* msg, + const upb_MiniTable* m, + const upb_MiniTableField* f, + int32_t value) { UPB_ASSERT(upb_MiniTableField_IsClosedEnum(f)); UPB_ASSUME(UPB_PRIVATE(_upb_MiniTableField_GetRep)(f) == kUpb_FieldRep_4Byte); - UPB_ASSERT(upb_MiniTableEnum_CheckValue( - upb_MiniTable_GetSubEnumTable(msg_mini_table, f), value)); + UPB_ASSERT( + upb_MiniTableEnum_CheckValue(upb_MiniTable_GetSubEnumTable(m, f), value)); upb_Message_SetBaseField(msg, f, &value); } @@ -3719,14 +3720,6 @@ UPB_API_INLINE bool upb_Message_HasBaseField(const upb_Message* msg, UPB_API_INLINE bool upb_Message_HasExtension(const upb_Message* msg, const upb_MiniTableExtension* e); -UPB_API_INLINE void upb_Message_SetBaseField(upb_Message* msg, - const upb_MiniTableField* f, - const void* val); - -UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, - const upb_MiniTableExtension* e, - const void* val, upb_Arena* a); - UPB_API_INLINE upb_MessageValue upb_Message_GetField(const upb_Message* msg, const upb_MiniTableField* f, upb_MessageValue default_val); @@ -3796,14 +3789,92 @@ UPB_API_INLINE uint64_t upb_Message_GetUInt64(const upb_Message* msg, const upb_MiniTableField* f, uint64_t default_val); -UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, - const upb_MiniTableField* f, bool value, - upb_Arena* a); - UPB_API_INLINE void upb_Message_SetClosedEnum( upb_Message* msg, const upb_MiniTable* msg_mini_table, const upb_MiniTableField* f, int32_t value); +// BaseField Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE void upb_Message_SetBaseField(upb_Message* msg, + const upb_MiniTableField* f, + const void* val); + +UPB_API_INLINE void upb_Message_SetBaseFieldBool(struct upb_Message* msg, + const upb_MiniTableField* f, + bool value); + +UPB_API_INLINE void upb_Message_SetBaseFieldDouble(struct upb_Message* msg, + const upb_MiniTableField* f, + double value); + +UPB_API_INLINE void upb_Message_SetBaseFieldFloat(struct upb_Message* msg, + const upb_MiniTableField* f, + float value); + +UPB_API_INLINE void upb_Message_SetBaseFieldInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + int32_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + int64_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldString(struct upb_Message* msg, + const upb_MiniTableField* f, + upb_StringView value); + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt32(struct upb_Message* msg, + const upb_MiniTableField* f, + uint32_t value); + +UPB_API_INLINE void upb_Message_SetBaseFieldUInt64(struct upb_Message* msg, + const upb_MiniTableField* f, + uint64_t value); + +// Extension Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetExtension(upb_Message* msg, + const upb_MiniTableExtension* e, + const void* value, upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionBool( + struct upb_Message* msg, const upb_MiniTableExtension* e, bool value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionDouble( + struct upb_Message* msg, const upb_MiniTableExtension* e, double value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionFloat( + struct upb_Message* msg, const upb_MiniTableExtension* e, float value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, int32_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, int64_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionString( + struct upb_Message* msg, const upb_MiniTableExtension* e, + upb_StringView value, upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionUInt32( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint32_t value, + upb_Arena* a); + +UPB_API_INLINE bool upb_Message_SetExtensionUInt64( + struct upb_Message* msg, const upb_MiniTableExtension* e, uint64_t value, + upb_Arena* a); + +// Universal Setters /////////////////////////////////////////////////////////// + +UPB_API_INLINE bool upb_Message_SetBool(upb_Message* msg, + const upb_MiniTableField* f, bool value, + upb_Arena* a); + UPB_API_INLINE bool upb_Message_SetDouble(upb_Message* msg, const upb_MiniTableField* f, double value, upb_Arena* a); @@ -3836,6 +3907,8 @@ UPB_API_INLINE bool upb_Message_SetUInt64(upb_Message* msg, const upb_MiniTableField* f, uint64_t value, upb_Arena* a); +//////////////////////////////////////////////////////////////////////////////// + UPB_API_INLINE void* upb_Message_ResizeArrayUninitialized( upb_Message* msg, const upb_MiniTableField* f, size_t size, upb_Arena* arena); From f7a866a9ead95176fa5022fe3d00ff2a69b8b993 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Sun, 12 May 2024 17:08:19 -0700 Subject: [PATCH 163/179] Make IntArrayList consistent with other primitive array lists, using code generator PiperOrigin-RevId: 633042614 --- java/core/src/main/java/com/google/protobuf/IntArrayList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/core/src/main/java/com/google/protobuf/IntArrayList.java b/java/core/src/main/java/com/google/protobuf/IntArrayList.java index eea507ae8f..e6a1aca1ec 100644 --- a/java/core/src/main/java/com/google/protobuf/IntArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/IntArrayList.java @@ -47,7 +47,7 @@ final class IntArrayList extends AbstractProtobufList */ private IntArrayList(int[] other, int size, boolean isMutable) { super(isMutable); - array = other; + this.array = other; this.size = size; } From f2d8c2bdd86f34b953c7bae9075daa7648025d3c Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Mon, 13 May 2024 05:48:10 -0700 Subject: [PATCH 164/179] Add a reserve method on ProxiedInRepeated PiperOrigin-RevId: 633179853 --- rust/cpp.rs | 21 ++++++++++++- rust/cpp_kernel/cpp_api.cc | 8 +++++ rust/repeated.rs | 6 ++++ rust/upb.rs | 21 +++++++++++++ rust/upb/array.rs | 1 + rust/upb/lib.rs | 4 +-- src/google/protobuf/compiler/rust/enum.cc | 9 ++++++ src/google/protobuf/compiler/rust/message.cc | 32 ++++++++++++++++++++ 8 files changed, 99 insertions(+), 3 deletions(-) diff --git a/rust/cpp.rs b/rust/cpp.rs index 1bd15266b8..87b482585f 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -374,7 +374,8 @@ macro_rules! impl_repeated_primitives { $get_thunk:ident, $set_thunk:ident, $clear_thunk:ident, - $copy_from_thunk:ident $(,)? + $copy_from_thunk:ident, + $reserve_thunk:ident $(,)? ]),* $(,)?) => { $( extern "C" { @@ -391,6 +392,9 @@ macro_rules! impl_repeated_primitives { v: <$t as CppTypeConversions>::ElemType); fn $clear_thunk(f: RawRepeatedField); fn $copy_from_thunk(src: RawRepeatedField, dst: RawRepeatedField); + fn $reserve_thunk( + f: RawRepeatedField, + additional: usize); } unsafe impl ProxiedInRepeated for $t { @@ -423,6 +427,9 @@ macro_rules! impl_repeated_primitives { fn repeated_copy_from(src: View>, mut dest: Mut>) { unsafe { $copy_from_thunk(src.as_raw(Private), dest.as_raw(Private)) } } + fn repeated_reserve(mut f: Mut>, additional: usize) { + unsafe { $reserve_thunk(f.as_raw(Private), additional) } + } } )* }; @@ -438,6 +445,7 @@ macro_rules! impl_repeated_primitives { [< __pb_rust_RepeatedField_ $t _set >], [< __pb_rust_RepeatedField_ $t _clear >], [< __pb_rust_RepeatedField_ $t _copy_from >], + [< __pb_rust_RepeatedField_ $t _reserve >], ], )*); } @@ -474,6 +482,17 @@ pub fn cast_enum_repeated_mut( } } +/// Cast a `RepeatedMut` to `RepeatedMut` and call +/// repeated_reserve. +pub fn reserve_enum_repeated_mut( + private: Private, + repeated: RepeatedMut, + additional: usize, +) { + let int_repeated = cast_enum_repeated_mut(private, repeated); + ProxiedInRepeated::repeated_reserve(int_repeated, additional); +} + #[derive(Debug)] pub struct InnerMap { pub(crate) raw: RawMap, diff --git a/rust/cpp_kernel/cpp_api.cc b/rust/cpp_kernel/cpp_api.cc index f37a1c7e5a..753331c287 100644 --- a/rust/cpp_kernel/cpp_api.cc +++ b/rust/cpp_kernel/cpp_api.cc @@ -42,6 +42,10 @@ extern "C" { void __pb_rust_RepeatedField_##rust_ty##_clear( \ google::protobuf::RepeatedField* r) { \ r->Clear(); \ + } \ + void __pb_rust_RepeatedField_##rust_ty##_reserve( \ + google::protobuf::RepeatedField* r, size_t additional) { \ + r->Reserve(r->size() + additional); \ } expose_repeated_field_methods(int32_t, i32); @@ -89,6 +93,10 @@ expose_repeated_field_methods(int64_t, i64); void __pb_rust_RepeatedField_##ty##_clear( \ google::protobuf::RepeatedPtrField* r) { \ r->Clear(); \ + } \ + void __pb_rust_RepeatedField_##ty##_reserve( \ + google::protobuf::RepeatedPtrField* r, size_t additional) { \ + r->Reserve(r->size() + additional); \ } expose_repeated_ptr_field_methods(ProtoStr); diff --git a/rust/repeated.rs b/rust/repeated.rs index 3183e42117..c7c9762488 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -296,6 +296,10 @@ pub unsafe trait ProxiedInRepeated: Proxied { /// Copies the values in the `src` repeated field into `dest`. fn repeated_copy_from(src: View>, dest: Mut>); + + /// Ensures that the repeated field has enough space allocated to insert at + /// least `additional` values without an allocation. + fn repeated_reserve(repeated: Mut>, additional: usize); } /// An iterator over the values inside of a [`View>`](RepeatedView). @@ -489,6 +493,8 @@ where ViewT: Into>, { fn extend>(&mut self, iter: I) { + let iter = iter.into_iter(); + T::repeated_reserve(self.as_mut(), iter.size_hint().0); for item in iter { self.push(item.into()); } diff --git a/rust/upb.rs b/rust/upb.rs index 895f47b417..858e7cf738 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -226,6 +226,16 @@ macro_rules! impl_repeated_base { ) } } + fn repeated_reserve(mut f: Mut>, additional: usize) { + // SAFETY: + // - `upb_Array_Reserve` is unsafe but assumed to be sound when called on a + // valid array. + unsafe { + let arena = f.raw_arena(Private); + let size = upb_Array_Size(f.as_raw(Private)); + assert!(upb_Array_Reserve(f.as_raw(Private), size + additional, arena)); + } + } }; } @@ -373,6 +383,17 @@ pub fn cast_enum_repeated_mut( } } +/// Cast a `RepeatedMut` to `RepeatedMut` and call +/// repeated_reserve. +pub fn reserve_enum_repeated_mut( + private: Private, + repeated: RepeatedMut, + additional: usize, +) { + let int_repeated = cast_enum_repeated_mut(private, repeated); + ProxiedInRepeated::repeated_reserve(int_repeated, additional); +} + /// Returns a static empty RepeatedView. pub fn empty_array() -> RepeatedView<'static, T> { // TODO: Consider creating a static empty array in C. diff --git a/rust/upb/array.rs b/rust/upb/array.rs index 8c0abf8c81..a8b2fb1ef7 100644 --- a/rust/upb/array.rs +++ b/rust/upb/array.rs @@ -12,6 +12,7 @@ extern "C" { pub fn upb_Array_Get(arr: RawArray, i: usize) -> upb_MessageValue; pub fn upb_Array_Append(arr: RawArray, val: upb_MessageValue, arena: RawArena) -> bool; pub fn upb_Array_Resize(arr: RawArray, size: usize, arena: RawArena) -> bool; + pub fn upb_Array_Reserve(arr: RawArray, size: usize, arena: RawArena) -> bool; pub fn upb_Array_MutableDataPtr(arr: RawArray) -> *mut std::ffi::c_void; pub fn upb_Array_DataPtr(arr: RawArray) -> *const std::ffi::c_void; pub fn upb_Array_GetMutable(arr: RawArray, i: usize) -> upb_MutableMessageValue; diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 299044f226..643018afe0 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -4,8 +4,8 @@ pub use arena::{upb_Arena, Arena, RawArena}; mod array; pub use array::{ upb_Array, upb_Array_Append, upb_Array_DataPtr, upb_Array_Get, upb_Array_GetMutable, - upb_Array_MutableDataPtr, upb_Array_New, upb_Array_Resize, upb_Array_Set, upb_Array_Size, - RawArray, + upb_Array_MutableDataPtr, upb_Array_New, upb_Array_Reserve, upb_Array_Resize, upb_Array_Set, + upb_Array_Size, RawArray, }; mod ctype; diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 857d283538..0ed50e98f0 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -450,6 +450,15 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { $pbr$::cast_enum_repeated_mut($pbi$::Private, dest) .copy_from($pbr$::cast_enum_repeated_view($pbi$::Private, src)) } + + fn repeated_reserve( + mut r: $pb$::Mut<$pb$::Repeated>, + additional: usize, + ) { + // SAFETY: + // - `f.as_raw()` is valid. + $pbr$::reserve_enum_repeated_mut($pbi$::Private, r, additional); + } } // SAFETY: this is an enum type diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 2b2babd172..f6e472886a 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -180,6 +180,8 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, {"repeated_copy_from_thunk", ThunkName(ctx, msg, "repeated_copy_from")}, + {"repeated_reserve_thunk", + ThunkName(ctx, msg, "repeated_reserve")}, }, R"rs( fn $new_thunk$() -> $pbr$::RawMessage; @@ -193,6 +195,7 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { fn $repeated_get_mut_thunk$(raw: $pbr$::RawRepeatedField, index: usize) -> $pbr$::RawMessage; fn $repeated_clear_thunk$(raw: $pbr$::RawRepeatedField); fn $repeated_copy_from_thunk$(dst: $pbr$::RawRepeatedField, src: $pbr$::RawRepeatedField); + fn $repeated_reserve_thunk$(raw: $pbr$::RawRepeatedField, additional: usize); )rs"); return; @@ -310,6 +313,8 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, {"repeated_copy_from_thunk", ThunkName(ctx, msg, "repeated_copy_from")}, + {"repeated_reserve_thunk", + ThunkName(ctx, msg, "repeated_reserve")}, }, R"rs( unsafe impl $pb$::ProxiedInRepeated for $Msg$ { @@ -372,6 +377,15 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { $repeated_copy_from_thunk$(dest.as_raw($pbi$::Private), src.as_raw($pbi$::Private)); } } + + fn repeated_reserve( + mut f: $pb$::Mut<$pb$::Repeated>, + additional: usize, + ) { + // SAFETY: + // - `f.as_raw()` is a valid `RepeatedPtrField*`. + unsafe { $repeated_reserve_thunk$(f.as_raw($pbi$::Private), additional) } + } } )rs"); return; @@ -465,6 +479,18 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { $pbr$::repeated_message_copy_from(src, dest, $std$::ptr::addr_of!($minitable$)); } } + + fn repeated_reserve( + mut f: $pb$::Mut<$pb$::Repeated>, + additional: usize, + ) { + // SAFETY: + // - `f.as_raw()` is a valid `upb_Array*`. + unsafe { + let size = $pbr$::upb_Array_Size(f.as_raw($pbi$::Private)); + $pbr$::upb_Array_Reserve(f.as_raw($pbi$::Private), size + additional, f.raw_arena($pbi$::Private)); + } + } } )rs"); return; @@ -1152,6 +1178,7 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { {"repeated_add_thunk", ThunkName(ctx, msg, "repeated_add")}, {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, {"repeated_copy_from_thunk", ThunkName(ctx, msg, "repeated_copy_from")}, + {"repeated_reserve_thunk", ThunkName(ctx, msg, "repeated_reserve")}, {"nested_msg_thunks", [&] { for (int i = 0; i < msg.nested_type_count(); ++i) { @@ -1217,6 +1244,11 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { const google::protobuf::RepeatedPtrField<$QualifiedMsg$>& src) { dst = src; } + void $repeated_reserve_thunk$( + google::protobuf::RepeatedPtrField<$QualifiedMsg$>* field, + size_t additional) { + field->Reserve(field->size() + additional); + } $accessor_thunks$ From 904266db57c146c165f4907f4ecf4c6f9a2cf696 Mon Sep 17 00:00:00 2001 From: Derek Benson Date: Mon, 13 May 2024 07:16:57 -0700 Subject: [PATCH 165/179] inline repeated field methods PiperOrigin-RevId: 633199701 --- rust/cpp.rs | 9 +++++++++ rust/repeated.rs | 22 ++++++++++++++++++++++ rust/upb.rs | 8 ++++++++ 3 files changed, 39 insertions(+) diff --git a/rust/cpp.rs b/rust/cpp.rs index 87b482585f..2a199dcc61 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -399,34 +399,43 @@ macro_rules! impl_repeated_primitives { unsafe impl ProxiedInRepeated for $t { #[allow(dead_code)] + #[inline] fn repeated_new(_: Private) -> Repeated<$t> { Repeated::from_inner(InnerRepeated { raw: unsafe { $new_thunk() } }) } #[allow(dead_code)] + #[inline] unsafe fn repeated_free(_: Private, f: &mut Repeated<$t>) { unsafe { $free_thunk(f.as_mut().as_raw(Private)) } } + #[inline] fn repeated_len(f: View>) -> usize { unsafe { $size_thunk(f.as_raw(Private)) } } + #[inline] fn repeated_push(mut f: Mut>, v: View<$t>) { unsafe { $add_thunk(f.as_raw(Private), v.into()) } } + #[inline] fn repeated_clear(mut f: Mut>) { unsafe { $clear_thunk(f.as_raw(Private)) } } + #[inline] unsafe fn repeated_get_unchecked(f: View>, i: usize) -> View<$t> { <$t as CppTypeConversions>::elem_to_view( unsafe { $get_thunk(f.as_raw(Private), i) }) } + #[inline] unsafe fn repeated_set_unchecked(mut f: Mut>, i: usize, v: View<$t>) { unsafe { $set_thunk(f.as_raw(Private), i, v.into()) } } + #[inline] fn repeated_copy_from(src: View>, mut dest: Mut>) { unsafe { $copy_from_thunk(src.as_raw(Private), dest.as_raw(Private)) } } + #[inline] fn repeated_reserve(mut f: Mut>, additional: usize) { unsafe { $reserve_thunk(f.as_raw(Private), additional) } } diff --git a/rust/repeated.rs b/rust/repeated.rs index c7c9762488..679e279e6b 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -61,6 +61,7 @@ impl<'msg, T: ?Sized> Debug for RepeatedMut<'msg, T> { #[doc(hidden)] impl<'msg, T: ?Sized> RepeatedView<'msg, T> { #[doc(hidden)] + #[inline] pub fn as_raw(&self, _private: Private) -> RawRepeatedField { self.raw } @@ -68,6 +69,7 @@ impl<'msg, T: ?Sized> RepeatedView<'msg, T> { /// # Safety /// - `inner` must be valid to read from for `'msg` #[doc(hidden)] + #[inline] pub unsafe fn from_raw(_private: Private, raw: RawRepeatedField) -> Self { Self { raw, _phantom: PhantomData } } @@ -78,11 +80,13 @@ where T: ProxiedInRepeated + ?Sized + 'msg, { /// Gets the length of the repeated field. + #[inline] pub fn len(&self) -> usize { T::repeated_len(*self) } /// Returns true if the repeated field has no values. + #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } @@ -90,6 +94,7 @@ where /// Gets the value at `index`. /// /// Returns `None` if `index > len`. + #[inline] pub fn get(self, index: usize) -> Option> { if index >= self.len() { return None; @@ -102,6 +107,7 @@ where /// /// # Safety /// Undefined behavior if `index >= len` + #[inline] pub unsafe fn get_unchecked(self, index: usize) -> View<'msg, T> { // SAFETY: in-bounds as promised unsafe { T::repeated_get_unchecked(self, index) } @@ -120,11 +126,13 @@ impl<'msg, T: ?Sized> RepeatedMut<'msg, T> { /// - There must be no aliasing references or mutations on the same /// underlying object. #[doc(hidden)] + #[inline] pub unsafe fn from_inner(_private: Private, inner: InnerRepeatedMut<'msg>) -> Self { Self { inner, _phantom: PhantomData } } #[doc(hidden)] + #[inline] pub fn as_raw(&mut self, _private: Private) -> RawRepeatedField { self.inner.raw } @@ -135,11 +143,13 @@ where T: ProxiedInRepeated + ?Sized + 'msg, { /// Gets the length of the repeated field. + #[inline] pub fn len(&self) -> usize { self.as_view().len() } /// Returns true if the repeated field has no values. + #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } @@ -147,6 +157,7 @@ where /// Gets the value at `index`. /// /// Returns `None` if `index > len`. + #[inline] pub fn get(&self, index: usize) -> Option> { self.as_view().get(index) } @@ -155,12 +166,14 @@ where /// /// # Safety /// Undefined behavior if `index >= len` + #[inline] pub unsafe fn get_unchecked(&self, index: usize) -> View { // SAFETY: in-bounds as promised unsafe { self.as_view().get_unchecked(index) } } /// Appends `val` to the end of the repeated field. + #[inline] pub fn push(&mut self, val: View) { // TODO: b/320936046 - Use SettableValue instead of View for added ergonomics. T::repeated_push(self.as_mut(), val); @@ -170,6 +183,7 @@ where /// /// # Panics /// Panics if `index >= len` + #[inline] pub fn set(&mut self, index: usize, val: View) { let len = self.len(); if index >= len { @@ -184,6 +198,7 @@ where /// /// # Safety /// Undefined behavior if `index >= len` + #[inline] pub unsafe fn set_unchecked(&mut self, index: usize, val: View) { // TODO: b/320936046 - Use SettableValue instead of View for added ergonomics. // SAFETY: `index` is in-bounds as promised by the caller. @@ -376,10 +391,12 @@ where { type Proxied = Repeated; + #[inline] fn as_view(&self) -> View<'_, Self::Proxied> { *self } + #[inline] fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> where 'msg: 'shorter, @@ -394,10 +411,12 @@ where { type Proxied = Repeated; + #[inline] fn as_view(&self) -> View<'_, Self::Proxied> { RepeatedView { raw: self.inner.raw, _phantom: PhantomData } } + #[inline] fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> where 'msg: 'shorter, @@ -410,10 +429,12 @@ impl<'msg, T> MutProxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + ?Sized + 'msg, { + #[inline] fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { RepeatedMut { inner: self.inner, _phantom: PhantomData } } + #[inline] fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> where 'msg: 'shorter, @@ -428,6 +449,7 @@ where { type Item = View<'msg, T>; + #[inline] fn next(&mut self) -> Option { let val = self.view.get(self.current_index); if val.is_some() { diff --git a/rust/upb.rs b/rust/upb.rs index 858e7cf738..554af66bc3 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -182,6 +182,7 @@ impl<'msg> InnerRepeatedMut<'msg> { macro_rules! impl_repeated_base { ($t:ty, $elem_t:ty, $ufield:ident, $upb_tag:expr) => { #[allow(dead_code)] + #[inline] fn repeated_new(_: Private) -> Repeated<$t> { let arena = Arena::new(); Repeated::from_inner(InnerRepeated { @@ -193,9 +194,11 @@ macro_rules! impl_repeated_base { unsafe fn repeated_free(_: Private, _f: &mut Repeated<$t>) { // No-op: the memory will be dropped by the arena. } + #[inline] fn repeated_len(f: View>) -> usize { unsafe { upb_Array_Size(f.as_raw(Private)) } } + #[inline] fn repeated_push(mut f: Mut>, v: View<$t>) { let arena = f.raw_arena(Private); unsafe { @@ -206,16 +209,19 @@ macro_rules! impl_repeated_base { )); } } + #[inline] fn repeated_clear(mut f: Mut>) { unsafe { upb_Array_Resize(f.as_raw(Private), 0, f.raw_arena(Private)); } } + #[inline] unsafe fn repeated_get_unchecked(f: View>, i: usize) -> View<$t> { unsafe { <$t as UpbTypeConversions>::from_message_value(upb_Array_Get(f.as_raw(Private), i)) } } + #[inline] unsafe fn repeated_set_unchecked(mut f: Mut>, i: usize, v: View<$t>) { let arena = f.raw_arena(Private); unsafe { @@ -226,6 +232,7 @@ macro_rules! impl_repeated_base { ) } } + #[inline] fn repeated_reserve(mut f: Mut>, additional: usize) { // SAFETY: // - `upb_Array_Reserve` is unsafe but assumed to be sound when called on a @@ -272,6 +279,7 @@ macro_rules! impl_repeated_bytes { unsafe impl ProxiedInRepeated for $t { impl_repeated_base!($t, PtrAndLen, str_val, $upb_tag); + #[inline] fn repeated_copy_from(src: View>, mut dest: Mut>) { let len = src.len(); // SAFETY: From ce026abf92d0862c645c449fa91c2138058d3e21 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Mon, 13 May 2024 08:46:41 -0700 Subject: [PATCH 166/179] Work around windows path length limitations by moving immutable -> full in open source (new generator paths) and shortening long file names. Re-enable windows bazel test. PiperOrigin-RevId: 633224191 --- .github/workflows/test_bazel.yml | 2 -- src/google/protobuf/compiler/java/BUILD.bazel | 2 +- src/google/protobuf/compiler/java/file.cc | 2 +- .../compiler/java/{immutable => full}/BUILD.bazel | 6 +++--- .../compiler/java/{immutable => full}/enum.cc | 2 +- .../compiler/java/{immutable => full}/enum.h | 0 .../compiler/java/{immutable => full}/enum_field.cc | 2 +- .../compiler/java/{immutable => full}/enum_field.h | 2 +- .../compiler/java/{immutable => full}/extension.cc | 2 +- .../compiler/java/{immutable => full}/extension.h | 0 .../java/{immutable => full}/field_generator.h | 0 .../java/{immutable => full}/generator_factory.cc | 8 ++++---- .../java/{immutable => full}/generator_factory.h | 0 .../make_field_gens.cc} | 12 ++++++------ .../make_field_gens.h} | 8 ++++---- .../compiler/java/{immutable => full}/map_field.cc | 2 +- .../compiler/java/{immutable => full}/map_field.h | 2 +- .../compiler/java/{immutable => full}/message.cc | 10 +++++----- .../compiler/java/{immutable => full}/message.h | 2 +- .../java/{immutable => full}/message_builder.cc | 10 +++++----- .../java/{immutable => full}/message_builder.h | 2 +- .../java/{immutable => full}/message_field.cc | 2 +- .../java/{immutable => full}/message_field.h | 2 +- .../java/{immutable => full}/primitive_field.cc | 2 +- .../java/{immutable => full}/primitive_field.h | 2 +- .../compiler/java/{immutable => full}/service.cc | 2 +- .../compiler/java/{immutable => full}/service.h | 0 .../java/{immutable => full}/string_field.cc | 2 +- .../compiler/java/{immutable => full}/string_field.h | 2 +- src/google/protobuf/compiler/java/lite/BUILD.bazel | 6 +++--- .../protobuf/compiler/java/lite/generator_factory.cc | 2 +- .../{make_field_generators.cc => make_field_gens.cc} | 2 +- .../{make_field_generators.h => make_field_gens.h} | 6 +++--- src/google/protobuf/compiler/java/lite/message.cc | 2 +- .../protobuf/compiler/java/lite/message_builder.cc | 2 +- src/google/protobuf/compiler/objectivec/BUILD.bazel | 4 ++-- src/google/protobuf/compiler/objectivec/enum.cc | 2 +- src/google/protobuf/compiler/objectivec/message.cc | 2 +- .../objectivec/text_format_decode_data_unittest.cc | 2 +- ...{text_format_decode_data.cc => tf_decode_data.cc} | 2 +- .../{text_format_decode_data.h => tf_decode_data.h} | 6 +++--- .../protobuf/compiler/rust/accessors/BUILD.bazel | 2 +- .../protobuf/compiler/rust/accessors/accessors.cc | 2 +- .../accessors/{accessor_generator.h => generator.h} | 0 src/google/protobuf/compiler/rust/accessors/map.cc | 2 +- .../compiler/rust/accessors/repeated_field.cc | 2 +- .../compiler/rust/accessors/singular_message.cc | 2 +- .../compiler/rust/accessors/singular_scalar.cc | 2 +- .../compiler/rust/accessors/singular_string.cc | 2 +- .../compiler/rust/accessors/unsupported_field.cc | 2 +- 50 files changed, 71 insertions(+), 73 deletions(-) rename src/google/protobuf/compiler/java/{immutable => full}/BUILD.bazel (97%) rename src/google/protobuf/compiler/java/{immutable => full}/enum.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/enum.h (100%) rename src/google/protobuf/compiler/java/{immutable => full}/enum_field.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/enum_field.h (98%) rename src/google/protobuf/compiler/java/{immutable => full}/extension.cc (98%) rename src/google/protobuf/compiler/java/{immutable => full}/extension.h (100%) rename src/google/protobuf/compiler/java/{immutable => full}/field_generator.h (100%) rename src/google/protobuf/compiler/java/{immutable => full}/generator_factory.cc (89%) rename src/google/protobuf/compiler/java/{immutable => full}/generator_factory.h (100%) rename src/google/protobuf/compiler/java/{immutable/make_field_generators.cc => full/make_field_gens.cc} (91%) rename src/google/protobuf/compiler/java/{immutable/make_field_generators.h => full/make_field_gens.h} (77%) rename src/google/protobuf/compiler/java/{immutable => full}/map_field.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/map_field.h (97%) rename src/google/protobuf/compiler/java/{immutable => full}/message.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/message.h (97%) rename src/google/protobuf/compiler/java/{immutable => full}/message_builder.cc (98%) rename src/google/protobuf/compiler/java/{immutable => full}/message_builder.h (97%) rename src/google/protobuf/compiler/java/{immutable => full}/message_field.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/message_field.h (99%) rename src/google/protobuf/compiler/java/{immutable => full}/primitive_field.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/primitive_field.h (98%) rename src/google/protobuf/compiler/java/{immutable => full}/service.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/service.h (100%) rename src/google/protobuf/compiler/java/{immutable => full}/string_field.cc (99%) rename src/google/protobuf/compiler/java/{immutable => full}/string_field.h (98%) rename src/google/protobuf/compiler/java/lite/{make_field_generators.cc => make_field_gens.cc} (98%) rename src/google/protobuf/compiler/java/lite/{make_field_generators.h => make_field_gens.h} (80%) rename src/google/protobuf/compiler/objectivec/{text_format_decode_data.cc => tf_decode_data.cc} (98%) rename src/google/protobuf/compiler/objectivec/{text_format_decode_data.h => tf_decode_data.h} (88%) rename src/google/protobuf/compiler/rust/accessors/{accessor_generator.h => generator.h} (100%) diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml index 8b6139c35a..3a488a1bb4 100644 --- a/.github/workflows/test_bazel.yml +++ b/.github/workflows/test_bazel.yml @@ -47,8 +47,6 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/bazel@v3 - # TODO Silence this until we have a fix. - if: runner.os != 'Windows' || matrix.bzlmod with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: examples diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index ad25fecb27..40cd4a5fd8 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -128,7 +128,7 @@ cc_library( "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:retention", "//src/google/protobuf/compiler:versions", - "//src/google/protobuf/compiler/java/immutable", + "//src/google/protobuf/compiler/java/full", "//src/google/protobuf/compiler/java/lite", "//src/google/protobuf/io", "//src/google/protobuf/io:printer", diff --git a/src/google/protobuf/compiler/java/file.cc b/src/google/protobuf/compiler/java/file.cc index 2ce6e8978e..6995513bd1 100644 --- a/src/google/protobuf/compiler/java/file.cc +++ b/src/google/protobuf/compiler/java/file.cc @@ -25,7 +25,7 @@ #include "google/protobuf/compiler/java/generator_common.h" #include "google/protobuf/compiler/java/generator_factory.h" #include "google/protobuf/compiler/java/helpers.h" -#include "google/protobuf/compiler/java/immutable/generator_factory.h" +#include "google/protobuf/compiler/java/full/generator_factory.h" #include "google/protobuf/compiler/java/lite/generator_factory.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/compiler/java/options.h" diff --git a/src/google/protobuf/compiler/java/immutable/BUILD.bazel b/src/google/protobuf/compiler/java/full/BUILD.bazel similarity index 97% rename from src/google/protobuf/compiler/java/immutable/BUILD.bazel rename to src/google/protobuf/compiler/java/full/BUILD.bazel index 0f01091bcd..1a7f874326 100644 --- a/src/google/protobuf/compiler/java/immutable/BUILD.bazel +++ b/src/google/protobuf/compiler/java/full/BUILD.bazel @@ -46,8 +46,8 @@ cc_library( cc_library( name = "mfg", - srcs = ["make_field_generators.cc"], - hdrs = ["make_field_generators.h"], + srcs = ["make_field_gens.cc"], + hdrs = ["make_field_gens.h"], strip_include_prefix = "/src", deps = [ ":fg", @@ -94,7 +94,7 @@ cc_library( ) cc_library( - name = "immutable", + name = "full", srcs = [ "extension.cc", "generator_factory.cc", diff --git a/src/google/protobuf/compiler/java/immutable/enum.cc b/src/google/protobuf/compiler/java/full/enum.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/enum.cc rename to src/google/protobuf/compiler/java/full/enum.cc index 9ceefcb1a8..c321c08f31 100644 --- a/src/google/protobuf/compiler/java/immutable/enum.cc +++ b/src/google/protobuf/compiler/java/full/enum.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/enum.h" +#include "google/protobuf/compiler/java/full/enum.h" #include #include diff --git a/src/google/protobuf/compiler/java/immutable/enum.h b/src/google/protobuf/compiler/java/full/enum.h similarity index 100% rename from src/google/protobuf/compiler/java/immutable/enum.h rename to src/google/protobuf/compiler/java/full/enum.h diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/full/enum_field.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/enum_field.cc rename to src/google/protobuf/compiler/java/full/enum_field.cc index 71207a9d49..e0ea6eea65 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/full/enum_field.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/enum_field.h" +#include "google/protobuf/compiler/java/full/enum_field.h" #include #include diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.h b/src/google/protobuf/compiler/java/full/enum_field.h similarity index 98% rename from src/google/protobuf/compiler/java/immutable/enum_field.h rename to src/google/protobuf/compiler/java/full/enum_field.h index 625b7c21af..c238fad42c 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.h +++ b/src/google/protobuf/compiler/java/full/enum_field.h @@ -15,7 +15,7 @@ #include #include "absl/container/flat_hash_map.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/immutable/extension.cc b/src/google/protobuf/compiler/java/full/extension.cc similarity index 98% rename from src/google/protobuf/compiler/java/immutable/extension.cc rename to src/google/protobuf/compiler/java/full/extension.cc index d731d82400..03fd9d8524 100644 --- a/src/google/protobuf/compiler/java/immutable/extension.cc +++ b/src/google/protobuf/compiler/java/full/extension.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/extension.h" +#include "google/protobuf/compiler/java/full/extension.h" #include "google/protobuf/compiler/java/context.h" #include "google/protobuf/compiler/java/doc_comment.h" diff --git a/src/google/protobuf/compiler/java/immutable/extension.h b/src/google/protobuf/compiler/java/full/extension.h similarity index 100% rename from src/google/protobuf/compiler/java/immutable/extension.h rename to src/google/protobuf/compiler/java/full/extension.h diff --git a/src/google/protobuf/compiler/java/immutable/field_generator.h b/src/google/protobuf/compiler/java/full/field_generator.h similarity index 100% rename from src/google/protobuf/compiler/java/immutable/field_generator.h rename to src/google/protobuf/compiler/java/full/field_generator.h diff --git a/src/google/protobuf/compiler/java/immutable/generator_factory.cc b/src/google/protobuf/compiler/java/full/generator_factory.cc similarity index 89% rename from src/google/protobuf/compiler/java/immutable/generator_factory.cc rename to src/google/protobuf/compiler/java/full/generator_factory.cc index f1b9989449..280987f345 100644 --- a/src/google/protobuf/compiler/java/immutable/generator_factory.cc +++ b/src/google/protobuf/compiler/java/full/generator_factory.cc @@ -12,10 +12,10 @@ #include #include "google/protobuf/compiler/java/context.h" -#include "google/protobuf/compiler/java/immutable/enum.h" -#include "google/protobuf/compiler/java/immutable/extension.h" -#include "google/protobuf/compiler/java/immutable/message.h" -#include "google/protobuf/compiler/java/immutable/service.h" +#include "google/protobuf/compiler/java/full/enum.h" +#include "google/protobuf/compiler/java/full/extension.h" +#include "google/protobuf/compiler/java/full/message.h" +#include "google/protobuf/compiler/java/full/service.h" #include "google/protobuf/descriptor.h" namespace google { diff --git a/src/google/protobuf/compiler/java/immutable/generator_factory.h b/src/google/protobuf/compiler/java/full/generator_factory.h similarity index 100% rename from src/google/protobuf/compiler/java/immutable/generator_factory.h rename to src/google/protobuf/compiler/java/full/generator_factory.h diff --git a/src/google/protobuf/compiler/java/immutable/make_field_generators.cc b/src/google/protobuf/compiler/java/full/make_field_gens.cc similarity index 91% rename from src/google/protobuf/compiler/java/immutable/make_field_generators.cc rename to src/google/protobuf/compiler/java/full/make_field_gens.cc index 7636c493bb..08400a68ec 100644 --- a/src/google/protobuf/compiler/java/immutable/make_field_generators.cc +++ b/src/google/protobuf/compiler/java/full/make_field_gens.cc @@ -15,12 +15,12 @@ #include "google/protobuf/compiler/java/context.h" #include "google/protobuf/compiler/java/generator_common.h" #include "google/protobuf/compiler/java/helpers.h" -#include "google/protobuf/compiler/java/immutable/enum_field.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" -#include "google/protobuf/compiler/java/immutable/map_field.h" -#include "google/protobuf/compiler/java/immutable/message_field.h" -#include "google/protobuf/compiler/java/immutable/primitive_field.h" -#include "google/protobuf/compiler/java/immutable/string_field.h" +#include "google/protobuf/compiler/java/full/enum_field.h" +#include "google/protobuf/compiler/java/full/field_generator.h" +#include "google/protobuf/compiler/java/full/map_field.h" +#include "google/protobuf/compiler/java/full/message_field.h" +#include "google/protobuf/compiler/java/full/primitive_field.h" +#include "google/protobuf/compiler/java/full/string_field.h" #include "google/protobuf/descriptor.h" namespace google { diff --git a/src/google/protobuf/compiler/java/immutable/make_field_generators.h b/src/google/protobuf/compiler/java/full/make_field_gens.h similarity index 77% rename from src/google/protobuf/compiler/java/immutable/make_field_generators.h rename to src/google/protobuf/compiler/java/full/make_field_gens.h index a5dab5dbec..0c02f97823 100644 --- a/src/google/protobuf/compiler/java/immutable/make_field_generators.h +++ b/src/google/protobuf/compiler/java/full/make_field_gens.h @@ -5,12 +5,12 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_IMMUTABLE_MAKE_FIELD_GENERATORS_H__ -#define GOOGLE_PROTOBUF_COMPILER_JAVA_IMMUTABLE_MAKE_FIELD_GENERATORS_H__ +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_IMMUTABLE_MAKE_FIELD_GENS_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVA_IMMUTABLE_MAKE_FIELD_GENS_H__ #include "google/protobuf/compiler/java/context.h" #include "google/protobuf/compiler/java/generator_common.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" namespace google { @@ -26,4 +26,4 @@ FieldGeneratorMap MakeImmutableFieldGenerators( } // namespace protobuf } // namespace google -#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_IMMUTABLE_MAKE_FIELD_GENERATORS_H__ +#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_IMMUTABLE_MAKE_FIELD_GENS_H__ diff --git a/src/google/protobuf/compiler/java/immutable/map_field.cc b/src/google/protobuf/compiler/java/full/map_field.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/map_field.cc rename to src/google/protobuf/compiler/java/full/map_field.cc index 1281862958..be36f7dc16 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.cc +++ b/src/google/protobuf/compiler/java/full/map_field.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/compiler/java/immutable/map_field.h" +#include "google/protobuf/compiler/java/full/map_field.h" #include diff --git a/src/google/protobuf/compiler/java/immutable/map_field.h b/src/google/protobuf/compiler/java/full/map_field.h similarity index 97% rename from src/google/protobuf/compiler/java/immutable/map_field.h rename to src/google/protobuf/compiler/java/full/map_field.h index 72613f8659..c41205c4cb 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.h +++ b/src/google/protobuf/compiler/java/full/map_field.h @@ -9,7 +9,7 @@ #define GOOGLE_PROTOBUF_COMPILER_JAVA_MAP_FIELD_H__ #include "google/protobuf/compiler/java/context.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/immutable/message.cc b/src/google/protobuf/compiler/java/full/message.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/message.cc rename to src/google/protobuf/compiler/java/full/message.cc index 0d75c69efb..10abbb90e8 100644 --- a/src/google/protobuf/compiler/java/immutable/message.cc +++ b/src/google/protobuf/compiler/java/full/message.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/message.h" +#include "google/protobuf/compiler/java/full/message.h" #include #include @@ -28,10 +28,10 @@ #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/generator_common.h" #include "google/protobuf/compiler/java/helpers.h" -#include "google/protobuf/compiler/java/immutable/enum.h" -#include "google/protobuf/compiler/java/immutable/extension.h" -#include "google/protobuf/compiler/java/immutable/make_field_generators.h" -#include "google/protobuf/compiler/java/immutable/message_builder.h" +#include "google/protobuf/compiler/java/full/enum.h" +#include "google/protobuf/compiler/java/full/extension.h" +#include "google/protobuf/compiler/java/full/make_field_gens.h" +#include "google/protobuf/compiler/java/full/message_builder.h" #include "google/protobuf/compiler/java/message_serialization.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/java/immutable/message.h b/src/google/protobuf/compiler/java/full/message.h similarity index 97% rename from src/google/protobuf/compiler/java/immutable/message.h rename to src/google/protobuf/compiler/java/full/message.h index 268f191fcd..3b929b8d3f 100644 --- a/src/google/protobuf/compiler/java/immutable/message.h +++ b/src/google/protobuf/compiler/java/full/message.h @@ -17,7 +17,7 @@ #include #include "google/protobuf/compiler/java/generator_factory.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" namespace google { diff --git a/src/google/protobuf/compiler/java/immutable/message_builder.cc b/src/google/protobuf/compiler/java/full/message_builder.cc similarity index 98% rename from src/google/protobuf/compiler/java/immutable/message_builder.cc rename to src/google/protobuf/compiler/java/full/message_builder.cc index 3bc0900c16..53de5d3246 100644 --- a/src/google/protobuf/compiler/java/immutable/message_builder.cc +++ b/src/google/protobuf/compiler/java/full/message_builder.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/message_builder.h" +#include "google/protobuf/compiler/java/full/message_builder.h" #include #include @@ -27,10 +27,10 @@ #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/generator_factory.h" #include "google/protobuf/compiler/java/helpers.h" -#include "google/protobuf/compiler/java/immutable/enum.h" -#include "google/protobuf/compiler/java/immutable/extension.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" -#include "google/protobuf/compiler/java/immutable/make_field_generators.h" +#include "google/protobuf/compiler/java/full/enum.h" +#include "google/protobuf/compiler/java/full/extension.h" +#include "google/protobuf/compiler/java/full/field_generator.h" +#include "google/protobuf/compiler/java/full/make_field_gens.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/immutable/message_builder.h b/src/google/protobuf/compiler/java/full/message_builder.h similarity index 97% rename from src/google/protobuf/compiler/java/immutable/message_builder.h rename to src/google/protobuf/compiler/java/full/message_builder.h index ad2b378721..f63f8f4b0a 100644 --- a/src/google/protobuf/compiler/java/immutable/message_builder.h +++ b/src/google/protobuf/compiler/java/full/message_builder.h @@ -17,7 +17,7 @@ #include #include "absl/container/btree_map.h" -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" namespace google { diff --git a/src/google/protobuf/compiler/java/immutable/message_field.cc b/src/google/protobuf/compiler/java/full/message_field.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/message_field.cc rename to src/google/protobuf/compiler/java/full/message_field.cc index 31eb0ea858..54ae8dea17 100644 --- a/src/google/protobuf/compiler/java/immutable/message_field.cc +++ b/src/google/protobuf/compiler/java/full/message_field.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/message_field.h" +#include "google/protobuf/compiler/java/full/message_field.h" #include diff --git a/src/google/protobuf/compiler/java/immutable/message_field.h b/src/google/protobuf/compiler/java/full/message_field.h similarity index 99% rename from src/google/protobuf/compiler/java/immutable/message_field.h rename to src/google/protobuf/compiler/java/full/message_field.h index ee3c4e5289..709ea4309d 100644 --- a/src/google/protobuf/compiler/java/immutable/message_field.h +++ b/src/google/protobuf/compiler/java/full/message_field.h @@ -14,7 +14,7 @@ #include -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/immutable/primitive_field.cc b/src/google/protobuf/compiler/java/full/primitive_field.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/primitive_field.cc rename to src/google/protobuf/compiler/java/full/primitive_field.cc index 3dc402e5c2..b9fe799f4d 100644 --- a/src/google/protobuf/compiler/java/immutable/primitive_field.cc +++ b/src/google/protobuf/compiler/java/full/primitive_field.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/primitive_field.h" +#include "google/protobuf/compiler/java/full/primitive_field.h" #include #include diff --git a/src/google/protobuf/compiler/java/immutable/primitive_field.h b/src/google/protobuf/compiler/java/full/primitive_field.h similarity index 98% rename from src/google/protobuf/compiler/java/immutable/primitive_field.h rename to src/google/protobuf/compiler/java/full/primitive_field.h index 9aa52bfb8e..6533228ce3 100644 --- a/src/google/protobuf/compiler/java/immutable/primitive_field.h +++ b/src/google/protobuf/compiler/java/full/primitive_field.h @@ -14,7 +14,7 @@ #include -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" namespace google { diff --git a/src/google/protobuf/compiler/java/immutable/service.cc b/src/google/protobuf/compiler/java/full/service.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/service.cc rename to src/google/protobuf/compiler/java/full/service.cc index f93f9136d7..97d70966b4 100644 --- a/src/google/protobuf/compiler/java/immutable/service.cc +++ b/src/google/protobuf/compiler/java/full/service.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/service.h" +#include "google/protobuf/compiler/java/full/service.h" #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" diff --git a/src/google/protobuf/compiler/java/immutable/service.h b/src/google/protobuf/compiler/java/full/service.h similarity index 100% rename from src/google/protobuf/compiler/java/immutable/service.h rename to src/google/protobuf/compiler/java/full/service.h diff --git a/src/google/protobuf/compiler/java/immutable/string_field.cc b/src/google/protobuf/compiler/java/full/string_field.cc similarity index 99% rename from src/google/protobuf/compiler/java/immutable/string_field.cc rename to src/google/protobuf/compiler/java/full/string_field.cc index 1321f51675..d5b5d582bc 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.cc +++ b/src/google/protobuf/compiler/java/full/string_field.cc @@ -10,7 +10,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/immutable/string_field.h" +#include "google/protobuf/compiler/java/full/string_field.h" #include #include diff --git a/src/google/protobuf/compiler/java/immutable/string_field.h b/src/google/protobuf/compiler/java/full/string_field.h similarity index 98% rename from src/google/protobuf/compiler/java/immutable/string_field.h rename to src/google/protobuf/compiler/java/full/string_field.h index cf95be0da8..1d2d1b76b5 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.h +++ b/src/google/protobuf/compiler/java/full/string_field.h @@ -15,7 +15,7 @@ #include -#include "google/protobuf/compiler/java/immutable/field_generator.h" +#include "google/protobuf/compiler/java/full/field_generator.h" #include "google/protobuf/descriptor.h" namespace google { diff --git a/src/google/protobuf/compiler/java/lite/BUILD.bazel b/src/google/protobuf/compiler/java/lite/BUILD.bazel index a26f8f9da6..ec4b222aa6 100644 --- a/src/google/protobuf/compiler/java/lite/BUILD.bazel +++ b/src/google/protobuf/compiler/java/lite/BUILD.bazel @@ -3,7 +3,7 @@ cc_library( srcs = [ "enum_field.cc", "extension.cc", - "make_field_generators.cc", + "make_field_gens.cc", "map_field.cc", "message_field.cc", "primitive_field.cc", @@ -11,7 +11,7 @@ cc_library( ], hdrs = [ "field_generator.h", - "make_field_generators.h", + "make_field_gens.h", # We don't actually want to put the remaining headers in `hdrs`. # They are logically private, and should be in srcs=[], but # unfortunately `strip_include_prefix` doesn't have any effect @@ -68,7 +68,7 @@ cc_library( "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", "//src/google/protobuf/compiler/java:internal_helpers", - "//src/google/protobuf/compiler/java/immutable:service", + "//src/google/protobuf/compiler/java/full:service", "//src/google/protobuf/io", "//src/google/protobuf/io:printer", "@com_google_absl//absl/container:btree", diff --git a/src/google/protobuf/compiler/java/lite/generator_factory.cc b/src/google/protobuf/compiler/java/lite/generator_factory.cc index c3f52fe92a..8b9ddb4518 100644 --- a/src/google/protobuf/compiler/java/lite/generator_factory.cc +++ b/src/google/protobuf/compiler/java/lite/generator_factory.cc @@ -12,7 +12,7 @@ #include #include "google/protobuf/compiler/java/context.h" -#include "google/protobuf/compiler/java/immutable/service.h" +#include "google/protobuf/compiler/java/full/service.h" #include "google/protobuf/compiler/java/lite/enum.h" #include "google/protobuf/compiler/java/lite/extension.h" #include "google/protobuf/compiler/java/lite/message.h" diff --git a/src/google/protobuf/compiler/java/lite/make_field_generators.cc b/src/google/protobuf/compiler/java/lite/make_field_gens.cc similarity index 98% rename from src/google/protobuf/compiler/java/lite/make_field_generators.cc rename to src/google/protobuf/compiler/java/lite/make_field_gens.cc index 6071662c58..c4b92deac3 100644 --- a/src/google/protobuf/compiler/java/lite/make_field_generators.cc +++ b/src/google/protobuf/compiler/java/lite/make_field_gens.cc @@ -9,7 +9,7 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include "google/protobuf/compiler/java/lite/make_field_generators.h" +#include "google/protobuf/compiler/java/lite/make_field_gens.h" #include #include diff --git a/src/google/protobuf/compiler/java/lite/make_field_generators.h b/src/google/protobuf/compiler/java/lite/make_field_gens.h similarity index 80% rename from src/google/protobuf/compiler/java/lite/make_field_generators.h rename to src/google/protobuf/compiler/java/lite/make_field_gens.h index 436c3f896e..ebc9f22e05 100644 --- a/src/google/protobuf/compiler/java/lite/make_field_generators.h +++ b/src/google/protobuf/compiler/java/lite/make_field_gens.h @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_LITE_MAKE_FIELD_GENERATORS_H__ -#define GOOGLE_PROTOBUF_COMPILER_JAVA_LITE_MAKE_FIELD_GENERATORS_H__ +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_LITE_MAKE_FIELD_GENS_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVA_LITE_MAKE_FIELD_GENS_H__ #include #include @@ -29,4 +29,4 @@ FieldGeneratorMap MakeImmutableFieldLiteGenerators( } // namespace protobuf } // namespace google -#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_LITE_MAKE_FIELD_GENERATORS_H__ +#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_LITE_MAKE_FIELD_GENS_H__ diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index 3363162dc1..81e95242aa 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -33,7 +33,7 @@ #include "google/protobuf/compiler/java/helpers.h" #include "google/protobuf/compiler/java/lite/enum.h" #include "google/protobuf/compiler/java/lite/extension.h" -#include "google/protobuf/compiler/java/lite/make_field_generators.h" +#include "google/protobuf/compiler/java/lite/make_field_gens.h" #include "google/protobuf/compiler/java/lite/message_builder.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/descriptor.pb.h" diff --git a/src/google/protobuf/compiler/java/lite/message_builder.cc b/src/google/protobuf/compiler/java/lite/message_builder.cc index 9ae4081555..c5dac22bb5 100644 --- a/src/google/protobuf/compiler/java/lite/message_builder.cc +++ b/src/google/protobuf/compiler/java/lite/message_builder.cc @@ -20,7 +20,7 @@ #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" #include "google/protobuf/compiler/java/lite/enum.h" -#include "google/protobuf/compiler/java/lite/make_field_generators.h" +#include "google/protobuf/compiler/java/lite/make_field_gens.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/objectivec/BUILD.bazel b/src/google/protobuf/compiler/objectivec/BUILD.bazel index b608b0b0f5..3152976102 100644 --- a/src/google/protobuf/compiler/objectivec/BUILD.bazel +++ b/src/google/protobuf/compiler/objectivec/BUILD.bazel @@ -73,7 +73,7 @@ cc_library( "message_field.cc", "oneof.cc", "primitive_field.cc", - "text_format_decode_data.cc", + "tf_decode_data.cc", ], hdrs = [ "enum.h", @@ -90,7 +90,7 @@ cc_library( "oneof.h", "options.h", "primitive_field.h", - "text_format_decode_data.h", + "tf_decode_data.h", ], copts = COPTS, strip_include_prefix = "/src", diff --git a/src/google/protobuf/compiler/objectivec/enum.cc b/src/google/protobuf/compiler/objectivec/enum.cc index fae53a2606..c8bfcbc3df 100644 --- a/src/google/protobuf/compiler/objectivec/enum.cc +++ b/src/google/protobuf/compiler/objectivec/enum.cc @@ -17,7 +17,7 @@ #include "google/protobuf/compiler/objectivec/helpers.h" #include "google/protobuf/compiler/objectivec/names.h" #include "google/protobuf/compiler/objectivec/options.h" -#include "google/protobuf/compiler/objectivec/text_format_decode_data.h" +#include "google/protobuf/compiler/objectivec/tf_decode_data.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/objectivec/message.cc b/src/google/protobuf/compiler/objectivec/message.cc index c3f2ef6d3a..cde92f60d0 100644 --- a/src/google/protobuf/compiler/objectivec/message.cc +++ b/src/google/protobuf/compiler/objectivec/message.cc @@ -26,7 +26,7 @@ #include "google/protobuf/compiler/objectivec/names.h" #include "google/protobuf/compiler/objectivec/oneof.h" #include "google/protobuf/compiler/objectivec/options.h" -#include "google/protobuf/compiler/objectivec/text_format_decode_data.h" +#include "google/protobuf/compiler/objectivec/tf_decode_data.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc b/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc index c373128163..d730948934 100644 --- a/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc +++ b/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc @@ -5,12 +5,12 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/compiler/objectivec/text_format_decode_data.h" #include #include #include +#include "google/protobuf/compiler/objectivec/tf_decode_data.h" // Must be included last #include "google/protobuf/port_def.inc" diff --git a/src/google/protobuf/compiler/objectivec/text_format_decode_data.cc b/src/google/protobuf/compiler/objectivec/tf_decode_data.cc similarity index 98% rename from src/google/protobuf/compiler/objectivec/text_format_decode_data.cc rename to src/google/protobuf/compiler/objectivec/tf_decode_data.cc index fe48f8d157..8eade5d17a 100644 --- a/src/google/protobuf/compiler/objectivec/text_format_decode_data.cc +++ b/src/google/protobuf/compiler/objectivec/tf_decode_data.cc @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "google/protobuf/compiler/objectivec/text_format_decode_data.h" +#include "google/protobuf/compiler/objectivec/tf_decode_data.h" #include #include diff --git a/src/google/protobuf/compiler/objectivec/text_format_decode_data.h b/src/google/protobuf/compiler/objectivec/tf_decode_data.h similarity index 88% rename from src/google/protobuf/compiler/objectivec/text_format_decode_data.h rename to src/google/protobuf/compiler/objectivec/tf_decode_data.h index f562d32542..1e7932afa2 100644 --- a/src/google/protobuf/compiler/objectivec/text_format_decode_data.h +++ b/src/google/protobuf/compiler/objectivec/tf_decode_data.h @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_TEXT_FORMAT_DECODE_DATA_H__ -#define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_TEXT_FORMAT_DECODE_DATA_H__ +#ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_TF_DECODE_DATA_H__ +#define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_TF_DECODE_DATA_H__ #include #include @@ -55,4 +55,4 @@ class PROTOC_EXPORT TextFormatDecodeData { #include "google/protobuf/port_undef.inc" -#endif // GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_TEXT_FORMAT_DECODE_DATA_H__ +#endif // GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_TF_DECODE_DATA_H__ diff --git a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel index d864d62587..d7bf99e34e 100644 --- a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel @@ -20,9 +20,9 @@ cc_library( ], hdrs = [ "accessor_case.h", - "accessor_generator.h", "accessors.h", "default_value.h", + "generator.h", ], copts = COPTS, strip_include_prefix = "/src", diff --git a/src/google/protobuf/compiler/rust/accessors/accessors.cc b/src/google/protobuf/compiler/rust/accessors/accessors.cc index 9aa654577f..30984c9dcf 100644 --- a/src/google/protobuf/compiler/rust/accessors/accessors.cc +++ b/src/google/protobuf/compiler/rust/accessors/accessors.cc @@ -11,7 +11,7 @@ #include "absl/log/absl_log.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/rust_field_type.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/rust/accessors/accessor_generator.h b/src/google/protobuf/compiler/rust/accessors/generator.h similarity index 100% rename from src/google/protobuf/compiler/rust/accessors/accessor_generator.h rename to src/google/protobuf/compiler/rust/accessors/generator.h diff --git a/src/google/protobuf/compiler/rust/accessors/map.cc b/src/google/protobuf/compiler/rust/accessors/map.cc index 397d210af9..e38f03da5b 100644 --- a/src/google/protobuf/compiler/rust/accessors/map.cc +++ b/src/google/protobuf/compiler/rust/accessors/map.cc @@ -9,7 +9,7 @@ #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index 82445759fd..8f5ee25d45 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -10,7 +10,7 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/rust/accessors/singular_message.cc b/src/google/protobuf/compiler/rust/accessors/singular_message.cc index c5f77c2ca2..01f44fad36 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_message.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_message.cc @@ -10,7 +10,7 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc index 9131b361a7..a3d5c62ddd 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc @@ -10,8 +10,8 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" #include "google/protobuf/compiler/rust/accessors/default_value.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index dd4de8ce7e..0962229bdc 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -10,7 +10,7 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/naming.h" #include "google/protobuf/descriptor.h" diff --git a/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc b/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc index a04e778cd0..1ca66f63ef 100644 --- a/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/unsupported_field.cc @@ -7,7 +7,7 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/rust/accessors/accessor_case.h" -#include "google/protobuf/compiler/rust/accessors/accessor_generator.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/descriptor.h" From fad47367859a9d9b6b9548a43f221b1e0db52fea Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 May 2024 16:00:07 +0000 Subject: [PATCH 167/179] Auto-generate files after cl/633224191 --- src/file_lists.cmake | 60 ++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/file_lists.cmake b/src/file_lists.cmake index e22a26cf24..b69fcef7e4 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -332,20 +332,20 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/field_common.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/file.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/enum.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/enum_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/extension.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/generator_factory.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/make_field_gens.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/map_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/message.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/message_builder.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/message_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/primitive_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/service.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/string_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/helpers.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/enum.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/enum_field.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/extension.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/generator_factory.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/make_field_generators.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/map_field.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/message.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/message_builder.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/message_field.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/primitive_field.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/service.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/string_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/internal_helpers.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/java_features.pb.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/kotlin_generator.cc @@ -353,7 +353,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/enum_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/extension.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/generator_factory.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/make_field_generators.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/make_field_gens.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/map_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/message_builder.cc @@ -379,7 +379,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/names.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/oneof.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/primitive_field.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/text_format_decode_data.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/tf_decode_data.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/php/names.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/php/php_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.cc @@ -458,23 +458,23 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/field_common.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/file.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/enum.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/enum_field.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/extension.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/field_generator.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/generator_factory.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/make_field_gens.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/map_field.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/message.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/message_builder.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/message_field.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/primitive_field.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/service.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/full/string_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/generator_common.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/generator_factory.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/helpers.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/enum.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/enum_field.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/extension.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/field_generator.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/generator_factory.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/make_field_generators.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/map_field.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/message.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/message_builder.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/message_field.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/primitive_field.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/service.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/immutable/string_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/internal_helpers.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/java_features.pb.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/kotlin_generator.h @@ -483,7 +483,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/extension.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/field_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/generator_factory.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/make_field_generators.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/make_field_gens.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/map_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/message.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/lite/message_builder.h @@ -512,7 +512,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/oneof.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/options.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/primitive_field.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/text_format_decode_data.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/tf_decode_data.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/php/names.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/php/php_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.h @@ -523,9 +523,9 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/retention.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_case.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessor_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessors.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/default_value.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/context.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/crate_mapping.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/enum.h From 7f23b700fe301df7f12b5aa5b111dab010d56d07 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 May 2024 09:28:46 -0700 Subject: [PATCH 168/179] Introduce new API DownCastToMessage/DynamicCastToMessage as a replacement for down_cast/dynamic_cast from MessageLite to Message. The new functions work even when RTTI is not present. Also, add an assertion in DownCast to make sure we don't use it for message types. Message types have dedicated cast functions that should be used instead. PiperOrigin-RevId: 633236522 --- src/google/protobuf/compiler/cpp/message.cc | 3 +- src/google/protobuf/compiler/cpp/service.cc | 4 +- src/google/protobuf/compiler/cpp/unittest.inc | 4 +- src/google/protobuf/compiler/parser.cc | 8 ++-- src/google/protobuf/descriptor.cc | 2 +- src/google/protobuf/extension_set_heavy.cc | 2 +- src/google/protobuf/extension_set_unittest.cc | 1 + .../generated_message_tctable_full.cc | 5 +-- src/google/protobuf/map_field.cc | 1 - src/google/protobuf/map_test.inc | 30 +++++++------- src/google/protobuf/message.cc | 15 ++++--- src/google/protobuf/message.h | 41 +++++++++++++++++++ src/google/protobuf/message_lite.h | 5 +++ src/google/protobuf/port.h | 32 +++++++++------ .../protobuf/reflection_visit_field_info.h | 10 ++--- src/google/protobuf/reflection_visit_fields.h | 4 +- .../repeated_field_reflection_unittest.inc | 13 +++--- 17 files changed, 116 insertions(+), 64 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 9e43396b9d..e9e0ae2e37 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -3747,8 +3747,7 @@ void MessageGenerator::GenerateMergeFrom(io::Printer* p) { format( "void $classname$::CheckTypeAndMergeFrom(\n" " const ::$proto_ns$::MessageLite& from) {\n" - " MergeFrom(*::_pbi::DownCast(\n" - " &from));\n" + " MergeFrom(::$proto_ns$::DownCastToGenerated<$classname$>(from));\n" "}\n"); } } diff --git a/src/google/protobuf/compiler/cpp/service.cc b/src/google/protobuf/compiler/cpp/service.cc index 88527c119e..9da41c198a 100644 --- a/src/google/protobuf/compiler/cpp/service.cc +++ b/src/google/protobuf/compiler/cpp/service.cc @@ -256,8 +256,8 @@ void ServiceGenerator::GenerateCallMethodCases(io::Printer* printer) { R"cc( case $index$: $name$(controller, - ::$proto_ns$::internal::DownCast(request), - ::$proto_ns$::internal::DownCast<$output$*>(response), done); + ::$proto_ns$::DownCastToGenerated<$input$>(request), + ::$proto_ns$::DownCastToGenerated<$output$>(response), done); break; )cc"); } diff --git a/src/google/protobuf/compiler/cpp/unittest.inc b/src/google/protobuf/compiler/cpp/unittest.inc index 8da0de29ea..6c1fe420e0 100644 --- a/src/google/protobuf/compiler/cpp/unittest.inc +++ b/src/google/protobuf/compiler/cpp/unittest.inc @@ -1314,13 +1314,13 @@ TEST_F(GENERATED_SERVICE_TEST_NAME, CallMethodTypeFailure) { EXPECT_DEBUG_DEATH( mock_service_.CallMethod(foo_, &mock_controller_, &foo_request_, &bar_response_, done_.get()), - "dynamic_cast"); + "DynamicCastToGenerated"); mock_service_.Reset(); EXPECT_DEBUG_DEATH( mock_service_.CallMethod(foo_, &mock_controller_, &bar_request_, &foo_response_, done_.get()), - "dynamic_cast"); + "DynamicCastToGenerated"); #endif // GTEST_HAS_DEATH_TEST } diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index 129ea2f3f2..c2595fe74a 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -38,6 +38,7 @@ #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/strtod.h" #include "google/protobuf/io/tokenizer.h" +#include "google/protobuf/message_lite.h" #include "google/protobuf/port.h" #include "google/protobuf/wire_format.h" @@ -49,8 +50,6 @@ namespace protobuf { namespace compiler { namespace { -using ::google::protobuf::internal::DownCast; - using TypeNameMap = absl::flat_hash_map; @@ -1566,8 +1565,9 @@ bool Parser::ParseOption(Message* options, } UninterpretedOption* uninterpreted_option = - DownCast(options->GetReflection()->AddMessage( - options, uninterpreted_option_field)); + DownCastToGenerated( + options->GetReflection()->AddMessage(options, + uninterpreted_option_field)); // Parse dot-separated name. { diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 47eeb9012d..7264daff6a 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -8590,7 +8590,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptionsImpl( *original_options, original_uninterpreted_options_field); for (int i = 0; i < num_uninterpreted_options; ++i) { src_path.push_back(i); - uninterpreted_option_ = DownCast( + uninterpreted_option_ = DownCastToGenerated( &original_options->GetReflection()->GetRepeatedMessage( *original_options, original_uninterpreted_options_field, i)); if (!InterpretSingleOption(options, src_path, diff --git a/src/google/protobuf/extension_set_heavy.cc b/src/google/protobuf/extension_set_heavy.cc index 06a6a7f7f7..60ad7b9e83 100644 --- a/src/google/protobuf/extension_set_heavy.cc +++ b/src/google/protobuf/extension_set_heavy.cc @@ -406,7 +406,7 @@ size_t ExtensionSet::Extension::SpaceUsedExcludingSelfLong() const { if (is_lazy) { total_size += lazymessage_value->SpaceUsedLong(); } else { - total_size += DownCast(message_value)->SpaceUsedLong(); + total_size += DownCastToMessage(message_value)->SpaceUsedLong(); } break; default: diff --git a/src/google/protobuf/extension_set_unittest.cc b/src/google/protobuf/extension_set_unittest.cc index 019eec6da3..f983f3ca49 100644 --- a/src/google/protobuf/extension_set_unittest.cc +++ b/src/google/protobuf/extension_set_unittest.cc @@ -26,6 +26,7 @@ #include "google/protobuf/dynamic_message.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/message_lite.h" #include "google/protobuf/test_util.h" #include "google/protobuf/test_util2.h" #include "google/protobuf/text_format.h" diff --git a/src/google/protobuf/generated_message_tctable_full.cc b/src/google/protobuf/generated_message_tctable_full.cc index 75b86d8971..430d375da0 100644 --- a/src/google/protobuf/generated_message_tctable_full.cc +++ b/src/google/protobuf/generated_message_tctable_full.cc @@ -44,7 +44,6 @@ namespace google { namespace protobuf { namespace internal { -using ::google::protobuf::internal::DownCast; const char* TcParser::GenericFallback(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return GenericFallbackImpl( @@ -64,7 +63,7 @@ const char* TcParser::ReflectionFallback(PROTOBUF_TC_PARAM_DECL) { return ptr; } - auto* full_msg = DownCast(msg); + auto* full_msg = DownCastToMessage(msg); auto* descriptor = full_msg->GetDescriptor(); auto* reflection = full_msg->GetReflection(); int field_number = WireFormatLite::GetTagFieldNumber(tag); @@ -88,7 +87,7 @@ const char* TcParser::ReflectionParseLoop(PROTOBUF_TC_PARAM_DECL) { (void)table; (void)hasbits; // Call into the wire format reflective parse loop. - return WireFormat::_InternalParse(DownCast(msg), ptr, ctx); + return WireFormat::_InternalParse(DownCastToMessage(msg), ptr, ctx); } const char* TcParser::MessageSetWireFormatParseLoop( diff --git a/src/google/protobuf/map_field.cc b/src/google/protobuf/map_field.cc index 6496d715f8..3045730932 100644 --- a/src/google/protobuf/map_field.cc +++ b/src/google/protobuf/map_field.cc @@ -21,7 +21,6 @@ namespace google { namespace protobuf { namespace internal { -using ::google::protobuf::internal::DownCast; VariantKey RealKeyToVariantKey::operator()(const MapKey& value) const { switch (value.type()) { diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index cb396b79e0..c67cd586c3 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -1578,7 +1578,7 @@ TEST_F(MapFieldReflectionTest, RegularFields) { message_int32_message.GetReflection()->GetInt32( message_int32_message, fd_map_int32_foreign_message_key); const ForeignMessage& value_int32_message = - DownCast( + DownCastToGenerated( message_int32_message.GetReflection()->GetMessage( message_int32_message, fd_map_int32_foreign_message_value)); EXPECT_EQ(value_int32_message.c(), Func(key_int32_message, 6)); @@ -1615,7 +1615,7 @@ TEST_F(MapFieldReflectionTest, RegularFields) { message_int32_message.GetReflection()->GetInt32( message_int32_message, fd_map_int32_foreign_message_key); const ForeignMessage& value_int32_message = - DownCast( + DownCastToGenerated( message_int32_message.GetReflection()->GetMessage( message_int32_message, fd_map_int32_foreign_message_value)); EXPECT_EQ(value_int32_message.c(), Func(key_int32_message, 6)); @@ -1652,7 +1652,7 @@ TEST_F(MapFieldReflectionTest, RegularFields) { int32_t key_int32_message = message_int32_message->GetReflection()->GetInt32( *message_int32_message, fd_map_int32_foreign_message_key); - ForeignMessage* value_int32_message = DownCast( + ForeignMessage* value_int32_message = DownCastToGenerated( message_int32_message->GetReflection()->MutableMessage( message_int32_message, fd_map_int32_foreign_message_value)); value_int32_message->set_c(Func(key_int32_message, -6)); @@ -1808,7 +1808,7 @@ TEST_F(MapFieldReflectionTest, RepeatedFieldRefForRegularFields) { message_int32_message.GetReflection()->GetInt32( message_int32_message, fd_map_int32_foreign_message_key); const ForeignMessage& value_int32_message = - DownCast( + DownCastToGenerated( message_int32_message.GetReflection()->GetMessage( message_int32_message, fd_map_int32_foreign_message_value)); EXPECT_EQ(value_int32_message.c(), Func(key_int32_message, 6)); @@ -1849,7 +1849,7 @@ TEST_F(MapFieldReflectionTest, RepeatedFieldRefForRegularFields) { message_int32_message.GetReflection()->GetInt32( message_int32_message, fd_map_int32_foreign_message_key); const ForeignMessage& value_int32_message = - DownCast( + DownCastToGenerated( message_int32_message.GetReflection()->GetMessage( message_int32_message, fd_map_int32_foreign_message_value)); EXPECT_EQ(value_int32_message.c(), Func(key_int32_message, 6)); @@ -1966,8 +1966,8 @@ TEST_F(MapFieldReflectionTest, RepeatedFieldRefForRegularFields) { const Message& message = *it; int32_t key = message.GetReflection()->GetInt32( message, fd_map_int32_foreign_message_key); - const ForeignMessage& sub_message = - DownCast(message.GetReflection()->GetMessage( + const ForeignMessage& sub_message = DownCastToGenerated( + message.GetReflection()->GetMessage( message, fd_map_int32_foreign_message_value)); result[key].MergeFrom(sub_message); ++index; @@ -2120,14 +2120,14 @@ TEST_F(MapFieldReflectionTest, RepeatedFieldRefForRegularFields) { { const Message& message0a = mmf_int32_foreign_message.Get(0, entry_int32_foreign_message.get()); - const ForeignMessage& sub_message0a = - DownCast(message0a.GetReflection()->GetMessage( + const ForeignMessage& sub_message0a = DownCastToGenerated( + message0a.GetReflection()->GetMessage( message0a, fd_map_int32_foreign_message_value)); int32_t int32_value0a = sub_message0a.c(); const Message& message9a = mmf_int32_foreign_message.Get(9, entry_int32_foreign_message.get()); - const ForeignMessage& sub_message9a = - DownCast(message9a.GetReflection()->GetMessage( + const ForeignMessage& sub_message9a = DownCastToGenerated( + message9a.GetReflection()->GetMessage( message9a, fd_map_int32_foreign_message_value)); int32_t int32_value9a = sub_message9a.c(); @@ -2135,14 +2135,14 @@ TEST_F(MapFieldReflectionTest, RepeatedFieldRefForRegularFields) { const Message& message0b = mmf_int32_foreign_message.Get(0, entry_int32_foreign_message.get()); - const ForeignMessage& sub_message0b = - DownCast(message0b.GetReflection()->GetMessage( + const ForeignMessage& sub_message0b = DownCastToGenerated( + message0b.GetReflection()->GetMessage( message0b, fd_map_int32_foreign_message_value)); int32_t int32_value0b = sub_message0b.c(); const Message& message9b = mmf_int32_foreign_message.Get(9, entry_int32_foreign_message.get()); - const ForeignMessage& sub_message9b = - DownCast(message9b.GetReflection()->GetMessage( + const ForeignMessage& sub_message9b = DownCastToGenerated( + message9b.GetReflection()->GetMessage( message9b, fd_map_int32_foreign_message_value)); int32_t int32_value9b = sub_message9b.c(); diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index caf941ad5b..2f4f84e4fd 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -63,12 +63,11 @@ void RegisterFileLevelMetadata(const DescriptorTable* descriptor_table); } // namespace internal -using internal::DownCast; using internal::ReflectionOps; using internal::WireFormat; void Message::MergeImpl(MessageLite& to, const MessageLite& from) { - ReflectionOps::Merge(DownCast(from), DownCast(&to)); + ReflectionOps::Merge(DownCastToMessage(from), DownCastToMessage(&to)); } void Message::MergeFrom(const Message& from) { @@ -82,7 +81,7 @@ void Message::MergeFrom(const Message& from) { } void Message::CheckTypeAndMergeFrom(const MessageLite& other) { - MergeFrom(*DownCast(&other)); + MergeFrom(DownCastToMessage(other)); } void Message::CopyFrom(const Message& from) { @@ -113,7 +112,7 @@ void Message::CopyFrom(const Message& from) { void Message::Clear() { ReflectionOps::Clear(this); } bool Message::IsInitializedImpl(const MessageLite& msg) { - return ReflectionOps::IsInitialized(DownCast(msg)); + return ReflectionOps::IsInitialized(DownCastToMessage(msg)); } void Message::FindInitializationErrors(std::vector* errors) const { @@ -189,20 +188,20 @@ size_t Message::SpaceUsedLong() const { } static std::string GetTypeNameImpl(const MessageLite& msg) { - return DownCast(msg).GetDescriptor()->full_name(); + return DownCastToMessage(msg).GetDescriptor()->full_name(); } static std::string InitializationErrorStringImpl(const MessageLite& msg) { - return DownCast(msg).InitializationErrorString(); + return DownCastToMessage(msg).InitializationErrorString(); } const internal::TcParseTableBase* Message::GetTcParseTableImpl( const MessageLite& msg) { - return DownCast(msg).GetReflection()->GetTcParseTable(); + return DownCastToMessage(msg).GetReflection()->GetTcParseTable(); } size_t Message::SpaceUsedLongImpl(const MessageLite& msg_lite) { - auto& msg = DownCast(msg_lite); + auto& msg = DownCastToMessage(msg_lite); return msg.GetReflection()->SpaceUsedLong(msg); } diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 88666e62e1..38249d1e51 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -1449,6 +1449,47 @@ void LinkMessageReflection() { internal::StrongReferenceToType(); } +// Tries to downcast this message from MessageLite to Message. Returns nullptr +// if this class is not an instance of Message. eg if the message was defined +// with optimized_for=LITE_RUNTIME. This works even if RTTI is disabled. +inline const Message* DynamicCastToMessage(const MessageLite* lite) { + return lite == nullptr || internal::GetClassData(*lite)->is_lite + ? nullptr + : static_cast(lite); +} +inline Message* DynamicCastToMessage(MessageLite* lite) { + return const_cast( + DynamicCastToMessage(static_cast(lite))); +} +inline const Message& DynamicCastToMessage(const MessageLite& lite) { + auto* res = DynamicCastToMessage(&lite); + ABSL_CHECK(res != nullptr) + << "Cannot to `Message` type " << lite.GetTypeName(); + return *res; +} +inline Message& DynamicCastToMessage(MessageLite& lite) { + return const_cast( + DynamicCastToMessage(static_cast(lite))); +} + +// A lightweight function for downcasting a MessageLite to Message. It should +// only be used when the caller is certain that the argument is a Message +// object. +inline const Message* DownCastToMessage(const MessageLite* lite) { + ABSL_CHECK(lite == nullptr || DynamicCastToMessage(lite) != nullptr); + return static_cast(lite); +} +inline Message* DownCastToMessage(MessageLite* lite) { + return const_cast( + DownCastToMessage(static_cast(lite))); +} +inline const Message& DownCastToMessage(const MessageLite& lite) { + return *DownCastToMessage(&lite); +} +inline Message& DownCastToMessage(MessageLite& lite) { + return *DownCastToMessage(&lite); +} + // ============================================================================= // Implementation details for {Get,Mutable}RawRepeatedPtrField. We provide // specializations for , and and diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index abb869e817..a625c20059 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -124,6 +124,7 @@ class PROTOBUF_EXPORT CachedSize { // For MessageLite to friend. class TypeId; +auto GetClassData(const MessageLite& msg); class SwapFieldHelper; @@ -705,6 +706,8 @@ class PROTOBUF_EXPORT MessageLite { template friend class internal::GenericTypeHandler; + friend auto internal::GetClassData(const MessageLite& msg); + void LogInitializationErrorMessage() const; bool MergeFromImpl(io::CodedInputStream* input, ParseFlags parse_flags); @@ -758,6 +761,8 @@ class TypeId { const MessageLite::ClassData* data_; }; +inline auto GetClassData(const MessageLite& msg) { return msg.GetClassData(); } + template bool MergeFromImpl(absl::string_view input, MessageLite* msg, const internal::TcParseTableBase* tc_table, diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h index 721afc90b1..a31722cd4d 100644 --- a/src/google/protobuf/port.h +++ b/src/google/protobuf/port.h @@ -159,28 +159,34 @@ struct ArenaInitialized { }; template -inline To DownCast(From* f) { - static_assert( - std::is_base_of::type>::value, - "illegal DownCast"); +void AssertDownCast(From* from) { + static_assert(std::is_base_of::value, "illegal DownCast"); + +#if defined(__cpp_concepts) + // Check that this function is not used to downcast message types. + // For those we should use {Down,Dynamic}CastTo{Message,Generated}. + static_assert(!requires { + std::derived_from, + typename std::remove_pointer_t::MessageLite>; + }); +#endif #if PROTOBUF_RTTI // RTTI: debug mode only! - assert(f == nullptr || dynamic_cast(f) != nullptr); + assert(from == nullptr || dynamic_cast(from) != nullptr); #endif +} + +template +inline To DownCast(From* f) { + AssertDownCast>(f); return static_cast(f); } template inline ToRef DownCast(From& f) { - using To = typename std::remove_reference::type; - static_assert(std::is_base_of::value, "illegal DownCast"); - -#if PROTOBUF_RTTI - // RTTI: debug mode only! - assert(dynamic_cast(&f) != nullptr); -#endif - return *static_cast(&f); + AssertDownCast>(&f); + return static_cast(f); } // Looks up the name of `T` via RTTI, if RTTI is available. diff --git a/src/google/protobuf/reflection_visit_field_info.h b/src/google/protobuf/reflection_visit_field_info.h index bca311a4f8..29056bc9da 100644 --- a/src/google/protobuf/reflection_visit_field_info.h +++ b/src/google/protobuf/reflection_visit_field_info.h @@ -200,10 +200,10 @@ struct DynamicExtensionInfoHelper { } static const Message& GetMessage(const Extension& ext) { - return DownCast(*ext.message_value); + return DownCastToMessage(*ext.message_value); } static Message& MutableMessage(Extension& ext) { - return DownCast(*ext.message_value); + return DownCastToMessage(*ext.message_value); } static void ClearMessage(Extension& ext) { ext.is_cleared = true; @@ -212,18 +212,18 @@ struct DynamicExtensionInfoHelper { static const Message& GetLazyMessage(const Extension& ext, const Message& prototype, Arena* arena) { - return DownCast( + return DownCastToMessage( ext.lazymessage_value->GetMessage(prototype, arena)); } static const Message& GetLazyMessageIgnoreUnparsed(const Extension& ext, const Message& prototype, Arena* arena) { - return DownCast( + return DownCastToMessage( ext.lazymessage_value->GetMessageIgnoreUnparsed(prototype, arena)); } static Message& MutableLazyMessage(Extension& ext, const Message& prototype, Arena* arena) { - return DownCast( + return DownCastToMessage( *ext.lazymessage_value->MutableMessage(prototype, arena)); } static void ClearLazyMessage(Extension& ext) { diff --git a/src/google/protobuf/reflection_visit_fields.h b/src/google/protobuf/reflection_visit_fields.h index 0e3acb0e75..e554a76ef9 100644 --- a/src/google/protobuf/reflection_visit_fields.h +++ b/src/google/protobuf/reflection_visit_fields.h @@ -422,7 +422,7 @@ void ReflectionVisit::VisitMessageFields(const Message& message, FieldDescriptor::CPPTYPE_MESSAGE) { if constexpr (info.is_repeated) { for (const auto& it : info.Get()) { - func(DownCast(it)); + func(DownCastToMessage(it)); } } else { func(info.Get()); @@ -452,7 +452,7 @@ void ReflectionVisit::VisitMessageFields(Message& message, CallbackFn&& func) { FieldDescriptor::CPPTYPE_MESSAGE) { if constexpr (info.is_repeated) { for (auto& it : info.Mutable()) { - func(DownCast(it)); + func(DownCastToMessage(it)); } } else { func(info.Mutable()); diff --git a/src/google/protobuf/repeated_field_reflection_unittest.inc b/src/google/protobuf/repeated_field_reflection_unittest.inc index 97d5c4128e..8e561adbe1 100644 --- a/src/google/protobuf/repeated_field_reflection_unittest.inc +++ b/src/google/protobuf/repeated_field_reflection_unittest.inc @@ -91,7 +91,7 @@ TEST(REFLECTION_TEST, RegularFields) { EXPECT_EQ(rf_double.Get(i), Func(i, 2)); EXPECT_EQ(rpf_string.Get(i), StrFunc(i, 5)); EXPECT_EQ(rpf_foreign_message.Get(i).c(), Func(i, 6)); - EXPECT_EQ(DownCast(&rpf_message.Get(i))->c(), + EXPECT_EQ(DownCastToGenerated(&rpf_message.Get(i))->c(), Func(i, 6)); // Check gets through mutable objects. @@ -99,7 +99,7 @@ TEST(REFLECTION_TEST, RegularFields) { EXPECT_EQ(mrf_double->Get(i), Func(i, 2)); EXPECT_EQ(mrpf_string->Get(i), StrFunc(i, 5)); EXPECT_EQ(mrpf_foreign_message->Get(i).c(), Func(i, 6)); - EXPECT_EQ(DownCast(&mrpf_message->Get(i))->c(), + EXPECT_EQ(DownCastToGenerated(&mrpf_message->Get(i))->c(), Func(i, 6)); // Check sets through mutable objects. @@ -111,7 +111,8 @@ TEST(REFLECTION_TEST, RegularFields) { EXPECT_EQ(message.repeated_double(i), Func(i, -2)); EXPECT_EQ(message.repeated_string(i), StrFunc(i, -5)); EXPECT_EQ(message.repeated_foreign_message(i).c(), Func(i, -6)); - DownCast(mrpf_message->Mutable(i))->set_c(Func(i, 7)); + DownCastToGenerated(mrpf_message->Mutable(i)) + ->set_c(Func(i, 7)); EXPECT_EQ(message.repeated_foreign_message(i).c(), Func(i, 7)); } @@ -271,7 +272,8 @@ TEST(REFLECTION_TEST, RepeatedFieldRefForRegularFields) { ForeignMessage scratch_space; EXPECT_EQ(rf_foreign_message.Get(i, &scratch_space).c(), Func(i, 6)); EXPECT_EQ( - DownCast(rf_message.Get(i, &scratch_space)).c(), + DownCastToGenerated(rf_message.Get(i, &scratch_space)) + .c(), Func(i, 6)); // Check gets through mutable objects. @@ -280,7 +282,8 @@ TEST(REFLECTION_TEST, RepeatedFieldRefForRegularFields) { EXPECT_EQ(mrf_string.Get(i), StrFunc(i, 5)); EXPECT_EQ(mrf_foreign_message.Get(i, &scratch_space).c(), Func(i, 6)); EXPECT_EQ( - DownCast(mrf_message.Get(i, &scratch_space)).c(), + DownCastToGenerated(mrf_message.Get(i, &scratch_space)) + .c(), Func(i, 6)); // Check sets through mutable objects. From 2238cdc091ed7e8a5079d480611fdf37d8bfb22c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 13 May 2024 10:55:38 -0700 Subject: [PATCH 169/179] Add impl Clone for Rust proto owned messages. PiperOrigin-RevId: 633265290 --- rust/test/shared/accessors_test.rs | 12 ++++++++++++ src/google/protobuf/compiler/rust/message.cc | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index e14bcafc4c..2c324515be 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -779,6 +779,18 @@ fn test_submsg_setter() { assert_that!(parent.optional_nested_message().bb(), eq(7)); } +#[test] +fn test_clone() { + let mut m = TestAllTypes::new(); + m.set_optional_int32(42); + let clone = m.clone(); + assert_that!(clone.optional_int32(), eq(42)); + m.clear_optional_int32(); + assert_that!(m.has_optional_int32(), eq(false)); + assert_that!(clone.has_optional_int32(), eq(true)); + assert_that!(clone.optional_int32(), eq(42)); +} + #[test] fn test_to_owned() { let mut m = TestAllTypes::new(); diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index f6e472886a..69d6f3ff06 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -1102,6 +1102,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $std$::clone::Clone for $Msg$ { + fn clone(&self) -> Self { + self.as_view().to_owned() + } + } + extern "C" { $Msg_externs$ From 30a40eefe5955c1f108376b7b97ba79be9fa6aa8 Mon Sep 17 00:00:00 2001 From: pbatg <23102240+pbatg@users.noreply.github.com> Date: Tue, 14 May 2024 08:13:33 -0700 Subject: [PATCH 170/179] Fixed broken link in upb README.md (#16717) I noticed this link was broken. I'm not familiar with the project, but the code in https://github.com/protocolbuffers/protobuf/tree/main/upb/python does appear to import upb headers, so I think it's the right URL. Closes #16717 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/16717 from pbatg:patch-1 c9e654308dac1cf29958ebfc5d50cb5e5260d520 PiperOrigin-RevId: 633589882 --- upb/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upb/README.md b/upb/README.md index a0c6849eca..61eaddd554 100644 --- a/upb/README.md +++ b/upb/README.md @@ -8,7 +8,7 @@ in C. upb is the core runtime for protobuf languages extensions in [Ruby](https://github.com/protocolbuffers/protobuf/tree/main/ruby), [PHP](https://github.com/protocolbuffers/protobuf/tree/main/php), and -[Python](https://github.com/protocolbuffers/protobuf/tree/main/upb/python). +[Python](https://github.com/protocolbuffers/protobuf/tree/main/python). While upb offers a C API, the C API & ABI **are not stable**. For this reason, upb is not generally offered as a C library for direct consumption, and there From 448e326200a69afd83ae86e44dcddb612f007763 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 May 2024 10:36:48 -0700 Subject: [PATCH 171/179] Use bool HasHasbits(const FieldDescriptor*) instead of manual checks. It is not sufficient to check schema_.HasHasbits() followed by naively skipping repeated and oneof fields as that can miss proto3 messages with message fields and other scalar fields without "optional" keyword. Use internal::cpp::HasHasbits(const FieldDescriptor*) instead. PiperOrigin-RevId: 633633184 --- .../protobuf/generated_message_reflection.cc | 5 +- .../generated_message_reflection_unittest.cc | 8 ++ src/google/protobuf/unittest_proto3.proto | 76 +++++++++++++++++++ 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 4d7d748105..92045dabec 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -1265,10 +1265,9 @@ void Reflection::InternalSwap(Message* lhs, Message* rhs) const { int fields_with_has_bits = 0; for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); - if (field->is_repeated() || schema_.InRealOneof(field)) { - continue; + if (internal::cpp::HasHasbit(field)) { + ++fields_with_has_bits; } - fields_with_has_bits++; } int has_bits_size = (fields_with_has_bits + 31) / 32; diff --git a/src/google/protobuf/generated_message_reflection_unittest.cc b/src/google/protobuf/generated_message_reflection_unittest.cc index 94a7c15c35..a432d74d67 100644 --- a/src/google/protobuf/generated_message_reflection_unittest.cc +++ b/src/google/protobuf/generated_message_reflection_unittest.cc @@ -41,6 +41,7 @@ #include "google/protobuf/unittest.pb.h" #include "google/protobuf/unittest_mset.pb.h" #include "google/protobuf/unittest_mset_wire_format.pb.h" +#include "google/protobuf/unittest_proto3.pb.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -1756,6 +1757,13 @@ TEST(GeneratedMessageReflection, ListFieldsSorted) { Pointee(Property(&FieldDescriptor::number, 101)))); } +TEST(GeneratedMessageReflection, SwapImplicitPresenceShouldWork) { + proto3_unittest::TestHasbits lhs, rhs; + rhs.mutable_child()->set_optional_int32(-1); + lhs.GetReflection()->Swap(&lhs, &rhs); + EXPECT_EQ(lhs.child().optional_int32(), -1); +} + } // namespace } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/unittest_proto3.proto b/src/google/protobuf/unittest_proto3.proto index 9067728ae2..d88521afea 100644 --- a/src/google/protobuf/unittest_proto3.proto +++ b/src/google/protobuf/unittest_proto3.proto @@ -205,3 +205,79 @@ message TestOneof2 { BAZ = 3; } } + +// If bool fields are incorrectly assumed to have hasbits, InternalSwap would +// result in swapping N more 32bit hasbits incorrectly. Considering padding, we +// need many bool fields to stress this. +message TestHasbits { + bool b1 = 1; + bool b2 = 2; + bool b3 = 3; + bool b4 = 4; + bool b5 = 5; + bool b6 = 6; + bool b7 = 7; + bool b8 = 8; + bool b9 = 9; + bool b10 = 10; + bool b11 = 11; + bool b12 = 12; + bool b13 = 13; + bool b14 = 14; + bool b15 = 15; + bool b16 = 16; + bool b17 = 17; + bool b18 = 18; + bool b19 = 19; + bool b20 = 20; + bool b21 = 21; + bool b22 = 22; + bool b23 = 23; + bool b24 = 24; + bool b25 = 25; + bool b26 = 26; + bool b27 = 27; + bool b28 = 28; + bool b29 = 29; + bool b30 = 30; + bool b31 = 31; + bool b32 = 32; + bool b33 = 33; + bool b34 = 34; + bool b35 = 35; + bool b36 = 36; + bool b37 = 37; + bool b38 = 38; + bool b39 = 39; + bool b40 = 40; + bool b41 = 41; + bool b42 = 42; + bool b43 = 43; + bool b44 = 44; + bool b45 = 45; + bool b46 = 46; + bool b47 = 47; + bool b48 = 48; + bool b49 = 49; + bool b50 = 50; + bool b51 = 51; + bool b52 = 52; + bool b53 = 53; + bool b54 = 54; + bool b55 = 55; + bool b56 = 56; + bool b57 = 57; + bool b58 = 58; + bool b59 = 59; + bool b60 = 60; + bool b61 = 61; + bool b62 = 62; + bool b63 = 63; + bool b64 = 64; + bool b65 = 65; + bool b66 = 66; + bool b67 = 67; + bool b68 = 68; + bool b69 = 69; + TestAllTypes child = 100; +} From 396d6617674622bf9be4b73088f2f17e9f6eb374 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 May 2024 11:12:50 -0700 Subject: [PATCH 172/179] Add support for setting extensions with Ptr in Upb C++ protos. PiperOrigin-RevId: 633646216 --- protos/protos.h | 55 +++++++++++------- protos_generator/tests/test_generated.cc | 73 +++++++++++++++++++++++- protos_generator/tests/test_model.proto | 13 +++++ 3 files changed, 118 insertions(+), 23 deletions(-) diff --git a/protos/protos.h b/protos/protos.h index 0bac4b683e..87547f9008 100644 --- a/protos/protos.h +++ b/protos/protos.h @@ -340,29 +340,28 @@ ABSL_MUST_USE_RESULT bool HasExtension( return HasExtension(protos::Ptr(message), id); } -template , typename = EnableIfMutableProto> +template , + typename = EnableIfMutableProto> void ClearExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::protos::internal::ExtensionIdentifier& id) { static_assert(!std::is_const_v, ""); upb_Message_ClearExtension(internal::GetInternalMsg(message), id.mini_table_ext()); } -template > +template > void ClearExtension( T* message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::protos::internal::ExtensionIdentifier& id) { ClearExtension(::protos::Ptr(message), id); } -template , typename = EnableIfMutableProto> +template , + typename = EnableIfMutableProto> absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::protos::internal::ExtensionIdentifier& id, const Extension& value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); @@ -371,11 +370,24 @@ absl::Status SetExtension( internal::GetInternalMsg(&value)); } -template , typename = EnableIfMutableProto> +template , + typename = EnableIfMutableProto> absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::protos::internal::ExtensionIdentifier& id, + Ptr value) { + static_assert(!std::is_const_v); + auto* message_arena = static_cast(message->GetInternalArena()); + return ::protos::internal::SetExtension(internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + internal::GetInternalMsg(value)); +} + +template , + typename = EnableIfMutableProto> +absl::Status SetExtension( + Ptr message, + const ::protos::internal::ExtensionIdentifier& id, Extension&& value) { Extension ext = std::move(value); static_assert(!std::is_const_v); @@ -386,25 +398,28 @@ absl::Status SetExtension( internal::GetInternalMsg(&ext), extension_arena); } -template > +template > absl::Status SetExtension( - T* message, - const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::protos::internal::ExtensionIdentifier& id, const Extension& value) { return ::protos::SetExtension(::protos::Ptr(message), id, value); } -template > +template > absl::Status SetExtension( - T* message, - const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::protos::internal::ExtensionIdentifier& id, Extension&& value) { return ::protos::SetExtension(::protos::Ptr(message), id, std::forward(value)); } +template > +absl::Status SetExtension( + T* message, const ::protos::internal::ExtensionIdentifier& id, + Ptr value) { + return ::protos::SetExtension(::protos::Ptr(message), id, value); +} + template > absl::StatusOr> GetExtension( diff --git a/protos_generator/tests/test_generated.cc b/protos_generator/tests/test_generated.cc index d745eb01cf..208a1be80a 100644 --- a/protos_generator/tests/test_generated.cc +++ b/protos_generator/tests/test_generated.cc @@ -5,6 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#include #include #include #include @@ -24,10 +25,13 @@ #include "protos_generator/tests/no_package.upb.proto.h" #include "protos_generator/tests/test_model.upb.proto.h" #include "upb/mem/arena.h" +#include "upb/mem/arena.hpp" namespace { using ::protos_generator::test::protos::ChildModel1; +using ::protos_generator::test::protos::container_ext; +using ::protos_generator::test::protos::ContainerExtension; using ::protos_generator::test::protos::other_ext; using ::protos_generator::test::protos::RED; using ::protos_generator::test::protos::TestEnum; @@ -39,7 +43,6 @@ using ::protos_generator::test::protos::TestModel_Category_VIDEO; using ::protos_generator::test::protos::theme; using ::protos_generator::test::protos::ThemeExtension; using ::testing::ElementsAre; -using ::testing::HasSubstr; // C++17 port of C++20 `requires` template @@ -442,7 +445,7 @@ TEST(CppGeneratedCode, RepeatedScalarIterator) { EXPECT_EQ(sum, 5 + 16 + 27); // Access by const reference. sum = 0; - for (const int& i : *test_model.mutable_value_array()) { + for (const auto& i : *test_model.mutable_value_array()) { sum += i; } EXPECT_EQ(sum, 5 + 16 + 27); @@ -551,7 +554,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForMessages) { } i = 0; - for (auto child : *test_model.mutable_child_models()) { + for (const auto& child : *test_model.mutable_child_models()) { if (i++ == 0) { EXPECT_EQ(child.child_str1(), kTestStr1); } else { @@ -725,6 +728,70 @@ TEST(CppGeneratedCode, SetExtension) { EXPECT_EQ(::protos::internal::GetInternalMsg(*ext), prior_message); } +TEST(CppGeneratedCode, SetExtensionWithPtr) { + ::protos::Arena arena_model; + ::protos::Ptr model = + ::protos::CreateMessage(arena_model); + void* prior_message; + { + // Use a nested scope to make sure the arenas are fused correctly. + ::protos::Arena arena; + ::protos::Ptr extension1 = + ::protos::CreateMessage(arena); + extension1->set_ext_name("Hello World"); + prior_message = ::protos::internal::GetInternalMsg(extension1); + EXPECT_EQ(false, ::protos::HasExtension(model, theme)); + auto res = ::protos::SetExtension(model, theme, extension1); + EXPECT_EQ(true, res.ok()); + } + EXPECT_EQ(true, ::protos::HasExtension(model, theme)); + auto ext = ::protos::GetExtension(model, theme); + EXPECT_TRUE(ext.ok()); + EXPECT_NE(::protos::internal::GetInternalMsg(*ext), prior_message); +} + +#ifndef _MSC_VER +TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { + ::protos::Arena arena; + ::protos::Ptr model = ::protos::CreateMessage(arena); + ThemeExtension extension1; + ContainerExtension extension2; + + const auto canSetExtension = [&](auto l) { + return Requires(l); + }; + EXPECT_TRUE(canSetExtension( + [](auto p) -> decltype(::protos::SetExtension(p, theme, extension1)) {})); + // Wrong extension value type should fail to compile. + EXPECT_TRUE(!canSetExtension( + [](auto p) -> decltype(::protos::SetExtension(p, theme, extension2)) {})); + // Wrong extension id with correct extension type should fail to compile. + EXPECT_TRUE( + !canSetExtension([](auto p) -> decltype(::protos::SetExtension( + p, container_ext, extension1)) {})); +} +#endif + +TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { + ::protos::Arena arena; + ::protos::Ptr model = ::protos::CreateMessage(arena); + void* prior_message; + { + // Use a nested scope to make sure the arenas are fused correctly. + ::protos::Ptr extension1 = + ::protos::CreateMessage(arena); + extension1->set_ext_name("Hello World"); + prior_message = ::protos::internal::GetInternalMsg(extension1); + EXPECT_EQ(false, ::protos::HasExtension(model, theme)); + auto res = ::protos::SetExtension(model, theme, extension1); + EXPECT_EQ(true, res.ok()); + } + EXPECT_EQ(true, ::protos::HasExtension(model, theme)); + auto ext = ::protos::GetExtension(model, theme); + EXPECT_TRUE(ext.ok()); + EXPECT_NE(::protos::internal::GetInternalMsg(*ext), prior_message); +} + TEST(CppGeneratedCode, SetExtensionFusingFailureShouldCopy) { // Use an initial block to disallow fusing. char initial_block[1000]; diff --git a/protos_generator/tests/test_model.proto b/protos_generator/tests/test_model.proto index 24b4406e95..34c875aed4 100644 --- a/protos_generator/tests/test_model.proto +++ b/protos_generator/tests/test_model.proto @@ -14,6 +14,8 @@ import "protos_generator/tests/child_model.proto"; message TestModelContainer { repeated TestModel models = 1; optional ChildModel3 proto_3_child = 2; + extensions 10000 to max + [verification = UNVERIFIED]; } message TestModel { @@ -138,6 +140,17 @@ extend TestModel { optional ThemeExtension theme = 12001; } +message ContainerExtension { + extend TestModelContainer { + optional ContainerExtension container_extension = 12004; + } + optional string ext_container_name = 1; +} + +extend TestModelContainer { + optional ContainerExtension container_ext = 12005; +} + message OtherExtension { optional string ext2_name = 1; } From c6e2778f9173ed925746197310f4d11e5f46df96 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Tue, 14 May 2024 12:17:28 -0700 Subject: [PATCH 173/179] Replace SmallSortedMap.EmptySet with equivalent Collections.emptySet() This reduces our code weight by a little (3 classes). Collections.emptySet also has a singleton empty iterator, so it doesn't allocate. PiperOrigin-RevId: 633667264 --- .../com/google/protobuf/SmallSortedMap.java | 44 +------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java b/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java index 17e8d77dba..55934d2187 100644 --- a/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java +++ b/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java @@ -14,7 +14,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -169,13 +168,13 @@ class SmallSortedMap, V> extends AbstractMap { /** @return An iterable over the overflow entries. */ public Iterable> getOverflowEntries() { return overflowEntries.isEmpty() - ? EmptySet.>iterable() + ? Collections.emptySet() : overflowEntries.entrySet(); } Iterable> getOverflowEntriesDescending() { return overflowEntriesDescending.isEmpty() - ? EmptySet.>iterable() + ? Collections.emptySet() : overflowEntriesDescending.entrySet(); } @@ -597,45 +596,6 @@ class SmallSortedMap, V> extends AbstractMap { } } - /** - * Helper class that holds immutable instances of an Iterable/Iterator that we return when the - * overflow entries is empty. This eliminates the creation of an Iterator object when there is - * nothing to iterate over. - */ - private static class EmptySet { - - private static final Iterator ITERATOR = - new Iterator() { - @Override - public boolean hasNext() { - return false; - } - - @Override - public Object next() { - throw new NoSuchElementException(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - - private static final Iterable ITERABLE = - new Iterable() { - @Override - public Iterator iterator() { - return ITERATOR; - } - }; - - @SuppressWarnings("unchecked") - static Iterable iterable() { - return (Iterable) ITERABLE; - } - } - @Override public boolean equals(Object o) { if (this == o) { From 5a9170701e874b69245dabbd428200cfa4e6f458 Mon Sep 17 00:00:00 2001 From: Jie Luo Date: Tue, 14 May 2024 12:23:37 -0700 Subject: [PATCH 174/179] internal change PiperOrigin-RevId: 633669104 --- python/pb_unit_tests/BUILD | 43 ------------------- .../descriptor_database_test_wrapper.py | 35 --------------- .../pb_unit_tests/json_format_test_wrapper.py | 35 --------------- python/pb_unit_tests/keywords_test_wrapper.py | 35 --------------- python/pb_unit_tests/numpy_test_wrapper.py | 36 ---------------- .../service_reflection_test_wrapper.py | 35 --------------- .../symbol_database_test_wrapper.py | 35 --------------- .../text_encoding_test_wrapper.py | 35 --------------- .../pb_unit_tests/text_format_test_wrapper.py | 35 --------------- .../unknown_fields_test_wrapper.py | 35 --------------- .../well_known_types_test_wrapper.py | 36 ---------------- .../pb_unit_tests/wire_format_test_wrapper.py | 35 --------------- 12 files changed, 430 deletions(-) delete mode 100644 python/pb_unit_tests/descriptor_database_test_wrapper.py delete mode 100644 python/pb_unit_tests/json_format_test_wrapper.py delete mode 100644 python/pb_unit_tests/keywords_test_wrapper.py delete mode 100644 python/pb_unit_tests/numpy_test_wrapper.py delete mode 100644 python/pb_unit_tests/service_reflection_test_wrapper.py delete mode 100644 python/pb_unit_tests/symbol_database_test_wrapper.py delete mode 100644 python/pb_unit_tests/text_encoding_test_wrapper.py delete mode 100644 python/pb_unit_tests/text_format_test_wrapper.py delete mode 100644 python/pb_unit_tests/unknown_fields_test_wrapper.py delete mode 100644 python/pb_unit_tests/well_known_types_test_wrapper.py delete mode 100644 python/pb_unit_tests/wire_format_test_wrapper.py diff --git a/python/pb_unit_tests/BUILD b/python/pb_unit_tests/BUILD index 74b01c8eef..4d6244b0ea 100644 --- a/python/pb_unit_tests/BUILD +++ b/python/pb_unit_tests/BUILD @@ -17,8 +17,6 @@ load(":pyproto_test_wrapper.bzl", "pyproto_test_wrapper") licenses(["notice"]) -pyproto_test_wrapper(name = "descriptor_database_test") - pyproto_test_wrapper(name = "descriptor_pool_test") pyproto_test_wrapper(name = "descriptor_test") @@ -27,55 +25,14 @@ pyproto_test_wrapper(name = "descriptor_test") pyproto_test_wrapper(name = "generator_test") # end:github_only -pyproto_test_wrapper(name = "json_format_test") - -pyproto_test_wrapper(name = "keywords_test") - pyproto_test_wrapper(name = "message_factory_test") -# begin:github_only -# This target has different dependencies and fails when using the wrapper -# TODO: Move this to using pyproto_test_wrapper -py_test( - name = "numpy_test", - srcs = ["numpy_test_wrapper.py"], - main = "numpy_test_wrapper.py", - target_compatible_with = select({ - "@system_python//:supported": [], - "//conditions:default": ["@platforms//:incompatible"], - }), - deps = [ - requirement("numpy"), - "//python:_message", - "//python/google/protobuf/internal/numpy:numpy_test", - ], -) -# end:github_only - -# begin:google_only -# pyproto_test_wrapper(name = "numpy_test") -# end:google_only - pyproto_test_wrapper(name = "proto_builder_test") -pyproto_test_wrapper(name = "service_reflection_test") - -pyproto_test_wrapper(name = "symbol_database_test") - -pyproto_test_wrapper(name = "text_encoding_test") - pyproto_test_wrapper(name = "message_test") pyproto_test_wrapper(name = "reflection_test") -pyproto_test_wrapper(name = "text_format_test") - -pyproto_test_wrapper(name = "unknown_fields_test") - -pyproto_test_wrapper(name = "well_known_types_test") - -pyproto_test_wrapper(name = "wire_format_test") - filegroup( name = "test_files", srcs = glob(["*.py"]), diff --git a/python/pb_unit_tests/descriptor_database_test_wrapper.py b/python/pb_unit_tests/descriptor_database_test_wrapper.py deleted file mode 100644 index 2e6081fc7c..0000000000 --- a/python/pb_unit_tests/descriptor_database_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.descriptor_database_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/json_format_test_wrapper.py b/python/pb_unit_tests/json_format_test_wrapper.py deleted file mode 100644 index 27d855cd48..0000000000 --- a/python/pb_unit_tests/json_format_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.json_format_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/keywords_test_wrapper.py b/python/pb_unit_tests/keywords_test_wrapper.py deleted file mode 100644 index d9401785d0..0000000000 --- a/python/pb_unit_tests/keywords_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.keywords_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/numpy_test_wrapper.py b/python/pb_unit_tests/numpy_test_wrapper.py deleted file mode 100644 index 62089e9cda..0000000000 --- a/python/pb_unit_tests/numpy_test_wrapper.py +++ /dev/null @@ -1,36 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest - -from google.protobuf.internal.numpy.numpy_test import * - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/service_reflection_test_wrapper.py b/python/pb_unit_tests/service_reflection_test_wrapper.py deleted file mode 100644 index bc0345c217..0000000000 --- a/python/pb_unit_tests/service_reflection_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.service_reflection_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/symbol_database_test_wrapper.py b/python/pb_unit_tests/symbol_database_test_wrapper.py deleted file mode 100644 index 16ea9656ba..0000000000 --- a/python/pb_unit_tests/symbol_database_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.symbol_database_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/text_encoding_test_wrapper.py b/python/pb_unit_tests/text_encoding_test_wrapper.py deleted file mode 100644 index 3eb8153baf..0000000000 --- a/python/pb_unit_tests/text_encoding_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.text_encoding_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/text_format_test_wrapper.py b/python/pb_unit_tests/text_format_test_wrapper.py deleted file mode 100644 index 535561d83c..0000000000 --- a/python/pb_unit_tests/text_format_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.text_format_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/unknown_fields_test_wrapper.py b/python/pb_unit_tests/unknown_fields_test_wrapper.py deleted file mode 100644 index 1807f6d1ff..0000000000 --- a/python/pb_unit_tests/unknown_fields_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.unknown_fields_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/well_known_types_test_wrapper.py b/python/pb_unit_tests/well_known_types_test_wrapper.py deleted file mode 100644 index 5006332489..0000000000 --- a/python/pb_unit_tests/well_known_types_test_wrapper.py +++ /dev/null @@ -1,36 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.well_known_types_test import * -import os -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/python/pb_unit_tests/wire_format_test_wrapper.py b/python/pb_unit_tests/wire_format_test_wrapper.py deleted file mode 100644 index 3b13a2b4b7..0000000000 --- a/python/pb_unit_tests/wire_format_test_wrapper.py +++ /dev/null @@ -1,35 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from google.protobuf.internal.wire_format_test import * -import unittest - -if __name__ == '__main__': - unittest.main(verbosity=2) From bb68eb22dde9fbefc0413b7c3f836b03d062df4c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 May 2024 12:36:44 -0700 Subject: [PATCH 175/179] Implement new Debug API with redaction. Implement emittingSingleLine TextFormat printer option. PiperOrigin-RevId: 633672722 --- java/core/BUILD.bazel | 1 + .../java/com/google/protobuf/DebugFormat.java | 79 +++++++++ .../java/com/google/protobuf/TextFormat.java | 146 +++++++++++++--- .../com/google/protobuf/DebugFormatTest.java | 158 ++++++++++++++++++ src/google/protobuf/unittest.proto | 6 + 5 files changed, 371 insertions(+), 19 deletions(-) create mode 100644 java/core/src/main/java/com/google/protobuf/DebugFormat.java create mode 100644 java/core/src/test/java/com/google/protobuf/DebugFormatTest.java diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 41802133ed..bc2b592589 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -523,6 +523,7 @@ LITE_TEST_EXCLUSIONS = [ "src/test/java/com/google/protobuf/AnyTest.java", "src/test/java/com/google/protobuf/CodedInputStreamTest.java", "src/test/java/com/google/protobuf/DeprecatedFieldTest.java", + "src/test/java/com/google/protobuf/DebugFormatTest.java", "src/test/java/com/google/protobuf/DescriptorsTest.java", "src/test/java/com/google/protobuf/DiscardUnknownFieldsTest.java", "src/test/java/com/google/protobuf/DynamicMessageTest.java", diff --git a/java/core/src/main/java/com/google/protobuf/DebugFormat.java b/java/core/src/main/java/com/google/protobuf/DebugFormat.java new file mode 100644 index 0000000000..2d63ef1719 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/DebugFormat.java @@ -0,0 +1,79 @@ +package com.google.protobuf; + +import com.google.protobuf.Descriptors.FieldDescriptor; + +/** + * Provides an explicit API for unstable, redacting debug output suitable for debug logging. This + * implementation is based on TextFormat, but should not be parsed. + */ +public final class DebugFormat { + + private final boolean isSingleLine; + + private DebugFormat(boolean singleLine) { + isSingleLine = singleLine; + } + + public static DebugFormat singleLine() { + return new DebugFormat(true); + } + + public static DebugFormat multiline() { + return new DebugFormat(false); + } + + public String toString(MessageOrBuilder message) { + return TextFormat.printer() + .emittingSingleLine(this.isSingleLine) + .enablingSafeDebugFormat(true) + .printToString(message); + } + + public String toString(FieldDescriptor field, Object value) { + return TextFormat.printer() + .emittingSingleLine(this.isSingleLine) + .enablingSafeDebugFormat(true) + .printFieldToString(field, value); + } + + public String toString(UnknownFieldSet fields) { + return TextFormat.printer() + .emittingSingleLine(this.isSingleLine) + .enablingSafeDebugFormat(true) + .printToString(fields); + } + + public Object lazyToString(MessageOrBuilder message) { + return new LazyDebugOutput(message, this); + } + + public Object lazyToString(UnknownFieldSet fields) { + return new LazyDebugOutput(fields, this); + } + + private static class LazyDebugOutput { + private final MessageOrBuilder message; + private final UnknownFieldSet fields; + private final DebugFormat format; + + LazyDebugOutput(MessageOrBuilder message, DebugFormat format) { + this.message = message; + this.fields = null; + this.format = format; + } + + LazyDebugOutput(UnknownFieldSet fields, DebugFormat format) { + this.message = null; + this.fields = fields; + this.format = format; + } + + @Override + public String toString() { + if (message != null) { + return format.toString(message); + } + return format.toString(fields); + } + } +} diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 71b2b33f67..8e5f2a0d87 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -38,6 +38,8 @@ public final class TextFormat { private static final String DEBUG_STRING_SILENT_MARKER = "\t "; + private static final String REDACTED_MARKER = "[REDACTED]"; + /** * Generates a human readable form of this message, useful for debugging and other purposes, with * no newline characters. This is just a trivial wrapper around {@link @@ -58,7 +60,7 @@ public final class TextFormat { */ public static void printUnknownFieldValue( final int tag, final Object value, final Appendable output) throws IOException { - printUnknownFieldValue(tag, value, multiLineOutput(output)); + printUnknownFieldValue(tag, value, setSingleLineOutput(output, false)); } private static void printUnknownFieldValue( @@ -109,7 +111,11 @@ public final class TextFormat { // Printer instance which escapes non-ASCII characters. private static final Printer DEFAULT = new Printer( - true, TypeRegistry.getEmptyTypeRegistry(), ExtensionRegistryLite.getEmptyRegistry()); + true, + TypeRegistry.getEmptyTypeRegistry(), + ExtensionRegistryLite.getEmptyRegistry(), + false, + false); /** Whether to escape non ASCII characters with backslash and octal. */ private final boolean escapeNonAscii; @@ -117,13 +123,25 @@ public final class TextFormat { private final TypeRegistry typeRegistry; private final ExtensionRegistryLite extensionRegistry; + /** + * Whether to enable redaction of sensitive fields and introduce randomization. Note that when + * this is enabled, the output will no longer be deserializable. + */ + private final boolean enablingSafeDebugFormat; + + private final boolean singleLine; + private Printer( boolean escapeNonAscii, TypeRegistry typeRegistry, - ExtensionRegistryLite extensionRegistry) { + ExtensionRegistryLite extensionRegistry, + boolean enablingSafeDebugFormat, + boolean singleLine) { this.escapeNonAscii = escapeNonAscii; this.typeRegistry = typeRegistry; this.extensionRegistry = extensionRegistry; + this.enablingSafeDebugFormat = enablingSafeDebugFormat; + this.singleLine = singleLine; } /** @@ -136,7 +154,8 @@ public final class TextFormat { * with the escape mode set to the given parameter. */ public Printer escapingNonAscii(boolean escapeNonAscii) { - return new Printer(escapeNonAscii, typeRegistry, extensionRegistry); + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); } /** @@ -149,7 +168,8 @@ public final class TextFormat { if (this.typeRegistry != TypeRegistry.getEmptyTypeRegistry()) { throw new IllegalArgumentException("Only one typeRegistry is allowed."); } - return new Printer(escapeNonAscii, typeRegistry, extensionRegistry); + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); } /** @@ -162,7 +182,34 @@ public final class TextFormat { if (this.extensionRegistry != ExtensionRegistryLite.getEmptyRegistry()) { throw new IllegalArgumentException("Only one extensionRegistry is allowed."); } - return new Printer(escapeNonAscii, typeRegistry, extensionRegistry); + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); + } + + /** + * Return a new Printer instance that outputs a redacted and unstable format suitable for + * debugging. + * + * @param enablingSafeDebugFormat If true, the new Printer will redact all proto fields that are + * marked by a debug_redact=true option, and apply an unstable prefix to the output. + * @return a new Printer that clones all other configurations from the current {@link Printer}, + * with the enablingSafeDebugFormat mode set to the given parameter. + */ + Printer enablingSafeDebugFormat(boolean enablingSafeDebugFormat) { + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); + } + + /** + * Return a new Printer instance with the specified line formatting status. + * + * @param singleLine If true, the new Printer will output no newline characters. + * @return a new Printer that clones all other configurations from the current {@link Printer}, + * with the singleLine mode set to the given parameter. + */ + public Printer emittingSingleLine(boolean singleLine) { + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); } /** @@ -171,12 +218,12 @@ public final class TextFormat { * original Protocol Buffer system) */ public void print(final MessageOrBuilder message, final Appendable output) throws IOException { - print(message, multiLineOutput(output)); + print(message, setSingleLineOutput(output, this.singleLine)); } /** Outputs a textual representation of {@code fields} to {@code output}. */ public void print(final UnknownFieldSet fields, final Appendable output) throws IOException { - printUnknownFields(fields, multiLineOutput(output)); + printUnknownFields(fields, setSingleLineOutput(output, this.singleLine)); } private void print(final MessageOrBuilder message, final TextGenerator generator) @@ -188,6 +235,14 @@ public final class TextFormat { printMessage(message, generator); } + private void applyUnstablePrefix(final Appendable output) { + try { + output.append(""); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + /** * Attempt to print the 'google.protobuf.Any' message in a human-friendly format. Returns false * if the message isn't a valid 'google.protobuf.Any' message (in which case the message should @@ -244,6 +299,9 @@ public final class TextFormat { public String printFieldToString(final FieldDescriptor field, final Object value) { try { final StringBuilder text = new StringBuilder(); + if (enablingSafeDebugFormat) { + applyUnstablePrefix(text); + } printField(field, value, text); return text.toString(); } catch (IOException e) { @@ -253,7 +311,7 @@ public final class TextFormat { public void printField(final FieldDescriptor field, final Object value, final Appendable output) throws IOException { - printField(field, value, multiLineOutput(output)); + printField(field, value, setSingleLineOutput(output, this.singleLine)); } private void printField( @@ -358,12 +416,19 @@ public final class TextFormat { public void printFieldValue( final FieldDescriptor field, final Object value, final Appendable output) throws IOException { - printFieldValue(field, value, multiLineOutput(output)); + printFieldValue(field, value, setSingleLineOutput(output, this.singleLine)); } private void printFieldValue( final FieldDescriptor field, final Object value, final TextGenerator generator) throws IOException { + if (shouldRedact(field)) { + generator.print(REDACTED_MARKER); + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + generator.eol(); + } + return; + } switch (field.getType()) { case INT32: case SINT32: @@ -429,10 +494,54 @@ public final class TextFormat { } } + private boolean shouldRedactOptionValue(EnumValueDescriptor optionValue) { + if (optionValue.getOptions().hasDebugRedact()) { + return optionValue.getOptions().getDebugRedact(); + } + return false; + } + + // The criteria for redacting a field is as follows: 1) The enablingSafeDebugFormat printer + // option + // must be on. 2) The field must be marked by a debug_redact=true option, or is marked by an + // option with an enum value that is marked by a debug_redact=true option. + private boolean shouldRedact(final FieldDescriptor field) { + if (!this.enablingSafeDebugFormat) { + return false; + } + if (field.getOptions().hasDebugRedact()) { + return field.getOptions().getDebugRedact(); + } + // Iterate through every option; if it's an enum, we check each enum value for debug_redact. + for (Map.Entry entry : + field.getOptions().getAllFields().entrySet()) { + Descriptors.FieldDescriptor option = entry.getKey(); + if (option.getType() != Descriptors.FieldDescriptor.Type.ENUM) { + continue; + } + if (option.isRepeated()) { + for (EnumValueDescriptor value : (List) entry.getValue()) { + if (shouldRedactOptionValue(value)) { + return true; + } + } + } else { + EnumValueDescriptor optionValue = (EnumValueDescriptor) entry.getValue(); + if (shouldRedactOptionValue(optionValue)) { + return true; + } + } + } + return false; + } + /** Like {@code print()}, but writes directly to a {@code String} and returns it. */ public String printToString(final MessageOrBuilder message) { try { final StringBuilder text = new StringBuilder(); + if (enablingSafeDebugFormat) { + applyUnstablePrefix(text); + } print(message, text); return text.toString(); } catch (IOException e) { @@ -443,6 +552,9 @@ public final class TextFormat { public String printToString(final UnknownFieldSet fields) { try { final StringBuilder text = new StringBuilder(); + if (enablingSafeDebugFormat) { + applyUnstablePrefix(text); + } print(fields, text); return text.toString(); } catch (IOException e) { @@ -457,7 +569,7 @@ public final class TextFormat { public String shortDebugString(final MessageOrBuilder message) { try { final StringBuilder text = new StringBuilder(); - print(message, singleLineOutput(text)); + print(message, setSingleLineOutput(text, true)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -471,7 +583,7 @@ public final class TextFormat { public String shortDebugString(final FieldDescriptor field, final Object value) { try { final StringBuilder text = new StringBuilder(); - printField(field, value, singleLineOutput(text)); + printField(field, value, setSingleLineOutput(text, true)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -485,7 +597,7 @@ public final class TextFormat { public String shortDebugString(final UnknownFieldSet fields) { try { final StringBuilder text = new StringBuilder(); - printUnknownFields(fields, singleLineOutput(text)); + printUnknownFields(fields, setSingleLineOutput(text, true)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -640,12 +752,8 @@ public final class TextFormat { } } - private static TextGenerator multiLineOutput(Appendable output) { - return new TextGenerator(output, false); - } - - private static TextGenerator singleLineOutput(Appendable output) { - return new TextGenerator(output, true); + private static TextGenerator setSingleLineOutput(Appendable output, boolean singleLine) { + return new TextGenerator(output, singleLine); } /** An inner class for writing text to the output stream. */ diff --git a/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java b/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java new file mode 100644 index 0000000000..3f20ecf6f0 --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java @@ -0,0 +1,158 @@ +package com.google.protobuf; + +import static com.google.common.truth.Truth.assertThat; +import static protobuf_unittest.UnittestProto.redactedExtension; + +import com.google.protobuf.Descriptors.FieldDescriptor; +import protobuf_unittest.UnittestProto.RedactedFields; +import protobuf_unittest.UnittestProto.TestNestedMessageRedaction; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class DebugFormatTest { + + private static final String REDACTED_REGEX = "\\[REDACTED\\]"; + private static final String UNSTABLE_PREFIX_SINGLE_LINE = getUnstablePrefix(true); + private static final String UNSTABLE_PREFIX_MULTILINE = getUnstablePrefix(false); + + private static String getUnstablePrefix(boolean singleLine) { + return ""; + } + + @Test + public void multilineMessageFormat_returnsMultiline() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + assertThat(result) + .matches( + String.format("%soptional_unredacted_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE)); + } + + @Test + public void singleLineMessageFormat_returnsSingleLine() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + String result = DebugFormat.singleLine().toString(message); + assertThat(result) + .matches( + String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE)); + } + + @Test + public void messageFormat_debugRedactFieldIsRedacted() { + RedactedFields message = RedactedFields.newBuilder().setOptionalRedactedString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%soptional_redacted_string: %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_debugRedactMessageIsRedacted() { + RedactedFields message = + RedactedFields.newBuilder() + .setOptionalRedactedMessage( + TestNestedMessageRedaction.newBuilder().setOptionalUnredactedNestedString("foo")) + .build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%soptional_redacted_message \\{\n %s\n\\}\n", + UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_debugRedactMapIsRedacted() { + RedactedFields message = RedactedFields.newBuilder().putMapRedactedString("foo", "bar").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%smap_redacted_string \\{\\n %s\n\\}\n", + UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_debugRedactExtensionIsRedacted() { + RedactedFields message = + RedactedFields.newBuilder().setExtension(redactedExtension, "foo").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%s\\[protobuf_unittest\\.redacted_extension\\]: %s\n", + UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_redactFalseIsNotRedacted() { + RedactedFields message = + RedactedFields.newBuilder().setOptionalRedactedFalseString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + assertThat(result) + .matches( + String.format( + "%soptional_redacted_false_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE)); + } + + @Test + public void messageFormat_nonSensitiveFieldIsNotRedacted() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format("%soptional_unredacted_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE)); + } + + @Test + public void descriptorDebugFormat_returnsExpectedFormat() { + FieldDescriptor field = + RedactedFields.getDescriptor().findFieldByName("optional_redacted_string"); + String result = DebugFormat.multiline().toString(field, "foo"); + assertThat(result) + .matches( + String.format( + "%soptional_redacted_string: %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void unstableFormat_isStablePerProcess() { + RedactedFields message1 = + RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + RedactedFields message2 = + RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + for (int i = 0; i < 5; i++) { + String result1 = DebugFormat.multiline().toString(message1); + String result2 = DebugFormat.multiline().toString(message2); + assertThat(result1).isEqualTo(result2); + } + } + + @Test + public void lazyDebugFormatMessage_supportsImplicitFormatting() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + Object lazyDebug = DebugFormat.singleLine().lazyToString(message); + + assertThat(String.format("%s", lazyDebug)) + .matches( + String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE)); + } + +} diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 7c65fb594f..cb46956d31 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -1738,6 +1738,12 @@ message RedactedFields { repeated TestNestedMessageRedaction repeated_unredacted_message = 8; map map_redacted_string = 9 [debug_redact = true]; map map_unredacted_string = 10; + optional string optional_redacted_false_string = 11 [debug_redact = false]; + extensions 20 to 30; +} + +extend RedactedFields { + optional string redacted_extension = 20 [debug_redact = true]; } message TestCord{ From dde03553c92867184ff5d351b7f087c052f39459 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 May 2024 13:01:17 -0700 Subject: [PATCH 176/179] Internal change PiperOrigin-RevId: 633679764 --- .../java/com/google/protobuf/TextFormat.java | 67 +++++++++++++------ .../com/google/protobuf/DebugFormatTest.java | 52 ++++++++++++++ 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 8e5f2a0d87..bb1237795d 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -60,11 +60,12 @@ public final class TextFormat { */ public static void printUnknownFieldValue( final int tag, final Object value, final Appendable output) throws IOException { - printUnknownFieldValue(tag, value, setSingleLineOutput(output, false)); + printUnknownFieldValue(tag, value, setSingleLineOutput(output, false), false); } private static void printUnknownFieldValue( - final int tag, final Object value, final TextGenerator generator) throws IOException { + final int tag, final Object value, final TextGenerator generator, boolean redact) + throws IOException { switch (WireFormat.getTagWireType(tag)) { case WireFormat.WIRETYPE_VARINT: generator.print(unsignedToString((Long) value)); @@ -82,7 +83,7 @@ public final class TextFormat { generator.print("{"); generator.eol(); generator.indent(); - Printer.printUnknownFields(message, generator); + Printer.printUnknownFields(message, generator, redact); generator.outdent(); generator.print("}"); } catch (InvalidProtocolBufferException e) { @@ -93,7 +94,7 @@ public final class TextFormat { } break; case WireFormat.WIRETYPE_START_GROUP: - Printer.printUnknownFields((UnknownFieldSet) value, generator); + Printer.printUnknownFields((UnknownFieldSet) value, generator, redact); break; default: throw new IllegalArgumentException("Bad tag: " + tag); @@ -223,7 +224,8 @@ public final class TextFormat { /** Outputs a textual representation of {@code fields} to {@code output}. */ public void print(final UnknownFieldSet fields, final Appendable output) throws IOException { - printUnknownFields(fields, setSingleLineOutput(output, this.singleLine)); + printUnknownFields( + fields, setSingleLineOutput(output, this.singleLine), this.enablingSafeDebugFormat); } private void print(final MessageOrBuilder message, final TextGenerator generator) @@ -597,7 +599,7 @@ public final class TextFormat { public String shortDebugString(final UnknownFieldSet fields) { try { final StringBuilder text = new StringBuilder(); - printUnknownFields(fields, setSingleLineOutput(text, true)); + printUnknownFields(fields, setSingleLineOutput(text, true), this.enablingSafeDebugFormat); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -605,16 +607,26 @@ public final class TextFormat { } private static void printUnknownFieldValue( - final int tag, final Object value, final TextGenerator generator) throws IOException { + final int tag, final Object value, final TextGenerator generator, boolean redact) + throws IOException { switch (WireFormat.getTagWireType(tag)) { case WireFormat.WIRETYPE_VARINT: - generator.print(unsignedToString((Long) value)); + generator.print( + redact + ? String.format("UNKNOWN_VARINT %s", REDACTED_MARKER) + : unsignedToString((Long) value)); break; case WireFormat.WIRETYPE_FIXED32: - generator.print(String.format((Locale) null, "0x%08x", (Integer) value)); + generator.print( + redact + ? String.format("UNKNOWN_FIXED32 %s", REDACTED_MARKER) + : String.format((Locale) null, "0x%08x", (Integer) value)); break; case WireFormat.WIRETYPE_FIXED64: - generator.print(String.format((Locale) null, "0x%016x", (Long) value)); + generator.print( + redact + ? String.format("UNKNOWN_FIXED64 %s", REDACTED_MARKER) + : String.format((Locale) null, "0x%016x", (Long) value)); break; case WireFormat.WIRETYPE_LENGTH_DELIMITED: try { @@ -623,18 +635,22 @@ public final class TextFormat { generator.print("{"); generator.eol(); generator.indent(); - printUnknownFields(message, generator); + printUnknownFields(message, generator, redact); generator.outdent(); generator.print("}"); } catch (InvalidProtocolBufferException e) { // If not parseable as a message, print as a String + if (redact) { + generator.print(String.format("UNKNOWN_STRING %s", REDACTED_MARKER)); + break; + } generator.print("\""); generator.print(escapeBytes((ByteString) value)); generator.print("\""); } break; case WireFormat.WIRETYPE_START_GROUP: - printUnknownFields((UnknownFieldSet) value, generator); + printUnknownFields((UnknownFieldSet) value, generator, redact); break; default: throw new IllegalArgumentException("Bad tag: " + tag); @@ -646,7 +662,7 @@ public final class TextFormat { for (Map.Entry field : message.getAllFields().entrySet()) { printField(field.getKey(), field.getValue(), generator); } - printUnknownFields(message.getUnknownFields(), generator); + printUnknownFields(message.getUnknownFields(), generator, this.enablingSafeDebugFormat); } private void printSingleField( @@ -692,27 +708,32 @@ public final class TextFormat { } private static void printUnknownFields( - final UnknownFieldSet unknownFields, final TextGenerator generator) throws IOException { + final UnknownFieldSet unknownFields, final TextGenerator generator, boolean redact) + throws IOException { if (unknownFields.isEmpty()) { return; } for (Map.Entry entry : unknownFields.asMap().entrySet()) { final int number = entry.getKey(); final UnknownFieldSet.Field field = entry.getValue(); - printUnknownField(number, WireFormat.WIRETYPE_VARINT, field.getVarintList(), generator); - printUnknownField(number, WireFormat.WIRETYPE_FIXED32, field.getFixed32List(), generator); - printUnknownField(number, WireFormat.WIRETYPE_FIXED64, field.getFixed64List(), generator); + printUnknownField( + number, WireFormat.WIRETYPE_VARINT, field.getVarintList(), generator, redact); + printUnknownField( + number, WireFormat.WIRETYPE_FIXED32, field.getFixed32List(), generator, redact); + printUnknownField( + number, WireFormat.WIRETYPE_FIXED64, field.getFixed64List(), generator, redact); printUnknownField( number, WireFormat.WIRETYPE_LENGTH_DELIMITED, field.getLengthDelimitedList(), - generator); + generator, + redact); for (final UnknownFieldSet value : field.getGroupList()) { generator.print(entry.getKey().toString()); generator.print(" {"); generator.eol(); generator.indent(); - printUnknownFields(value, generator); + printUnknownFields(value, generator, redact); generator.outdent(); generator.print("}"); generator.eol(); @@ -721,12 +742,16 @@ public final class TextFormat { } private static void printUnknownField( - final int number, final int wireType, final List values, final TextGenerator generator) + final int number, + final int wireType, + final List values, + final TextGenerator generator, + boolean redact) throws IOException { for (final Object value : values) { generator.print(String.valueOf(number)); generator.print(": "); - printUnknownFieldValue(wireType, value, generator); + printUnknownFieldValue(wireType, value, generator, redact); generator.eol(); } } diff --git a/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java b/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java index 3f20ecf6f0..daf2924d98 100644 --- a/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java @@ -5,6 +5,7 @@ import static protobuf_unittest.UnittestProto.redactedExtension; import com.google.protobuf.Descriptors.FieldDescriptor; import protobuf_unittest.UnittestProto.RedactedFields; +import protobuf_unittest.UnittestProto.TestEmptyMessage; import protobuf_unittest.UnittestProto.TestNestedMessageRedaction; import org.junit.Test; import org.junit.runner.RunWith; @@ -155,4 +156,55 @@ public final class DebugFormatTest { String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE)); } + private UnknownFieldSet makeUnknownFieldSet() { + return UnknownFieldSet.newBuilder() + .addField( + 5, + UnknownFieldSet.Field.newBuilder() + .addVarint(1) + .addFixed32(2) + .addFixed64(3) + .addLengthDelimited(ByteString.copyFromUtf8("4")) + .addLengthDelimited( + UnknownFieldSet.newBuilder() + .addField(12, UnknownFieldSet.Field.newBuilder().addVarint(6).build()) + .build() + .toByteString()) + .addGroup( + UnknownFieldSet.newBuilder() + .addField(10, UnknownFieldSet.Field.newBuilder().addVarint(5).build()) + .build()) + .build()) + .addField( + 8, UnknownFieldSet.Field.newBuilder().addVarint(1).addVarint(2).addVarint(3).build()) + .addField( + 15, + UnknownFieldSet.Field.newBuilder() + .addVarint(0xABCDEF1234567890L) + .addFixed32(0xABCD1234) + .addFixed64(0xABCDEF1234567890L) + .build()) + .build(); + } + + @Test + public void unknownFieldsDebugFormat_returnsExpectedFormat() { + TestEmptyMessage unknownFields = + TestEmptyMessage.newBuilder().setUnknownFields(makeUnknownFieldSet()).build(); + + assertThat(DebugFormat.multiline().toString(unknownFields)) + .matches( + String.format("%s5: UNKNOWN_VARINT %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX) + + String.format("5: UNKNOWN_FIXED32 %s\n", REDACTED_REGEX) + + String.format("5: UNKNOWN_FIXED64 %s\n", REDACTED_REGEX) + + String.format("5: UNKNOWN_STRING %s\n", REDACTED_REGEX) + + String.format("5: \\{\n 12: UNKNOWN_VARINT %s\n\\}\n", REDACTED_REGEX) + + String.format("5 \\{\n 10: UNKNOWN_VARINT %s\n\\}\n", REDACTED_REGEX) + + String.format("8: UNKNOWN_VARINT %s\n", REDACTED_REGEX) + + String.format("8: UNKNOWN_VARINT %s\n", REDACTED_REGEX) + + String.format("8: UNKNOWN_VARINT %s\n", REDACTED_REGEX) + + String.format("15: UNKNOWN_VARINT %s\n", REDACTED_REGEX) + + String.format("15: UNKNOWN_FIXED32 %s\n", REDACTED_REGEX) + + String.format("15: UNKNOWN_FIXED64 %s\n", REDACTED_REGEX)); + } } From 9b463ac0483881d508a199d9853657e01c98c424 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 May 2024 15:09:08 -0700 Subject: [PATCH 177/179] Make the underlying type of the enum by 8-bits instead of using bitfields for it. It silences a warning in gcc 8/9. PiperOrigin-RevId: 633719795 --- src/google/protobuf/descriptor.h | 2 +- src/google/protobuf/generated_message_tctable_gen.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 48d79022e2..e4898950b2 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -2893,7 +2893,7 @@ typename FieldOpts::CType EffectiveStringCType(const FieldDesc* field) { } #ifndef SWIG -enum class Utf8CheckMode { +enum class Utf8CheckMode : uint8_t { kStrict = 0, // Parsing will fail if non UTF-8 data is in string fields. kVerify = 1, // Only log an error but parsing will succeed. kNone = 2, // No UTF-8 check. diff --git a/src/google/protobuf/generated_message_tctable_gen.h b/src/google/protobuf/generated_message_tctable_gen.h index 7dbcd8e1b3..a42de2b479 100644 --- a/src/google/protobuf/generated_message_tctable_gen.h +++ b/src/google/protobuf/generated_message_tctable_gen.h @@ -96,7 +96,7 @@ struct PROTOBUF_EXPORT TailCallTableInfo { uint16_t type_card; // For internal caching. - cpp::Utf8CheckMode utf8_check_mode : 8; + cpp::Utf8CheckMode utf8_check_mode; }; std::vector field_entries; From df630fa6f15c4b6581d6e8c24b76be420c8367ef Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Tue, 14 May 2024 15:41:40 -0700 Subject: [PATCH 178/179] MODULE.bazel fixes for protobuf BCR release. - Specifies bazel version for presubmits - Set C++14 version (default is c++11 otherwise, which is unsupported) - Update MODULE.bazel version + updater since publish-to-bcr can't handle constants and adds a duplicate version number PiperOrigin-RevId: 633729225 --- .bcr/presubmit.yml | 10 ++++++++++ MODULE.bazel | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.bcr/presubmit.yml b/.bcr/presubmit.yml index a8ed870656..4e1cdff44d 100644 --- a/.bcr/presubmit.yml +++ b/.bcr/presubmit.yml @@ -1,10 +1,15 @@ matrix: platform: ["debian10", "macos", "ubuntu2004", "windows"] + bazel: [6.x, 7.x] tasks: verify_targets: name: "Verify build targets" platform: ${{ platform }} + bazel: ${{ bazel }} + build_flags: + - '--host_cxxopt=-std=c++14' + - '--cxxopt=-std=c++14' build_targets: - '@protobuf//:protobuf' - '@protobuf//:protobuf_lite' @@ -17,9 +22,14 @@ bcr_test_module: matrix: platform: ["debian10", "macos", "ubuntu2004", "windows"] + bazel: [6.x, 7.x] tasks: run_test_module: name: "Run test module" platform: ${{ platform }} + bazel: ${{ bazel }} + build_flags: + - '--host_cxxopt=-std=c++14' + - '--cxxopt=-std=c++14' build_targets: - "//..." diff --git a/MODULE.bazel b/MODULE.bazel index a389538d3c..b91ab82941 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,10 +1,9 @@ # TODO: migrate all dependencies from WORKSPACE to MODULE.bazel # https://github.com/protocolbuffers/protobuf/issues/14313 -PROTOBUF_VERSION = "28.0-dev" module( name = "protobuf", - version = PROTOBUF_VERSION, + version = "28.0-dev", # Automatically updated on release compatibility_level = 1, repo_name = "com_google_protobuf", ) From 6b9a81a0429276e24195064d8733117a25a87148 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 14 May 2024 17:47:45 -0700 Subject: [PATCH 179/179] Add extension declarations for known features. This will prevent users from accidentally overriding these with different types (e.g. https://github.com/protocolbuffers/protobuf/issues/16757 and https://github.com/protocolbuffers/protobuf/issues/16756). PiperOrigin-RevId: 633760581 --- src/google/protobuf/descriptor.pb.cc | 74 ++++++++++++++-------------- src/google/protobuf/descriptor.proto | 23 +++++++-- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index ba06a6be83..f5985f0056 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -1897,7 +1897,7 @@ const char descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto[] AB "ive_int_value\030\005 \001(\003\022\024\n\014double_value\030\006 \001(" "\001\022\024\n\014string_value\030\007 \001(\014\022\027\n\017aggregate_val" "ue\030\010 \001(\t\0323\n\010NamePart\022\021\n\tname_part\030\001 \002(\t\022" - "\024\n\014is_extension\030\002 \002(\010\"\333\t\n\nFeatureSet\022\202\001\n" + "\024\n\014is_extension\030\002 \002(\010\"\303\t\n\nFeatureSet\022\202\001\n" "\016field_presence\030\001 \001(\0162).google.protobuf." "FeatureSet.FieldPresenceB\?\210\001\001\230\001\004\230\001\001\242\001\r\022\010" "EXPLICIT\030\346\007\242\001\r\022\010IMPLICIT\030\347\007\242\001\r\022\010EXPLICIT" @@ -1927,47 +1927,47 @@ const char descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto[] AB "GE_ENCODING_UNKNOWN\020\000\022\023\n\017LENGTH_PREFIXED" "\020\001\022\r\n\tDELIMITED\020\002\"H\n\nJsonFormat\022\027\n\023JSON_" "FORMAT_UNKNOWN\020\000\022\t\n\005ALLOW\020\001\022\026\n\022LEGACY_BE" - "ST_EFFORT\020\002*\006\010\350\007\020\351\007*\006\010\351\007\020\352\007*\006\010\352\007\020\353\007*\006\010\206N" - "\020\207N*\006\010\213N\020\220N*\006\010\220N\020\221NJ\006\010\347\007\020\350\007\"\202\003\n\022FeatureS" - "etDefaults\022N\n\010defaults\030\001 \003(\0132<.google.pr" - "otobuf.FeatureSetDefaults.FeatureSetEdit" - "ionDefault\0221\n\017minimum_edition\030\004 \001(\0162\030.go" - "ogle.protobuf.Edition\0221\n\017maximum_edition" - "\030\005 \001(\0162\030.google.protobuf.Edition\032\265\001\n\030Fea" - "tureSetEditionDefault\022)\n\007edition\030\003 \001(\0162\030" - ".google.protobuf.Edition\0229\n\024overridable_" - "features\030\004 \001(\0132\033.google.protobuf.Feature" - "Set\0223\n\016fixed_features\030\005 \001(\0132\033.google.pro" - "tobuf.FeatureSet\"\325\001\n\016SourceCodeInfo\022:\n\010l" - "ocation\030\001 \003(\0132(.google.protobuf.SourceCo" - "deInfo.Location\032\206\001\n\010Location\022\020\n\004path\030\001 \003" - "(\005B\002\020\001\022\020\n\004span\030\002 \003(\005B\002\020\001\022\030\n\020leading_comm" - "ents\030\003 \001(\t\022\031\n\021trailing_comments\030\004 \001(\t\022!\n" - "\031leading_detached_comments\030\006 \003(\t\"\234\002\n\021Gen" - "eratedCodeInfo\022A\n\nannotation\030\001 \003(\0132-.goo" - "gle.protobuf.GeneratedCodeInfo.Annotatio" - "n\032\303\001\n\nAnnotation\022\020\n\004path\030\001 \003(\005B\002\020\001\022\023\n\013so" - "urce_file\030\002 \001(\t\022\r\n\005begin\030\003 \001(\005\022\013\n\003end\030\004 " - "\001(\005\022H\n\010semantic\030\005 \001(\01626.google.protobuf." - "GeneratedCodeInfo.Annotation.Semantic\"(\n" - "\010Semantic\022\010\n\004NONE\020\000\022\007\n\003SET\020\001\022\t\n\005ALIAS\020\002*" - "\247\002\n\007Edition\022\023\n\017EDITION_UNKNOWN\020\000\022\023\n\016EDIT" - "ION_LEGACY\020\204\007\022\023\n\016EDITION_PROTO2\020\346\007\022\023\n\016ED" - "ITION_PROTO3\020\347\007\022\021\n\014EDITION_2023\020\350\007\022\021\n\014ED" - "ITION_2024\020\351\007\022\027\n\023EDITION_1_TEST_ONLY\020\001\022\027" - "\n\023EDITION_2_TEST_ONLY\020\002\022\035\n\027EDITION_99997" - "_TEST_ONLY\020\235\215\006\022\035\n\027EDITION_99998_TEST_ONL" - "Y\020\236\215\006\022\035\n\027EDITION_99999_TEST_ONLY\020\237\215\006\022\023\n\013" - "EDITION_MAX\020\377\377\377\377\007B~\n\023com.google.protobuf" - "B\020DescriptorProtosH\001Z-google.golang.org/" - "protobuf/types/descriptorpb\370\001\001\242\002\003GPB\252\002\032G" - "oogle.Protobuf.Reflection" + "ST_EFFORT\020\002*\006\010\350\007\020\213N*\006\010\213N\020\220N*\006\010\220N\020\221NJ\006\010\347\007" + "\020\350\007\"\202\003\n\022FeatureSetDefaults\022N\n\010defaults\030\001" + " \003(\0132<.google.protobuf.FeatureSetDefault" + "s.FeatureSetEditionDefault\0221\n\017minimum_ed" + "ition\030\004 \001(\0162\030.google.protobuf.Edition\0221\n" + "\017maximum_edition\030\005 \001(\0162\030.google.protobuf" + ".Edition\032\265\001\n\030FeatureSetEditionDefault\022)\n" + "\007edition\030\003 \001(\0162\030.google.protobuf.Edition" + "\0229\n\024overridable_features\030\004 \001(\0132\033.google." + "protobuf.FeatureSet\0223\n\016fixed_features\030\005 " + "\001(\0132\033.google.protobuf.FeatureSet\"\325\001\n\016Sou" + "rceCodeInfo\022:\n\010location\030\001 \003(\0132(.google.p" + "rotobuf.SourceCodeInfo.Location\032\206\001\n\010Loca" + "tion\022\020\n\004path\030\001 \003(\005B\002\020\001\022\020\n\004span\030\002 \003(\005B\002\020\001" + "\022\030\n\020leading_comments\030\003 \001(\t\022\031\n\021trailing_c" + "omments\030\004 \001(\t\022!\n\031leading_detached_commen" + "ts\030\006 \003(\t\"\234\002\n\021GeneratedCodeInfo\022A\n\nannota" + "tion\030\001 \003(\0132-.google.protobuf.GeneratedCo" + "deInfo.Annotation\032\303\001\n\nAnnotation\022\020\n\004path" + "\030\001 \003(\005B\002\020\001\022\023\n\013source_file\030\002 \001(\t\022\r\n\005begin" + "\030\003 \001(\005\022\013\n\003end\030\004 \001(\005\022H\n\010semantic\030\005 \001(\01626." + "google.protobuf.GeneratedCodeInfo.Annota" + "tion.Semantic\"(\n\010Semantic\022\010\n\004NONE\020\000\022\007\n\003S" + "ET\020\001\022\t\n\005ALIAS\020\002*\247\002\n\007Edition\022\023\n\017EDITION_U" + "NKNOWN\020\000\022\023\n\016EDITION_LEGACY\020\204\007\022\023\n\016EDITION" + "_PROTO2\020\346\007\022\023\n\016EDITION_PROTO3\020\347\007\022\021\n\014EDITI" + "ON_2023\020\350\007\022\021\n\014EDITION_2024\020\351\007\022\027\n\023EDITION" + "_1_TEST_ONLY\020\001\022\027\n\023EDITION_2_TEST_ONLY\020\002\022" + "\035\n\027EDITION_99997_TEST_ONLY\020\235\215\006\022\035\n\027EDITIO" + "N_99998_TEST_ONLY\020\236\215\006\022\035\n\027EDITION_99999_T" + "EST_ONLY\020\237\215\006\022\023\n\013EDITION_MAX\020\377\377\377\377\007B~\n\023com" + ".google.protobufB\020DescriptorProtosH\001Z-go" + "ogle.golang.org/protobuf/types/descripto" + "rpb\370\001\001\242\002\003GPB\252\002\032Google.Protobuf.Reflectio" + "n" }; static ::absl::once_flag descriptor_table_google_2fprotobuf_2fdescriptor_2eproto_once; PROTOBUF_CONSTINIT const ::_pbi::DescriptorTable descriptor_table_google_2fprotobuf_2fdescriptor_2eproto = { false, false, - 9985, + 9961, descriptor_table_protodef_google_2fprotobuf_2fdescriptor_2eproto, "google/protobuf/descriptor.proto", &descriptor_table_google_2fprotobuf_2fdescriptor_2eproto_once, diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index 8e5e21480a..b4a62d3efa 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -1066,11 +1066,24 @@ message FeatureSet { reserved 999; - extensions 1000; // for Protobuf C++ - extensions 1001; // for Protobuf Java - extensions 1002; // for Protobuf Go - - extensions 9990; // for deprecated Java Proto1 + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { number: 1002, full_name: ".pb.go", type: ".pb.GoFeatures" }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; extensions 9995 to 9999; // For internal testing extensions 10000; // for https://github.com/bufbuild/protobuf-es