Merge branch 'protocolbuffers:main' into main

pull/13361/head
桂后昌 2 years ago committed by GitHub
commit 04dae9fa16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 60
      .github/workflows/test_cpp.yml
  2. 8
      .github/workflows/test_csharp.yml
  3. 22
      .github/workflows/test_java.yml
  4. 8
      .github/workflows/test_objectivec.yml
  5. 28
      .github/workflows/test_php.yml
  6. 2
      .github/workflows/test_php_ext.yml
  7. 8
      .github/workflows/test_python.yml
  8. 81
      .github/workflows/test_ruby.yml
  9. 4
      .github/workflows/test_rust.yml
  10. 4
      build_defs/internal_shell.bzl
  11. 6
      conformance/BUILD.bazel
  12. 2
      conformance/failure_list_jruby_ffi.txt
  13. 37
      java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java
  14. 7
      java/core/src/main/java/com/google/protobuf/MapField.java
  15. 232
      java/core/src/main/java/com/google/protobuf/MapFieldBuilder.java
  16. 48
      java/core/src/main/java/com/google/protobuf/MapFieldReflectionAccessor.java
  17. 148
      java/core/src/main/java/com/google/protobuf/TextFormat.java
  18. 2
      java/core/src/main/java/com/google/protobuf/TypeRegistry.java
  19. 97
      java/core/src/test/java/com/google/protobuf/TextFormatTest.java
  20. 11
      java/core/src/test/java/com/google/protobuf/TypeRegistryTest.java
  21. 2
      objectivec/BUILD.bazel
  22. 4
      protobuf_deps.bzl
  23. 8
      ruby/.gitignore
  24. 158
      ruby/BUILD.bazel
  25. 33
      ruby/README.md
  26. 48
      ruby/Rakefile
  27. 103
      ruby/ext/google/protobuf_c/BUILD.bazel
  28. 3
      ruby/ext/google/protobuf_c/Rakefile
  29. 62
      ruby/ext/google/protobuf_c/convert.c
  30. 3
      ruby/ext/google/protobuf_c/extconf.rb
  31. 44
      ruby/ext/google/protobuf_c/glue.c
  32. 51
      ruby/ext/google/protobuf_c/message.c
  33. 87
      ruby/ext/google/protobuf_c/shared_convert.c
  34. 49
      ruby/ext/google/protobuf_c/shared_convert.h
  35. 88
      ruby/ext/google/protobuf_c/shared_message.c
  36. 48
      ruby/ext/google/protobuf_c/shared_message.h
  37. 26
      ruby/google-protobuf.gemspec
  38. 46
      ruby/lib/google/BUILD.bazel
  39. 42
      ruby/lib/google/protobuf.rb
  40. 177
      ruby/lib/google/protobuf/ffi/descriptor.rb
  41. 93
      ruby/lib/google/protobuf/ffi/descriptor_pool.rb
  42. 184
      ruby/lib/google/protobuf/ffi/enum_descriptor.rb
  43. 236
      ruby/lib/google/protobuf/ffi/ffi.rb
  44. 332
      ruby/lib/google/protobuf/ffi/field_descriptor.rb
  45. 71
      ruby/lib/google/protobuf/ffi/file_descriptor.rb
  46. 89
      ruby/lib/google/protobuf/ffi/internal/arena.rb
  47. 328
      ruby/lib/google/protobuf/ffi/internal/convert.rb
  48. 58
      ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb
  49. 48
      ruby/lib/google/protobuf/ffi/internal/type_safety.rb
  50. 419
      ruby/lib/google/protobuf/ffi/map.rb
  51. 664
      ruby/lib/google/protobuf/ffi/message.rb
  52. 53
      ruby/lib/google/protobuf/ffi/object_cache.rb
  53. 111
      ruby/lib/google/protobuf/ffi/oneof_descriptor.rb
  54. 526
      ruby/lib/google/protobuf/ffi/repeated_field.rb
  55. 73
      ruby/lib/google/protobuf_ffi.rb
  56. 43
      ruby/lib/google/protobuf_native.rb
  57. 94
      ruby/lib/google/tasks/ffi.rake
  58. 9
      ruby/tests/BUILD.bazel
  59. 37
      ruby/tests/implementation.rb
  60. 2
      ruby/tests/object_cache_test.rb
  61. 14
      rust/BUILD
  62. 1
      rust/cpp.rs
  63. 249
      rust/string.rs
  64. 44
      rust/test/BUILD
  65. 10
      rust/test/shared/accessors_test.rs
  66. 1
      rust/upb.rs
  67. 328
      rust/utf8.rs
  68. 1
      src/file_lists.cmake
  69. 2
      src/google/protobuf/BUILD.bazel
  70. 4
      src/google/protobuf/any.pb.cc
  71. 2
      src/google/protobuf/any.pb.h
  72. 24
      src/google/protobuf/api.pb.cc
  73. 6
      src/google/protobuf/api.pb.h
  74. 10
      src/google/protobuf/compiler/BUILD.bazel
  75. 5
      src/google/protobuf/compiler/command_line_interface_unittest.cc
  76. 39
      src/google/protobuf/compiler/cpp/field.cc
  77. 62
      src/google/protobuf/compiler/cpp/field.h
  78. 27
      src/google/protobuf/compiler/cpp/field_generators/cord_field.cc
  79. 162
      src/google/protobuf/compiler/cpp/field_generators/enum_field.cc
  80. 6
      src/google/protobuf/compiler/cpp/field_generators/map_field.cc
  81. 143
      src/google/protobuf/compiler/cpp/field_generators/message_field.cc
  82. 156
      src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc
  83. 188
      src/google/protobuf/compiler/cpp/field_generators/string_field.cc
  84. 17
      src/google/protobuf/compiler/cpp/file.cc
  85. 2
      src/google/protobuf/compiler/cpp/generator.cc
  86. 34
      src/google/protobuf/compiler/cpp/helpers.cc
  87. 19
      src/google/protobuf/compiler/cpp/helpers.h
  88. 14
      src/google/protobuf/compiler/cpp/message.cc
  89. 1
      src/google/protobuf/compiler/cpp/options.h
  90. 86
      src/google/protobuf/compiler/cpp/parse_function_generator.cc
  91. 25
      src/google/protobuf/compiler/plugin.pb.cc
  92. 8
      src/google/protobuf/compiler/plugin.pb.h
  93. 6
      src/google/protobuf/compiler/python/generator.cc
  94. 47
      src/google/protobuf/compiler/python/pyi_generator.cc
  95. 6
      src/google/protobuf/compiler/python/pyi_generator.h
  96. 1
      src/google/protobuf/compiler/rust/BUILD.bazel
  97. 3
      src/google/protobuf/compiler/rust/accessors/accessors.cc
  98. 2
      src/google/protobuf/compiler/rust/accessors/accessors.h
  99. 82
      src/google/protobuf/compiler/rust/accessors/singular_message.cc
  100. 51
      src/google/protobuf/compiler/rust/message.cc
  101. Some files were not shown because too many files have changed in this diff Show More

@ -26,20 +26,16 @@ jobs:
- { name: No-RTTI, flags: --cxxopt=-fno-rtti }
include:
# Set defaults
- image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:309dae3122031447d714414814d262e5f31cb93c0e248e9c02f9d8cdafd7e3b9
- image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:04cd765285bc52cbbf51d66c8c66d8603579cf0f19cc42df26b09d2c270541fb
- targets: //pkg/... //src/... @com_google_protobuf_examples//...
# Override cases with custom images
- config: { name: "TCMalloc" }
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:4df3b4749e787ba0a671ec0b783d0f1ba05f60be4c9e9fd72c875550a0cde1ea"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:e4493c58e7c3f3a0775e0b19a46bc822e866a5275b7a5463f509af8072f283db"
targets: "//src/..."
- config: { name: "aarch64" }
targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-3af05275178e16af30961976af126eabbbb2c733"
# TODO(b/278116805) Enable this once Bazel 6 is supported.
#- config: { name: "Bazel6" }
# targets: "//src/..."
# image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.0.0-6361b3a6e5c97e9951d03a4de28542fc45f1adab"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-222e7e87028b7098e088f5ca7cae06d32f483eb5"
name: Linux ${{ matrix.config.name }}
runs-on: ubuntu-latest
steps:
@ -48,7 +44,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: ${{ matrix.image }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -68,9 +64,9 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-5.4.0-2d15d9e888c9e7f90961dbd3afc8ea209717fb4b
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-518b4fcd8d0ded2484c94f02e835526cacfdac2d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: cpp_linux/gcc-${{ matrix.version }}
bazel: test //pkg/... //src/... @com_google_protobuf_examples//...
@ -90,19 +86,19 @@ jobs:
submodules: recursive
- name: Cross compile protoc for ${{ matrix.arch }}
id: cross-compile
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-${{ matrix.arch }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-release-${{ matrix.arch }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:${{ matrix.arch }}-3af05275178e16af30961976af126eabbbb2c733
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:${{ matrix.arch }}-222e7e87028b7098e088f5ca7cae06d32f483eb5
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
entrypoint: bash
command: >
@ -137,12 +133,12 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-cmake-${{ matrix.name }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-e6272cdfe97c6df307e17b83f3a7a70844f6fc08
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -162,12 +158,12 @@ jobs:
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-cmake-install
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.10.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -191,12 +187,12 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-cmake-examples
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.10.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -231,14 +227,14 @@ jobs:
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-cmake-gcc-${{ matrix.name }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-5.4.0-307caa02808127e49720f3e77d6a9f3b3ef5a915
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-518b4fcd8d0ded2484c94f02e835526cacfdac2d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
entrypoint: bash
command: >-
@ -261,12 +257,12 @@ jobs:
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-cmake-${{ matrix.name }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-1da1e086a7d1863b8bdd181ef6388a02dcd62f3a
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -285,12 +281,12 @@ jobs:
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: linux-cmake-32-bit
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea
platform: linux/386
@ -331,7 +327,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel: ${{ matrix.bazel }}
@ -386,7 +382,7 @@ jobs:
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: ${{ matrix.name }}
vsversion: ${{ matrix.vsversion }}
@ -394,7 +390,7 @@ jobs:
# Install phase.
- name: Configure CMake for install
if: matrix.install-flags
uses: protocolbuffers/protobuf-ci/bash@v1
uses: protocolbuffers/protobuf-ci/bash@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }}
@ -416,7 +412,7 @@ jobs:
run: cmake --build . --target clean && rm CMakeCache.txt
- name: Configure CMake
uses: protocolbuffers/protobuf-ci/bash@v1
uses: protocolbuffers/protobuf-ci/bash@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }}

@ -21,9 +21,9 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-508417e5215994ade7585d28ba3aad681a25fa5d
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: csharp_linux
bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel
@ -60,7 +60,7 @@ jobs:
- name: Build protobuf C# tests under x86_64 docker image
# Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
# (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -77,7 +77,7 @@ jobs:
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8
skip-staleness-check: true

@ -19,19 +19,19 @@ jobs:
include:
- name: OpenJDK 8
version: '8'
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-03a376b5d6ef66f827fc307716e3b841cc26b709
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
targets: //java/... //java/internal:java_version
- name: OpenJDK 11
version: '11'
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-03a376b5d6ef66f827fc307716e3b841cc26b709
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
targets: //java/... //java/internal:java_version
- name: OpenJDK 17
version: '17'
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-03a376b5d6ef66f827fc307716e3b841cc26b709
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
targets: //java/... //java/internal:java_version
- name: aarch64
version: 'aarch64'
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-3af05275178e16af30961976af126eabbbb2c733
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-222e7e87028b7098e088f5ca7cae06d32f483eb5
targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test
name: Linux ${{ matrix.name }}
@ -42,7 +42,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: ${{ matrix.image }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -58,9 +58,9 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run Linkage Monitor test
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-03a376b5d6ef66f827fc307716e3b841cc26b709
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: java_linux/8
bazel: test --test_output=all //java:linkage_monitor --spawn_strategy=standalone
@ -75,9 +75,9 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Build protoc
id: build-protoc
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-x86_64
- name: Move protoc into place and clean up
@ -89,9 +89,9 @@ jobs:
mvn -e -B -Dhttps.protocols=TLSv1.2 install -Dmaven.test.skip=true
working-directory: java
- name: Generate pom.xml files from the template
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-03a376b5d6ef66f827fc307716e3b841cc26b709
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: java_linux/11
# protobuf-java and protobuf-java-util are the member of the BOM

@ -39,13 +39,13 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
uses: protocolbuffers/protobuf-ci/ccache@v2
with:
cache-prefix: objectivec_${{ matrix.platform }}_${{ matrix.xc_config }}
support-modules: true
- name: Run tests
uses: protocolbuffers/protobuf-ci/bash@v1
uses: protocolbuffers/protobuf-ci/bash@v2
env:
CC: ${{ github.workspace }}/ci/clang_wrapper
CXX: ${{ github.workspace }}/ci/clang_wrapper++
@ -81,7 +81,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Pod lib lint
uses: protocolbuffers/protobuf-ci/bash@v1
uses: protocolbuffers/protobuf-ci/bash@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: |
@ -122,7 +122,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: bazel ${{ matrix.config.bazel_action }}
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel: ${{ matrix.config.bazel_action }} ${{ matrix.config.flags }} ${{ matrix.bazel_targets }}

@ -44,14 +44,14 @@ jobs:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup composer
uses: protocolbuffers/protobuf-ci/composer-setup@v1
uses: protocolbuffers/protobuf-ci/composer-setup@v2
with:
cache-prefix: php-${{ matrix.version-short }}
directory: php
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-508417e5215994ade7585d28ba3aad681a25fa5d
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
extra-flags: -e COMPOSER_HOME=/workspace/composer-cache
command: ${{ matrix.command }}
@ -85,20 +85,20 @@ jobs:
- name: Cross compile protoc for i386
id: cross-compile
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-i386
- name: Setup composer
uses: protocolbuffers/protobuf-ci/composer-setup@v1
uses: protocolbuffers/protobuf-ci/composer-setup@v2
with:
cache-prefix: php-${{ matrix.version }}
directory: php
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: ${{ env.image }}
skip-staleness-check: true
@ -123,20 +123,20 @@ jobs:
- name: Cross compile protoc for aarch64
id: cross-compile
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-aarch64
- name: Setup composer
uses: protocolbuffers/protobuf-ci/composer-setup@v1
uses: protocolbuffers/protobuf-ci/composer-setup@v2
with:
cache-prefix: php-8.0
directory: php
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310
platform: linux/arm64
@ -180,13 +180,13 @@ jobs:
run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1)
- name: Setup composer
uses: protocolbuffers/protobuf-ci/composer-setup@v1
uses: protocolbuffers/protobuf-ci/composer-setup@v2
with:
cache-prefix: php-${{ matrix.version }}
directory: php
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: php_macos/${{ matrix.version }}
@ -199,7 +199,7 @@ jobs:
popd
- name: Run conformance tests
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: php_macos/${{ matrix.version }}

@ -22,7 +22,7 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Package extension
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: php_ext/${{ matrix.version }}

@ -30,7 +30,7 @@ jobs:
targets: //python/... //python:aarch64_test
# TODO(b/262628111) Enable this once conformance tests are fixed.
flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-3af05275178e16af30961976af126eabbbb2c733
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-222e7e87028b7098e088f5ca7cae06d32f483eb5
name: Linux ${{ matrix.type }} ${{ matrix.version }}
runs-on: ubuntu-latest
@ -40,9 +40,9 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.version) }}
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea', matrix.version) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION
@ -85,7 +85,7 @@ jobs:
source venv/bin/activate
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
env:
KOKORO_PYTHON_VERSION: ${{ matrix.version }}
with:

@ -17,17 +17,18 @@ jobs:
fail-fast: false
matrix:
include:
- { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1}
- { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1}
- { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1}
- { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1}
- { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1}
- { name: JRuby 9.3, ruby: jruby-9.3.10.0, bazel: 5.1.1}
- { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1}
- { name: Ruby 2.7 (Bazel6), ruby: ruby-2.7.0, bazel: 6.0.0}
- { name: JRuby 9.4 (Bazel6), ruby: jruby-9.4.3.0, bazel: 6.0.0}
# Test both FFI and Native implementations on the highest and lowest
# Ruby versions for CRuby and JRuby, but only on Bazel 5.x.
- { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: NATIVE }
- { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: FFI }
- { name: Ruby 3.0, ruby: ruby-3.0.2 }
- { name: Ruby 3.1, ruby: ruby-3.1.0 }
- { name: Ruby 3.2, ruby: ruby-3.2.0, ffi: NATIVE }
- { name: Ruby 3.2, ruby: ruby-3.2.0, ffi: FFI }
- { name: JRuby 9.4, ruby: jruby-9.4.3.0, ffi: NATIVE }
- { name: JRuby 9.4, ruby: jruby-9.4.3.0, ffi: FFI }
name: Linux ${{ matrix.name }}
name: Linux ${{ matrix.name }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
@ -35,12 +36,12 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }}
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-6.3.0-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea', matrix.ruby) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }}
linux-aarch64:
name: Linux aarch64
@ -53,14 +54,14 @@ jobs:
- name: Cross compile protoc for aarch64
id: cross-compile
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-aarch64
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v1
uses: protocolbuffers/protobuf-ci/docker@v2
with:
image: arm64v8/ruby:2.7.3-buster
skip-staleness-check: true
@ -78,10 +79,18 @@ jobs:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
# This is the full set of versions we support on MacOS.
version: [ "2.7", "3.0", "3.1", "3.2" ]
include:
# Test both FFI and Native implementations on the highest and lowest
# Ruby versions for CRuby, but only on Bazel 5.x.
# Quote versions numbers otherwise 3.0 will render as 3
- { version: "2.7", ffi: NATIVE }
- { version: "2.7", ffi: FFI }
- { version: "3.0" }
- { version: "3.1" }
- { version: "3.2", ffi: NATIVE }
- { version: "3.2", ffi: FFI }
name: MacOS Ruby ${{ matrix.version }}
name: MacOS Ruby ${{ matrix.version }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }}
runs-on: macos-12
steps:
- name: Checkout pending changes
@ -98,27 +107,28 @@ jobs:
run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1)
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v1
uses: protocolbuffers/protobuf-ci/bazel@v2
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_macos/${{ matrix.version }}
bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }}
bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }} --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }}
test_ruby_gems:
strategy:
fail-fast: false
matrix:
include:
- { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1}
- { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1}
- { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1}
- { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1}
- { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1}
- { name: JRuby 9.3, ruby: jruby-9.3.10.0, bazel: 5.1.1}
- { name: JRuby 9.4, ruby: jruby-9.4.3.0, bazel: 5.1.1}
- { name: Ruby 2.7 (Bazel6), ruby: ruby-2.7.0, bazel: 6.0.0}
- { name: JRuby 9.4 (Bazel6), ruby: jruby-9.4.3.0, bazel: 6.0.0}
name: Install ${{ matrix.name }}
# Test both FFI and Native implementations on the highest and lowest
# Ruby versions for CRuby and JRuby, but only on Bazel 5.x.
- { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: NATIVE }
- { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: FFI }
- { name: Ruby 3.0, ruby: ruby-3.0.2}
- { name: Ruby 3.1, ruby: ruby-3.1.0}
- { name: Ruby 3.2, ruby: ruby-3.2.0, ffi: NATIVE }
- { name: Ruby 3.2, ruby: ruby-3.2.0, ffi: FFI }
- { name: JRuby 9.4, ruby: jruby-9.4.3.0, ffi: NATIVE }
- { name: JRuby 9.4, ruby: jruby-9.4.3.0, ffi: FFI }
name: Install ${{ matrix.name }}${{ matrix.ffi == 'FFI' && ' FFI' || '' }}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
@ -126,16 +136,17 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-508417e5215994ade7585d28ba3aad681a25fa5d
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-6.3.0-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }}
bash: >
bazel --version;
ruby --version;
bazel build //ruby:release //:protoc $BAZEL_FLAGS;
bazel build //ruby:release //:protoc ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled' || '' }} $BAZEL_FLAGS;
gem install bazel-bin/ruby/google-protobuf-*;
bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto;
bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto;
ruby ruby/tests/basic.rb
${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/basic.rb;
${{ matrix.ffi == 'FFI' && 'PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} ruby ruby/tests/implementation.rb

@ -21,9 +21,9 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.0.0-6361b3a6e5c97e9951d03a4de28542fc45f1adab
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: rust_linux
bazel: |

@ -32,7 +32,7 @@ def inline_sh_binary(
native.genrule(
name = name + "_genrule",
srcs = srcs,
exec_tools = tools,
tools = tools,
outs = [name + ".sh"],
cmd = "cat <<'EOF' >$(OUTS)\n#!/bin/bash -exu\n%s\nEOF\n" % cmd,
visibility = ["//visibility:private"],
@ -77,7 +77,7 @@ def inline_sh_test(
native.genrule(
name = name + "_genrule",
srcs = srcs,
exec_tools = tools,
tools = tools,
outs = [name + ".sh"],
cmd = "cat <<'EOF' >$(OUTS)\n#!/bin/bash -exu\n%s\nEOF\n" % cmd,
visibility = ["//visibility:private"],

@ -24,6 +24,7 @@ exports_files([
"failure_list_python_cpp.txt",
"failure_list_ruby.txt",
"failure_list_jruby.txt",
"failure_list_jruby_ffi.txt",
"text_format_failure_list_cpp.txt",
"text_format_failure_list_csharp.txt",
"text_format_failure_list_java.txt",
@ -34,6 +35,7 @@ exports_files([
"text_format_failure_list_python_cpp.txt",
"text_format_failure_list_ruby.txt",
"text_format_failure_list_jruby.txt",
"text_format_failure_list_jruby_ffi.txt",
])
cc_proto_library(
@ -326,12 +328,12 @@ ruby_binary(
name = "conformance_ruby",
testonly = True,
srcs = ["conformance_ruby.rb"],
visibility = ["//ruby:__subpackages__"],
deps = [
":conformance_ruby_proto",
"//:test_messages_proto2_ruby_proto",
"//:test_messages_proto3_ruby_proto",
"//:test_messages_proto3_ruby_proto",
],
visibility = ["//ruby:__subpackages__"],
)
################################################################################

@ -0,0 +1,2 @@
Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput

@ -959,13 +959,27 @@ public abstract class GeneratedMessageV3 extends AbstractMessage implements Seri
* map field directly and thus enables us to access the map field as a list.
*/
@SuppressWarnings({"unused", "rawtypes"})
protected MapFieldReflectionAccessor internalGetMapFieldReflection(int fieldNumber) {
return internalGetMapField(fieldNumber);
}
/** TODO(b/258340024): Remove, exists for compatibility with generated code. */
@Deprecated
@SuppressWarnings({"unused", "rawtypes"})
protected MapField internalGetMapField(int fieldNumber) {
// Note that we can't use descriptor names here because this method will
// be called when descriptor is being initialized.
throw new IllegalArgumentException("No map fields found in " + getClass().getName());
}
/** Like {@link #internalGetMapField} but return a mutable version. */
/** Like {@link #internalGetMapFieldReflection} but return a mutable version. */
@SuppressWarnings({"unused", "rawtypes"})
protected MapFieldReflectionAccessor internalGetMutableMapFieldReflection(int fieldNumber) {
return internalGetMutableMapField(fieldNumber);
}
/** TODO(b/258340024): Remove, exists for compatibility with generated code. */
@Deprecated
@SuppressWarnings({"unused", "rawtypes"})
protected MapField internalGetMutableMapField(int fieldNumber) {
// Note that we can't use descriptor names here because this method will
@ -2047,6 +2061,13 @@ public abstract class GeneratedMessageV3 extends AbstractMessage implements Seri
* generated API only allows us to access it as a map. This method returns the underlying map
* field directly and thus enables us to access the map field as a list.
*/
@SuppressWarnings("unused")
protected MapFieldReflectionAccessor internalGetMapFieldReflection(int fieldNumber) {
return internalGetMapField(fieldNumber);
}
/** TODO(b/258340024): Remove, exists for compatibility with generated code. */
@Deprecated
@SuppressWarnings({"rawtypes", "unused"})
protected MapField internalGetMapField(int fieldNumber) {
// Note that we can't use descriptor names here because this method will
@ -2802,7 +2823,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage implements Seri
final FieldDescriptor descriptor, final Class<? extends GeneratedMessageV3> messageClass) {
field = descriptor;
Method getDefaultInstanceMethod = getMethodOrDie(messageClass, "getDefaultInstance");
MapField<?, ?> defaultMapField =
MapFieldReflectionAccessor defaultMapField =
getMapField((GeneratedMessageV3) invokeOrDie(getDefaultInstanceMethod, null));
mapEntryMessageDefaultInstance = defaultMapField.getMapEntryMessageDefaultInstance();
}
@ -2810,16 +2831,16 @@ public abstract class GeneratedMessageV3 extends AbstractMessage implements Seri
private final FieldDescriptor field;
private final Message mapEntryMessageDefaultInstance;
private MapField<?, ?> getMapField(GeneratedMessageV3 message) {
return (MapField<?, ?>) message.internalGetMapField(field.getNumber());
private MapFieldReflectionAccessor getMapField(GeneratedMessageV3 message) {
return message.internalGetMapFieldReflection(field.getNumber());
}
private MapField<?, ?> getMapField(GeneratedMessageV3.Builder<?> builder) {
return (MapField<?, ?>) builder.internalGetMapField(field.getNumber());
private MapFieldReflectionAccessor getMapField(GeneratedMessageV3.Builder<?> builder) {
return builder.internalGetMapFieldReflection(field.getNumber());
}
private MapField<?, ?> getMutableMapField(GeneratedMessageV3.Builder<?> builder) {
return (MapField<?, ?>) builder.internalGetMutableMapField(field.getNumber());
private MapFieldReflectionAccessor getMutableMapField(GeneratedMessageV3.Builder<?> builder) {
return builder.internalGetMutableMapFieldReflection(field.getNumber());
}
private Message coerceType(Message value) {

@ -53,7 +53,7 @@ import java.util.Set;
* <p>THREAD-SAFETY NOTE: Read-only access is thread-safe. Users can call getMap() and getList()
* concurrently in multiple threads. If write-access is needed, all access must be synchronized.
*/
public class MapField<K, V> implements MutabilityOracle {
public class MapField<K, V> extends MapFieldReflectionAccessor implements MutabilityOracle {
/**
* Indicates where the data of this map field is currently stored.
@ -225,6 +225,7 @@ public class MapField<K, V> implements MutabilityOracle {
}
/** Gets the content of this MapField as a read-only List. */
@Override
List<Message> getList() {
if (mode == StorageMode.MAP) {
synchronized (this) {
@ -238,6 +239,7 @@ public class MapField<K, V> implements MutabilityOracle {
}
/** Gets a mutable List view of this MapField. */
@Override
List<Message> getMutableList() {
if (mode != StorageMode.LIST) {
if (mode == StorageMode.MAP) {
@ -250,6 +252,7 @@ public class MapField<K, V> implements MutabilityOracle {
}
/** Gets the default instance of the message stored in the list view of this map field. */
@Override
Message getMapEntryMessageDefaultInstance() {
return converter.getMessageDefaultInstance();
}
@ -278,7 +281,7 @@ public class MapField<K, V> implements MutabilityOracle {
}
/** An internal map that checks for mutability before delegating. */
private static class MutabilityAwareMap<K, V> implements Map<K, V> {
static class MutabilityAwareMap<K, V> implements Map<K, V> {
private final MutabilityOracle mutabilityOracle;
private final Map<K, V> delegate;

@ -0,0 +1,232 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.protobuf;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Internal representation of map fields in generated builders.
*
* <p>This class supports accessing the map field as a {@link Map} to be used in generated API and
* also supports accessing the field as a {@link List} to be used in reflection API. It keeps track
* of where the data is currently stored and do necessary conversions between map and list.
*
* <p>This class is a protobuf implementation detail. Users shouldn't use this class directly.
*/
public class MapFieldBuilder<
KeyT,
MessageOrBuilderT extends MessageOrBuilder,
MessageT extends MessageOrBuilderT,
BuilderT extends MessageOrBuilderT>
extends MapFieldReflectionAccessor {
// Only one of the three fields may be non-null at any time.
/** nullable */
Map<KeyT, MessageOrBuilderT> builderMap = new LinkedHashMap<>();
/** nullable */
Map<KeyT, MessageT> messageMap = null;
// messageList elements are always MapEntry<KeyT, MessageT>, but we need a List<Message> for
// reflection.
/** nullable */
List<Message> messageList = null;
Converter<KeyT, MessageOrBuilderT, MessageT> converter;
/** Convert a MessageOrBuilder to a Message regardless of which it holds. */
public interface Converter<
KeyT, MessageOrBuilderT extends MessageOrBuilder, MessageT extends MessageOrBuilderT> {
MessageT build(MessageOrBuilderT val);
MapEntry<KeyT, MessageT> defaultEntry();
}
public MapFieldBuilder(Converter<KeyT, MessageOrBuilderT, MessageT> converter) {
this.converter = converter;
}
@SuppressWarnings("unchecked")
private List<MapEntry<KeyT, MessageT>> getMapEntryList() {
ArrayList<MapEntry<KeyT, MessageT>> list = new ArrayList<>(messageList.size());
for (Message entry : messageList) {
list.add((MapEntry<KeyT, MessageT>) entry);
}
return list;
}
public Map<KeyT, MessageOrBuilderT> ensureBuilderMap() {
if (builderMap != null) {
return builderMap;
}
if (messageMap != null) {
builderMap = new LinkedHashMap<>(messageMap.size());
for (Map.Entry<KeyT, MessageT> entry : messageMap.entrySet()) {
builderMap.put(entry.getKey(), entry.getValue());
}
messageMap = null;
return builderMap;
}
builderMap = new LinkedHashMap<>(messageList.size());
for (MapEntry<KeyT, MessageT> entry : getMapEntryList()) {
builderMap.put(entry.getKey(), entry.getValue());
}
messageList = null;
return builderMap;
}
public List<Message> ensureMessageList() {
if (messageList != null) {
return messageList;
}
if (builderMap != null) {
messageList = new ArrayList<>(builderMap.size());
for (Map.Entry<KeyT, MessageOrBuilderT> entry : builderMap.entrySet()) {
messageList.add(
converter.defaultEntry().toBuilder()
.setKey(entry.getKey())
.setValue(converter.build(entry.getValue()))
.build());
}
builderMap = null;
return messageList;
}
messageList = new ArrayList<>(messageMap.size());
for (Map.Entry<KeyT, MessageT> entry : messageMap.entrySet()) {
messageList.add(
converter.defaultEntry().toBuilder()
.setKey(entry.getKey())
.setValue(entry.getValue())
.build());
}
messageMap = null;
return messageList;
}
public Map<KeyT, MessageT> ensureMessageMap() {
messageMap = populateMutableMap();
builderMap = null;
messageList = null;
return messageMap;
}
public Map<KeyT, MessageT> getImmutableMap() {
return new MapField.MutabilityAwareMap<>(MutabilityOracle.IMMUTABLE, populateMutableMap());
}
private Map<KeyT, MessageT> populateMutableMap() {
if (messageMap != null) {
return messageMap;
}
if (builderMap != null) {
Map<KeyT, MessageT> toReturn = new LinkedHashMap<>(builderMap.size());
for (Map.Entry<KeyT, MessageOrBuilderT> entry : builderMap.entrySet()) {
toReturn.put(entry.getKey(), converter.build(entry.getValue()));
}
return toReturn;
}
Map<KeyT, MessageT> toReturn = new LinkedHashMap<>(messageList.size());
for (MapEntry<KeyT, MessageT> entry : getMapEntryList()) {
toReturn.put(entry.getKey(), entry.getValue());
}
return toReturn;
}
public void mergeFrom(MapField<KeyT, MessageT> other) {
ensureBuilderMap().putAll(MapFieldLite.copy(other.getMap()));
}
public void clear() {
builderMap = new LinkedHashMap<>();
messageMap = null;
messageList = null;
}
private boolean typedEquals(MapFieldBuilder<KeyT, MessageOrBuilderT, MessageT, BuilderT> other) {
return MapFieldLite.<KeyT, MessageOrBuilderT>equals(
ensureBuilderMap(), other.ensureBuilderMap());
}
@SuppressWarnings("unchecked")
@Override
public boolean equals(Object object) {
if (!(object instanceof MapFieldBuilder)) {
return false;
}
return typedEquals((MapFieldBuilder<KeyT, MessageOrBuilderT, MessageT, BuilderT>) object);
}
@Override
public int hashCode() {
return MapFieldLite.<KeyT, MessageOrBuilderT>calculateHashCodeForMap(ensureBuilderMap());
}
/** Returns a deep copy of this MapFieldBuilder. */
public MapFieldBuilder<KeyT, MessageOrBuilderT, MessageT, BuilderT> copy() {
MapFieldBuilder<KeyT, MessageOrBuilderT, MessageT, BuilderT> clone =
new MapFieldBuilder<>(converter);
clone.ensureBuilderMap().putAll(ensureBuilderMap());
return clone;
}
/** Converts this MapFieldBuilder to a MapField. */
public MapField<KeyT, MessageT> build(MapEntry<KeyT, MessageT> defaultEntry) {
MapField<KeyT, MessageT> mapField = MapField.newMapField(defaultEntry);
Map<KeyT, MessageT> map = mapField.getMutableMap();
for (Map.Entry<KeyT, MessageOrBuilderT> entry : ensureBuilderMap().entrySet()) {
map.put(entry.getKey(), converter.build(entry.getValue()));
}
mapField.makeImmutable();
return mapField;
}
// MapFieldReflectionAccessor implementation.
/** Gets the content of this MapField as a read-only List. */
@Override
List<Message> getList() {
return ensureMessageList();
}
/** Gets a mutable List view of this MapField. */
@Override
List<Message> getMutableList() {
return ensureMessageList();
}
/** Gets the default instance of the message stored in the list view of this map field. */
@Override
Message getMapEntryMessageDefaultInstance() {
return converter.defaultEntry();
}
}

@ -0,0 +1,48 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.protobuf;
import java.util.List;
/**
* A base class for package private shared methods between MapField and MapFieldBuilder to allow
* reflection to access both.
*/
public abstract class MapFieldReflectionAccessor {
/** Gets the content of this MapField as a read-only List. */
abstract List<Message> getList();
/** Gets a mutable List view of this MapField. */
abstract List<Message> getMutableList();
/** Gets the default instance of the message stored in the list view of this map field. */
abstract Message getMapEntryMessageDefaultInstance();
}

@ -1577,6 +1577,7 @@ public final class TextFormat {
private final boolean allowUnknownExtensions;
private final SingularOverwritePolicy singularOverwritePolicy;
private TextFormatParseInfoTree.Builder parseInfoTreeBuilder;
private final int recursionLimit;
private Parser(
TypeRegistry typeRegistry,
@ -1584,13 +1585,15 @@ public final class TextFormat {
boolean allowUnknownEnumValues,
boolean allowUnknownExtensions,
SingularOverwritePolicy singularOverwritePolicy,
TextFormatParseInfoTree.Builder parseInfoTreeBuilder) {
TextFormatParseInfoTree.Builder parseInfoTreeBuilder,
int recursionLimit) {
this.typeRegistry = typeRegistry;
this.allowUnknownFields = allowUnknownFields;
this.allowUnknownEnumValues = allowUnknownEnumValues;
this.allowUnknownExtensions = allowUnknownExtensions;
this.singularOverwritePolicy = singularOverwritePolicy;
this.parseInfoTreeBuilder = parseInfoTreeBuilder;
this.recursionLimit = recursionLimit;
}
/** Returns a new instance of {@link Builder}. */
@ -1607,6 +1610,7 @@ public final class TextFormat {
SingularOverwritePolicy.ALLOW_SINGULAR_OVERWRITES;
private TextFormatParseInfoTree.Builder parseInfoTreeBuilder = null;
private TypeRegistry typeRegistry = TypeRegistry.getEmptyTypeRegistry();
private int recursionLimit = 100;
/**
* Sets the TypeRegistry for resolving Any. If this is not set, TextFormat will not be able to
@ -1653,6 +1657,15 @@ public final class TextFormat {
return this;
}
/**
* Set the maximum recursion limit that the parser will allow. If the depth of the message
* exceeds this limit then the parser will stop and throw an exception.
*/
public Builder setRecursionLimit(int recursionLimit) {
this.recursionLimit = recursionLimit;
return this;
}
public Parser build() {
return new Parser(
typeRegistry,
@ -1660,7 +1673,8 @@ public final class TextFormat {
allowUnknownEnumValues,
allowUnknownExtensions,
singularOverwritePolicy,
parseInfoTreeBuilder);
parseInfoTreeBuilder,
recursionLimit);
}
}
@ -1784,7 +1798,7 @@ public final class TextFormat {
List<UnknownField> unknownFields = new ArrayList<UnknownField>();
while (!tokenizer.atEnd()) {
mergeField(tokenizer, extensionRegistry, target, unknownFields);
mergeField(tokenizer, extensionRegistry, target, unknownFields, recursionLimit);
}
checkUnknownFields(unknownFields);
}
@ -1794,9 +1808,16 @@ public final class TextFormat {
final Tokenizer tokenizer,
final ExtensionRegistry extensionRegistry,
final MessageReflection.MergeTarget target,
List<UnknownField> unknownFields)
List<UnknownField> unknownFields,
int recursionLimit)
throws ParseException {
mergeField(tokenizer, extensionRegistry, target, parseInfoTreeBuilder, unknownFields);
mergeField(
tokenizer,
extensionRegistry,
target,
parseInfoTreeBuilder,
unknownFields,
recursionLimit);
}
/** Parse a single field from {@code tokenizer} and merge it into {@code target}. */
@ -1805,7 +1826,8 @@ public final class TextFormat {
final ExtensionRegistry extensionRegistry,
final MessageReflection.MergeTarget target,
TextFormatParseInfoTree.Builder parseTreeBuilder,
List<UnknownField> unknownFields)
List<UnknownField> unknownFields,
int recursionLimit)
throws ParseException {
FieldDescriptor field = null;
String name;
@ -1815,8 +1837,17 @@ public final class TextFormat {
ExtensionRegistry.ExtensionInfo extension = null;
if ("google.protobuf.Any".equals(type.getFullName()) && tokenizer.tryConsume("[")) {
if (recursionLimit < 1) {
throw tokenizer.parseException("Message is nested too deep");
}
mergeAnyFieldValue(
tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, type);
tokenizer,
extensionRegistry,
target,
parseTreeBuilder,
unknownFields,
type,
recursionLimit - 1);
return;
}
@ -1895,7 +1926,7 @@ public final class TextFormat {
// Skips unknown fields.
if (field == null) {
detectSilentMarker(tokenizer, type, name);
guessFieldTypeAndSkip(tokenizer, type);
guessFieldTypeAndSkip(tokenizer, type, recursionLimit);
return;
}
@ -1913,7 +1944,8 @@ public final class TextFormat {
field,
extension,
childParseTreeBuilder,
unknownFields);
unknownFields,
recursionLimit);
} else {
consumeFieldValues(
tokenizer,
@ -1922,7 +1954,8 @@ public final class TextFormat {
field,
extension,
parseTreeBuilder,
unknownFields);
unknownFields,
recursionLimit);
}
} else {
detectSilentMarker(tokenizer, type, field.getFullName());
@ -1934,7 +1967,8 @@ public final class TextFormat {
field,
extension,
parseTreeBuilder,
unknownFields);
unknownFields,
recursionLimit);
}
if (parseTreeBuilder != null) {
@ -1981,7 +2015,8 @@ public final class TextFormat {
final FieldDescriptor field,
final ExtensionRegistry.ExtensionInfo extension,
final TextFormatParseInfoTree.Builder parseTreeBuilder,
List<UnknownField> unknownFields)
List<UnknownField> unknownFields,
int recursionLimit)
throws ParseException {
// Support specifying repeated field values as a comma-separated list.
// Ex."foo: [1, 2, 3]"
@ -1995,7 +2030,8 @@ public final class TextFormat {
field,
extension,
parseTreeBuilder,
unknownFields);
unknownFields,
recursionLimit);
if (tokenizer.tryConsume("]")) {
// End of list.
break;
@ -2011,7 +2047,8 @@ public final class TextFormat {
field,
extension,
parseTreeBuilder,
unknownFields);
unknownFields,
recursionLimit);
}
}
@ -2023,7 +2060,8 @@ public final class TextFormat {
final FieldDescriptor field,
final ExtensionRegistry.ExtensionInfo extension,
final TextFormatParseInfoTree.Builder parseTreeBuilder,
List<UnknownField> unknownFields)
List<UnknownField> unknownFields,
int recursionLimit)
throws ParseException {
if (singularOverwritePolicy == SingularOverwritePolicy.FORBID_SINGULAR_OVERWRITES
&& !field.isRepeated()) {
@ -2047,6 +2085,10 @@ public final class TextFormat {
Object value = null;
if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
if (recursionLimit < 1) {
throw tokenizer.parseException("Message is nested too deep");
}
final String endToken;
if (tokenizer.tryConsume("<")) {
endToken = ">";
@ -2055,37 +2097,24 @@ public final class TextFormat {
endToken = "}";
}
// Try to parse human readable format of Any in the form: [type_url]: { ... }
if (field.getMessageType().getFullName().equals("google.protobuf.Any")
&& tokenizer.tryConsume("[")) {
// Use Proto reflection here since depending on Any would intoduce a cyclic dependency
// (java_proto_library for any_java_proto depends on the protobuf_impl).
Message anyBuilder = DynamicMessage.getDefaultInstance(field.getMessageType());
MessageReflection.MergeTarget anyField = target.newMergeTargetForField(field, anyBuilder);
mergeAnyFieldValue(
Message defaultInstance = (extension == null) ? null : extension.defaultInstance;
MessageReflection.MergeTarget subField =
target.newMergeTargetForField(field, defaultInstance);
while (!tokenizer.tryConsume(endToken)) {
if (tokenizer.atEnd()) {
throw tokenizer.parseException("Expected \"" + endToken + "\".");
}
mergeField(
tokenizer,
extensionRegistry,
anyField,
subField,
parseTreeBuilder,
unknownFields,
field.getMessageType());
value = anyField.finish();
tokenizer.consume(endToken);
} else {
Message defaultInstance = (extension == null) ? null : extension.defaultInstance;
MessageReflection.MergeTarget subField =
target.newMergeTargetForField(field, defaultInstance);
while (!tokenizer.tryConsume(endToken)) {
if (tokenizer.atEnd()) {
throw tokenizer.parseException("Expected \"" + endToken + "\".");
}
mergeField(tokenizer, extensionRegistry, subField, parseTreeBuilder, unknownFields);
}
value = subField.finish();
recursionLimit - 1);
}
value = subField.finish();
} else {
switch (field.getType()) {
case INT32:
@ -2197,7 +2226,8 @@ public final class TextFormat {
MergeTarget target,
final TextFormatParseInfoTree.Builder parseTreeBuilder,
List<UnknownField> unknownFields,
Descriptor anyDescriptor)
Descriptor anyDescriptor,
int recursionLimit)
throws ParseException {
// Try to parse human readable format of Any in the form: [type_url]: { ... }
StringBuilder typeUrlBuilder = new StringBuilder();
@ -2243,7 +2273,13 @@ public final class TextFormat {
MessageReflection.BuilderAdapter contentTarget =
new MessageReflection.BuilderAdapter(contentBuilder);
while (!tokenizer.tryConsume(anyEndToken)) {
mergeField(tokenizer, extensionRegistry, contentTarget, parseTreeBuilder, unknownFields);
mergeField(
tokenizer,
extensionRegistry,
contentTarget,
parseTreeBuilder,
unknownFields,
recursionLimit);
}
target.setField(anyDescriptor.findFieldByName("type_url"), typeUrlBuilder.toString());
@ -2252,10 +2288,11 @@ public final class TextFormat {
}
/** Skips the next field including the field's name and value. */
private void skipField(Tokenizer tokenizer, Descriptor type) throws ParseException {
private void skipField(Tokenizer tokenizer, Descriptor type, int recursionLimit)
throws ParseException {
String name = consumeFullTypeName(tokenizer);
detectSilentMarker(tokenizer, type, name);
guessFieldTypeAndSkip(tokenizer, type);
guessFieldTypeAndSkip(tokenizer, type, recursionLimit);
// For historical reasons, fields may optionally be separated by commas or
// semicolons.
@ -2267,7 +2304,8 @@ public final class TextFormat {
/**
* Skips the whole body of a message including the beginning delimiter and the ending delimiter.
*/
private void skipFieldMessage(Tokenizer tokenizer, Descriptor type) throws ParseException {
private void skipFieldMessage(Tokenizer tokenizer, Descriptor type, int recursionLimit)
throws ParseException {
final String delimiter;
if (tokenizer.tryConsume("<")) {
delimiter = ">";
@ -2276,7 +2314,7 @@ public final class TextFormat {
delimiter = "}";
}
while (!tokenizer.lookingAt(">") && !tokenizer.lookingAt("}")) {
skipField(tokenizer, type);
skipField(tokenizer, type, recursionLimit);
}
tokenizer.consume(delimiter);
}
@ -2302,16 +2340,20 @@ public final class TextFormat {
* be a message or the input is ill-formed. For short-formed repeated fields (i.e. with "[]"),
* if it is repeated scalar, there must be a ":" between the field name and the starting "[" .
*/
private void guessFieldTypeAndSkip(Tokenizer tokenizer, Descriptor type) throws ParseException {
private void guessFieldTypeAndSkip(Tokenizer tokenizer, Descriptor type, int recursionLimit)
throws ParseException {
boolean semicolonConsumed = tokenizer.tryConsume(":");
if (tokenizer.lookingAt("[")) {
// Short repeated field form. If a semicolon was consumed, it could be repeated scalar or
// repeated message. If not, it must be repeated message.
skipFieldShortFormedRepeated(tokenizer, semicolonConsumed, type);
skipFieldShortFormedRepeated(tokenizer, semicolonConsumed, type, recursionLimit);
} else if (semicolonConsumed && !tokenizer.lookingAt("{") && !tokenizer.lookingAt("<")) {
skipFieldValue(tokenizer);
} else {
skipFieldMessage(tokenizer, type);
if (recursionLimit < 1) {
throw tokenizer.parseException("Message is nested too deep");
}
skipFieldMessage(tokenizer, type, recursionLimit - 1);
}
}
@ -2321,7 +2363,8 @@ public final class TextFormat {
* <p>Reports an error if scalar type is not allowed but showing up inside "[]".
*/
private void skipFieldShortFormedRepeated(
Tokenizer tokenizer, boolean scalarAllowed, Descriptor type) throws ParseException {
Tokenizer tokenizer, boolean scalarAllowed, Descriptor type, int recursionLimit)
throws ParseException {
if (!tokenizer.tryConsume("[") || tokenizer.tryConsume("]")) {
// Try skipping "[]".
return;
@ -2330,7 +2373,10 @@ public final class TextFormat {
while (true) {
if (tokenizer.lookingAt("{") || tokenizer.lookingAt("<")) {
// Try skipping message field inside "[]"
skipFieldMessage(tokenizer, type);
if (recursionLimit < 1) {
throw tokenizer.parseException("Message is nested too deep");
}
skipFieldMessage(tokenizer, type, recursionLimit - 1);
} else if (scalarAllowed) {
// Try skipping scalar field inside "[]".
skipFieldValue(tokenizer);

@ -82,7 +82,7 @@ public class TypeRegistry {
private static String getTypeName(String typeUrl) throws InvalidProtocolBufferException {
String[] parts = typeUrl.split("/");
if (parts.length == 1) {
if (parts.length <= 1) {
throw new InvalidProtocolBufferException("Invalid type url found: " + typeUrl);
}
return parts[parts.length - 1];

@ -56,6 +56,7 @@ import protobuf_unittest.UnittestProto.TestAllTypes;
import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
import protobuf_unittest.UnittestProto.TestEmptyMessage;
import protobuf_unittest.UnittestProto.TestOneof2;
import protobuf_unittest.UnittestProto.TestRecursiveMessage;
import protobuf_unittest.UnittestProto.TestRequired;
import protobuf_unittest.UnittestProto.TestReservedFields;
import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet;
@ -1842,4 +1843,100 @@ public class TextFormatTest {
assertThat(TextFormat.printer().printToString(message))
.isEqualTo("optional_float: -0.0\noptional_double: -0.0\n");
}
private TestRecursiveMessage makeRecursiveMessage(int depth) {
if (depth == 0) {
return TestRecursiveMessage.newBuilder().setI(5).build();
} else {
return TestRecursiveMessage.newBuilder().setA(makeRecursiveMessage(depth - 1)).build();
}
}
@Test
public void testDefaultRecursionLimit() throws Exception {
String depth100 = TextFormat.printer().printToString(makeRecursiveMessage(100));
String depth101 = TextFormat.printer().printToString(makeRecursiveMessage(101));
TextFormat.parse(depth100, TestRecursiveMessage.class);
try {
TextFormat.parse(depth101, TestRecursiveMessage.class);
assertWithMessage("Parsing deep message should have failed").fail();
} catch (TextFormat.ParseException e) {
assertThat(e).hasMessageThat().contains("too deep");
}
}
@Test
public void testRecursionLimitWithUnknownFields() throws Exception {
TextFormat.Parser parser =
TextFormat.Parser.newBuilder().setAllowUnknownFields(true).setRecursionLimit(2).build();
TestRecursiveMessage.Builder depth2 = TestRecursiveMessage.newBuilder();
parser.merge("u { u { i: 0 } }", depth2);
try {
TestRecursiveMessage.Builder depth3 = TestRecursiveMessage.newBuilder();
parser.merge("u { u { u { } } }", depth3);
assertWithMessage("Parsing deep message should have failed").fail();
} catch (TextFormat.ParseException e) {
assertThat(e).hasMessageThat().contains("too deep");
}
}
@Test
public void testRecursionLimitWithKnownAndUnknownFields() throws Exception {
TextFormat.Parser parser =
TextFormat.Parser.newBuilder().setAllowUnknownFields(true).setRecursionLimit(2).build();
TestRecursiveMessage.Builder depth2 = TestRecursiveMessage.newBuilder();
parser.merge("a { u { i: 0 } }", depth2);
try {
TestRecursiveMessage.Builder depth3 = TestRecursiveMessage.newBuilder();
parser.merge("a { u { u { } } }", depth3);
assertWithMessage("Parsing deep message should have failed").fail();
} catch (TextFormat.ParseException e) {
assertThat(e).hasMessageThat().contains("too deep");
}
}
@Test
public void testRecursionLimitWithAny() throws Exception {
TextFormat.Parser parser =
TextFormat.Parser.newBuilder()
.setRecursionLimit(2)
.setTypeRegistry(TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build())
.build();
TestAny.Builder depth2 = TestAny.newBuilder();
parser.merge(
"value { [type.googleapis.com/protobuf_unittest.TestAllTypes] { optional_int32: 1 } }",
depth2);
try {
TestAny.Builder depth3 = TestAny.newBuilder();
parser.merge(
"value { [type.googleapis.com/protobuf_unittest.TestAllTypes] { optional_nested_message {"
+ "} } }",
depth3);
assertWithMessage("Parsing deep message should have failed").fail();
} catch (TextFormat.ParseException e) {
assertThat(e).hasMessageThat().contains("too deep");
}
}
@Test
public void testRecursionLimitWithTopLevelAny() throws Exception {
TextFormat.Parser parser =
TextFormat.Parser.newBuilder()
.setRecursionLimit(2)
.setTypeRegistry(
TypeRegistry.newBuilder().add(TestRecursiveMessage.getDescriptor()).build())
.build();
Any.Builder depth2 = Any.newBuilder();
parser.merge(
"[type.googleapis.com/protobuf_unittest.TestRecursiveMessage] { a { i: 0 } }", depth2);
try {
Any.Builder depth3 = Any.newBuilder();
parser.merge(
"[type.googleapis.com/protobuf_unittest.TestRecursiveMessage] { a { a { i: 0 } } }",
depth3);
assertWithMessage("Parsing deep message should have failed").fail();
} catch (TextFormat.ParseException e) {
assertThat(e).hasMessageThat().contains("too deep");
}
}
}

@ -31,6 +31,7 @@
package com.google.protobuf;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import com.google.protobuf.Descriptors.Descriptor;
import protobuf_unittest.UnittestProto;
@ -41,6 +42,16 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public final class TypeRegistryTest {
@Test
public void getDescriptorForTypeUrl_throwsExceptionForUnknownTypes() throws Exception {
assertThrows(
InvalidProtocolBufferException.class,
() -> TypeRegistry.getEmptyTypeRegistry().getDescriptorForTypeUrl("UnknownType"));
assertThrows(
InvalidProtocolBufferException.class,
() -> TypeRegistry.getEmptyTypeRegistry().getDescriptorForTypeUrl("///"));
}
@Test
public void findDescriptorByFullName() throws Exception {
Descriptor descriptor = UnittestProto.TestAllTypes.getDescriptor();

@ -42,7 +42,7 @@ genrule(
for wkt in _OBJC_WKT_NAMES
for ext in _OBJC_EXTS
]),
exec_tools = ["//:protoc"],
tools = ["//:protoc"],
tags = ["manual"],
)

@ -110,8 +110,8 @@ def protobuf_deps():
_github_archive(
name = "rules_ruby",
repo = "https://github.com/protocolbuffers/rules_ruby",
commit = "8fca842a3006c3d637114aba4f6bf9695bb3a432",
sha256 = "2619f9a23cee6f6a198d9ef284b6f6cbc901545ee9a9aac9ffa6b83dbf17cf0c",
commit = "b7f3e9756f3c45527be27bc38840d5a1ba690436",
sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8",
)
if not native.existing_rule("rules_jvm_external"):

8
ruby/.gitignore vendored

@ -6,4 +6,10 @@ protobuf-jruby.iml
target/
pkg/
tmp/
tests/google/
tests/google/
ext/google/protobuf_c/third_party/utf8_range/utf8_range.h
ext/google/protobuf_c/third_party/utf8_range/range2-sse.c
ext/google/protobuf_c/third_party/utf8_range/range2-neon.c
ext/google/protobuf_c/third_party/utf8_range/naive.c
ext/google/protobuf_c/third_party/utf8_range/LICENSE
lib/google/protobuf/*_pb.rb

@ -2,6 +2,8 @@
#
# See also code generation logic under /src/google/protobuf/compiler/ruby.
load("@bazel_skylib//lib:selects.bzl", "selects")
load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
load("@rules_ruby//ruby:defs.bzl", "ruby_library")
load("//build_defs:internal_shell.bzl", "inline_sh_binary")
@ -13,12 +15,83 @@ load("//:protobuf_version.bzl", "PROTOBUF_RUBY_VERSION")
# Ruby Runtime
################################################################################
string_flag(
name = "ffi",
build_setting_default = "disabled",
values = [
"enabled",
"disabled",
],
)
config_setting(
name = "ffi_enabled",
flag_values = {
":ffi": "enabled",
},
)
config_setting(
name = "ffi_disabled",
flag_values = {
":ffi": "disabled",
},
)
selects.config_setting_group(
name = "jruby_ffi",
match_all = [
":ffi_enabled",
"@rules_ruby//ruby/runtime:config_jruby",
],
)
selects.config_setting_group(
name = "jruby_native",
match_all = [
":ffi_disabled",
"@rules_ruby//ruby/runtime:config_jruby",
],
)
selects.config_setting_group(
name = "ruby_ffi",
match_all = [
":ffi_enabled",
"@rules_ruby//ruby/runtime:config_ruby",
],
)
selects.config_setting_group(
name = "ruby_native",
match_all = [
":ffi_disabled",
"@rules_ruby//ruby/runtime:config_ruby",
],
)
selects.config_setting_group(
name = "macos_ffi_enabled",
match_all = [
":ffi_enabled",
"@platforms//os:osx",
],
)
selects.config_setting_group(
name = "linux_ffi_enabled",
match_all = [
":ffi_enabled",
"@platforms//os:linux",
],
)
ruby_library(
name = "protobuf",
deps = ["//ruby/lib/google:protobuf_lib"],
visibility = [
"//visibility:public",
],
deps = ["//ruby/lib/google:protobuf_lib"],
)
# Note: these can be greatly simplified using inline_sh_binary in Bazel 6,
@ -27,18 +100,25 @@ ruby_library(
genrule(
name = "jruby_release",
srcs = [
"//ruby/lib/google:copy_jar",
"//ruby/lib/google:dist_files",
"//:well_known_ruby_protos",
"google-protobuf.gemspec",
"@utf8_range//:utf8_range_srcs",
"@utf8_range//:LICENSE",
"//ruby/lib/google:copy_jar",
"//ruby/lib/google:dist_files",
"//ruby/ext/google/protobuf_c:dist_files",
"//:well_known_ruby_protos",
"google-protobuf.gemspec",
],
outs = ["google-protobuf-"+PROTOBUF_RUBY_VERSION+"-java.gem"],
outs = ["google-protobuf-" + PROTOBUF_RUBY_VERSION + "-java.gem"],
cmd = """
set -eux
mkdir tmp
for src in $(SRCS); do
cp --parents -L "$$src" tmp
done
mkdir -p "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range"
for utf in $(execpaths @utf8_range//:utf8_range_srcs) $(execpath @utf8_range//:LICENSE); do
mv "tmp/$$utf" "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range"
done
for wkt in $(execpaths //:well_known_ruby_protos); do
mv "tmp/$$wkt" "tmp/ruby/lib/google/protobuf/"
done
@ -59,14 +139,14 @@ genrule(
genrule(
name = "ruby_release",
srcs = [
"@utf8_range//:utf8_range_srcs",
"@utf8_range//:LICENSE",
"//:well_known_ruby_protos",
"//ruby/ext/google/protobuf_c:dist_files",
"//ruby/lib/google:dist_files",
"google-protobuf.gemspec",
"@utf8_range//:utf8_range_srcs",
"@utf8_range//:LICENSE",
"//:well_known_ruby_protos",
"//ruby/ext/google/protobuf_c:dist_files",
"//ruby/lib/google:dist_files",
"google-protobuf.gemspec",
],
outs = ["google-protobuf-"+PROTOBUF_RUBY_VERSION+".gem"],
outs = ["google-protobuf-" + PROTOBUF_RUBY_VERSION + ".gem"],
cmd = """
set -eux
mkdir tmp
@ -102,7 +182,6 @@ filegroup(
tags = ["manual"],
)
################################################################################
# Tests
################################################################################
@ -111,34 +190,65 @@ filegroup(
internal_ruby_proto_library(
name = "test_ruby_protos",
srcs = ["//ruby/tests:test_protos"],
deps = ["//:well_known_ruby_protos"],
includes = [".", "src", "ruby/tests"],
includes = [
".",
"ruby/tests",
"src",
],
visibility = [
"//ruby:__subpackages__",
],
deps = ["//:well_known_ruby_protos"],
)
conformance_test(
name = "conformance_test",
failure_list = "//conformance:failure_list_ruby.txt",
target_compatible_with = select({
":ruby_native": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
testee = "//conformance:conformance_ruby",
text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt",
)
conformance_test(
name = "conformance_test_ffi",
env = {
"PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION": "ffi",
},
failure_list = "//conformance:failure_list_ruby.txt",
target_compatible_with = select({
"@rules_ruby//ruby/runtime:config_ruby": [],
":ruby_ffi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
testee = "//conformance:conformance_ruby",
text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt",
)
conformance_test(
name = "conformance_test_jruby",
failure_list = "//conformance:failure_list_jruby.txt",
target_compatible_with = select({
":jruby_native": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
testee = "//conformance:conformance_ruby",
text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt",
)
conformance_test(
name = "conformance_test_jruby_ffi",
env = {
"PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION": "ffi",
},
failure_list = "//conformance:failure_list_jruby_ffi.txt",
target_compatible_with = select({
"@rules_ruby//ruby/runtime:config_jruby": [],
":jruby_ffi": [],
"//conditions:default": ["@platforms//:incompatible"],
}),
testee = "//conformance:conformance_ruby",
text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt",
)
################################################################################
@ -148,15 +258,15 @@ conformance_test(
pkg_files(
name = "dist_files",
srcs = [
"//ruby/ext/google/protobuf_c:dist_files",
"//ruby/lib/google:dist_files",
"//ruby/src/main/java:dist_files",
"//ruby/tests:dist_files",
".gitignore",
"BUILD.bazel",
"Gemfile",
"Rakefile",
"README.md",
"Rakefile",
"//ruby/ext/google/protobuf_c:dist_files",
"//ruby/lib/google:dist_files",
"//ruby/src/main/java:dist_files",
"//ruby/tests:dist_files",
],
strip_prefix = strip_prefix.from_root(""),
visibility = ["//pkg:__pkg__"],

@ -2,10 +2,11 @@ This directory contains the Ruby extension that implements Protocol Buffers
functionality in Ruby.
The Ruby extension makes use of generated Ruby code that defines message and
enum types in a Ruby DSL. You may write definitions in this DSL directly, but
we recommend using protoc's Ruby generation support with .proto files. The
build process in this directory only installs the extension; you need to
install protoc as well to have Ruby code generation functionality.
enum types in a Ruby DSL. You may write definitions in this DSL directly, but we
recommend using protoc's Ruby generation support with .proto files. The build
process in this directory only installs the extension; you need to install
protoc as well to have Ruby code generation functionality. You can build protoc
from source using `bazel build //:protoc`.
Installation from Gem
---------------------
@ -50,6 +51,18 @@ puts MyTestMessage.encode_json(mymessage)
Installation from Source (Building Gem)
---------------------------------------
Protocol Buffers has a new experimental backend that uses the
[ffi](https://github.com/ffi/ffi) gem to provide a unified C-based
implementation across Ruby interpreters based on
[UPB](https://github.com/protocolbuffers/upb). For now, use of the FFI
implementation is opt-in. If any of the following are true, the traditional
platform-native implementations (MRI-ruby based on CRuby, Java based on JRuby)
are used instead of the new FFI-based implementation: 1. `ffi` and
`ffi-compiler` gems are not installed 2. `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`
environment variable has a value other than `FFI` (case-insensitive). 3. FFI is
unable to load the native library at runtime.
To build this Ruby extension, you will need:
@ -81,6 +94,12 @@ To run the specs:
$ rake test
To run the specs while using the FFI-based implementation:
```
$ PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI rake test
```
This gem includes the upb parsing and serialization library as a single-file
amalgamation. It is up-to-date with upb git commit
`535bc2fe2f2b467f59347ffc9449e11e47791257`.
@ -93,6 +112,12 @@ From the project root (rather than the `ruby` directory):
$ bazel test //ruby/tests/...
```
To run tests against the FFI implementation:
```
$ bazel test //ruby/tests/... //ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI
```
Version Number Scheme
---------------------

@ -2,6 +2,7 @@ require "rubygems"
require "rubygems/package_task"
require "rake/extensiontask" unless RUBY_PLATFORM == "java"
require "rake/testtask"
import 'lib/google/tasks/ffi.rake'
spec = Gem::Specification.load("google-protobuf.gemspec")
@ -68,6 +69,24 @@ unless ENV['IN_DOCKER'] == 'true' or ENV['BAZEL'] == 'true'
end
end
task :copy_third_party do
unless File.exist? 'ext/google/protobuf_c/third_party/utf8_range'
FileUtils.mkdir_p 'ext/google/protobuf_c/third_party/utf8_range'
# We need utf8_range in-tree.
if ENV['BAZEL'] == 'true'
utf8_root = '../external/utf8_range'
else
utf8_root = '../third_party/utf8_range'
end
%w[
utf8_range.h naive.c range2-neon.c range2-neon.c range2-sse.c LICENSE
].each do |file|
FileUtils.cp File.join(utf8_root, file),
"ext/google/protobuf_c/third_party/utf8_range"
end
end
end
if RUBY_PLATFORM == "java"
task :clean => :require_mvn do
system("mvn --batch-mode clean")
@ -82,20 +101,6 @@ if RUBY_PLATFORM == "java"
end
else
unless ENV['IN_DOCKER'] == 'true'
# We need utf8_range in-tree.
if ENV['BAZEL'] == 'true'
utf8_root = '../external/utf8_range'
else
utf8_root = '../third_party/utf8_range'
end
FileUtils.mkdir_p("ext/google/protobuf_c")
FileUtils.cp(utf8_root+"/utf8_range.h", "ext/google/protobuf_c")
FileUtils.cp(utf8_root+"/naive.c", "ext/google/protobuf_c")
FileUtils.cp(utf8_root+"/range2-neon.c", "ext/google/protobuf_c")
FileUtils.cp(utf8_root+"/range2-sse.c", "ext/google/protobuf_c")
end
Rake::ExtensionTask.new("protobuf_c", spec) do |ext|
unless RUBY_PLATFORM =~ /darwin/
# TODO: also set "no_native to true" for mac if possible. As is,
@ -133,7 +138,7 @@ else
['x86-mingw32', 'x64-mingw32', 'x64-mingw-ucrt', 'x86_64-linux', 'x86-linux'].each do |plat|
RakeCompilerDock.sh <<-"EOT", platform: plat
bundle && \
IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0
IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0
EOT
end
end
@ -141,7 +146,7 @@ else
if RUBY_PLATFORM =~ /darwin/
task 'gem:native' do
system "rake genproto"
system "rake cross native gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0"
system "rake cross native gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0"
end
else
task 'gem:native' => [:genproto, 'gem:windows', 'gem:java']
@ -152,6 +157,14 @@ task :genproto => genproto_output
task :clean do
sh "rm -f #{genproto_output.join(' ')}"
sh "rm -f google-protobuf-*gem"
sh "rm -f Gemfile.lock"
sh "rm -rf pkg"
sh "rm -rf tmp"
# Handles third_party and any platform specific directories built by FFI
Pathname('ext/google/protobuf_c').children.select(&:directory?).each do |dir|
sh "rm -rf #{dir}"
end
end
Gem::PackageTask.new(spec) do |pkg|
@ -169,7 +182,8 @@ Rake::TestTask.new(:gc_test => ENV['BAZEL'] == 'true' ? [] : :build) do |t|
t.test_files = FileList["tests/gc_test.rb"]
end
task :build => [:clean, :genproto, :compile]
task :build => [:clean, :genproto, :copy_third_party, :compile, :"ffi-protobuf:default"]
Rake::Task[:gem].enhance [:copy_third_party, :genproto]
task :default => [:build]
# vim:sw=2:et

@ -6,31 +6,115 @@ package(default_visibility = ["//ruby:__subpackages__"])
cc_library(
name = "protobuf_c",
srcs = glob([
"*.h",
"*.c",
]),
srcs = [
"convert.c",
"convert.h",
"defs.c",
"defs.h",
"map.c",
"map.h",
"message.c",
"message.h",
"protobuf.c",
"protobuf.h",
"repeated_field.c",
"repeated_field.h",
"ruby-upb.c",
"ruby-upb.h",
"shared_convert.c",
"shared_convert.h",
"shared_message.c",
"shared_message.h",
"wrap_memcpy.c",
],
linkstatic = True,
target_compatible_with = select({
"@rules_ruby//ruby/runtime:config_jruby": ["@platforms//:incompatible"],
"//conditions:default": [],
}),
deps = [
"@rules_ruby//ruby/runtime:headers",
"@utf8_range//:utf8_range",
"@utf8_range",
],
alwayslink = True,
)
# Needs to be compiled with UPB_BUILD_API in order to expose functions called
# via FFI directly by Ruby.
cc_library(
name = "upb_api",
srcs = [
"ruby-upb.c",
],
hdrs = [
"ruby-upb.h",
],
copts = ["-fvisibility=hidden"],
linkstatic = False,
local_defines = [
"UPB_BUILD_API",
],
target_compatible_with = select({
"@rules_ruby//ruby/runtime:config_jruby": ["@platforms//:incompatible"],
"//ruby:ffi_disabled": ["@platforms//:incompatible"],
"//conditions:default": [],
}),
deps = [
"@utf8_range",
],
)
cc_library(
name = "protobuf_c_ffi",
srcs = [
"glue.c",
"shared_convert.c",
"shared_convert.h",
"shared_message.c",
"shared_message.h",
],
copts = [
"-std=gnu99",
"-O3",
"-Wall",
"-Wsign-compare",
"-Wno-declaration-after-statement",
],
linkstatic = True,
alwayslink = True,
local_defines = [
"NDEBUG",
],
target_compatible_with = select({
"//ruby:ffi_disabled": ["@platforms//:incompatible"],
"//conditions:default": [],
}),
deps = [":upb_api"],
alwayslink = 1,
)
apple_binary(
name = "bundle",
name = "ffi_bundle",
binary_type = "loadable_bundle",
linkopts = [
"-undefined,dynamic_lookup",
"-multiply_defined,suppress",
],
minimum_os_version = "10.11",
platform_type = "macos",
tags = ["manual"],
deps = [
":protobuf_c_ffi",
],
)
apple_binary(
name = "bundle",
binary_type = "loadable_bundle",
linkopts = [
"-undefined,dynamic_lookup",
"-multiply_defined,suppress",
],
minimum_os_version = "10.11",
platform_type = "macos",
tags = ["manual"],
deps = [
":protobuf_c",
@ -43,6 +127,7 @@ pkg_files(
"*.h",
"*.c",
"*.rb",
"Rakefile",
]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//ruby:__pkg__"],
@ -65,8 +150,8 @@ genrule(
staleness_test(
name = "test_amalgamation_staleness",
outs = [
"ruby-upb.h",
"ruby-upb.c",
"ruby-upb.h",
],
generated_pattern = "generated-in/%s",
)

@ -0,0 +1,3 @@
import '../../../lib/google/tasks/ffi.rake'
task default: ['ffi-protobuf:default']

@ -41,6 +41,7 @@
#include "message.h"
#include "protobuf.h"
#include "shared_convert.h"
static upb_StringView Convert_StringData(VALUE str, upb_Arena* arena) {
upb_StringView ret;
@ -111,8 +112,7 @@ static int32_t Convert_ToEnum(VALUE value, const char* name,
case T_SYMBOL: {
const upb_EnumValueDef* ev =
upb_EnumDef_FindValueByName(e, rb_id2name(SYM2ID(value)));
if (!ev)
goto unknownval;
if (!ev) goto unknownval;
val = upb_EnumValueDef_Number(ev);
break;
}
@ -255,7 +255,7 @@ VALUE Convert_UpbToRuby(upb_MessageValue upb_val, TypeInfo type_info,
case kUpb_CType_UInt64:
return ULL2NUM(upb_val.int64_val);
case kUpb_CType_Enum: {
const upb_EnumValueDef *ev = upb_EnumDef_FindValueByNumber(
const upb_EnumValueDef* ev = upb_EnumDef_FindValueByNumber(
type_info.def.enumdef, upb_val.int32_val);
if (ev) {
return ID2SYM(rb_intern(upb_EnumValueDef_Name(ev)));
@ -312,50 +312,26 @@ upb_MessageValue Msgval_DeepCopy(upb_MessageValue msgval, TypeInfo type_info,
bool Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
TypeInfo type_info) {
switch (type_info.type) {
case kUpb_CType_Bool:
return memcmp(&val1, &val2, 1) == 0;
case kUpb_CType_Float:
case kUpb_CType_Int32:
case kUpb_CType_UInt32:
case kUpb_CType_Enum:
return memcmp(&val1, &val2, 4) == 0;
case kUpb_CType_Double:
case kUpb_CType_Int64:
case kUpb_CType_UInt64:
return memcmp(&val1, &val2, 8) == 0;
case kUpb_CType_String:
case kUpb_CType_Bytes:
return val1.str_val.size == val2.str_val.size &&
memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) ==
0;
case kUpb_CType_Message:
return Message_Equal(val1.msg_val, val2.msg_val, type_info.def.msgdef);
default:
rb_raise(rb_eRuntimeError, "Internal error, unexpected type");
upb_Status status;
upb_Status_Clear(&status);
bool return_value = shared_Msgval_IsEqual(val1, val2, type_info.type,
type_info.def.msgdef, &status);
if (upb_Status_IsOk(&status)) {
return return_value;
} else {
rb_raise(rb_eRuntimeError, upb_Status_ErrorMessage(&status));
}
}
uint64_t Msgval_GetHash(upb_MessageValue val, TypeInfo type_info,
uint64_t seed) {
switch (type_info.type) {
case kUpb_CType_Bool:
return _upb_Hash(&val, 1, seed);
case kUpb_CType_Float:
case kUpb_CType_Int32:
case kUpb_CType_UInt32:
case kUpb_CType_Enum:
return _upb_Hash(&val, 4, seed);
case kUpb_CType_Double:
case kUpb_CType_Int64:
case kUpb_CType_UInt64:
return _upb_Hash(&val, 8, seed);
case kUpb_CType_String:
case kUpb_CType_Bytes:
return _upb_Hash(val.str_val.data, val.str_val.size, seed);
case kUpb_CType_Message:
return Message_Hash(val.msg_val, type_info.def.msgdef, seed);
default:
rb_raise(rb_eRuntimeError, "Internal error, unexpected type");
upb_Status status;
upb_Status_Clear(&status);
uint64_t return_value = shared_Msgval_GetHash(
val, type_info.type, type_info.def.msgdef, seed, &status);
if (upb_Status_IsOk(&status)) {
return return_value;
} else {
rb_raise(rb_eRuntimeError, upb_Status_ErrorMessage(&status));
}
}

@ -22,6 +22,7 @@ $INCFLAGS += " -I$(srcdir)/third_party/utf8_range"
$srcs = ["protobuf.c", "convert.c", "defs.c", "message.c",
"repeated_field.c", "map.c", "ruby-upb.c", "wrap_memcpy.c",
"naive.c", "range2-neon.c", "range2-sse.c"]
"naive.c", "range2-neon.c", "range2-sse.c", "shared_convert.c",
"shared_message.c"]
create_makefile(ext_name)

@ -0,0 +1,44 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// -----------------------------------------------------------------------------
// Exposing inlined UPB functions. Strictly free of dependencies on
// Ruby interpreter internals.
#include "ruby-upb.h"
upb_Arena* Arena_create() { return upb_Arena_Init(NULL, 0, &upb_alloc_global); }
google_protobuf_FileDescriptorProto* FileDescriptorProto_parse(
const char* serialized_file_proto, size_t length) {
upb_Arena* arena = Arena_create();
return google_protobuf_FileDescriptorProto_parse(serialized_file_proto,
length, arena);
}

@ -35,6 +35,7 @@
#include "map.h"
#include "protobuf.h"
#include "repeated_field.h"
#include "shared_message.h"
static VALUE cParseError = Qnil;
static VALUE cAbstractMessage = Qnil;
@ -694,29 +695,13 @@ static VALUE Message_dup(VALUE _self) {
// Support function for Message_eq, and also used by other #eq functions.
bool Message_Equal(const upb_Message* m1, const upb_Message* m2,
const upb_MessageDef* m) {
if (m1 == m2) return true;
size_t size1, size2;
int encode_opts =
kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic;
upb_Arena* arena_tmp = upb_Arena_New();
const upb_MiniTable* layout = upb_MessageDef_MiniTable(m);
// Compare deterministically serialized payloads with no unknown fields.
char* data1;
char* data2;
upb_EncodeStatus status1 =
upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1);
upb_EncodeStatus status2 =
upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2);
if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) {
bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0);
upb_Arena_Free(arena_tmp);
return ret;
upb_Status status;
upb_Status_Clear(&status);
bool return_value = shared_Message_Equal(m1, m2, m, &status);
if (upb_Status_IsOk(&status)) {
return return_value;
} else {
upb_Arena_Free(arena_tmp);
rb_raise(cParseError, "Error comparing messages");
rb_raise(cParseError, upb_Status_ErrorMessage(&status));
}
}
@ -741,23 +726,13 @@ static VALUE Message_eq(VALUE _self, VALUE _other) {
uint64_t Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
uint64_t seed) {
upb_Arena* arena = upb_Arena_New();
char* data;
size_t size;
// Hash a deterministically serialized payloads with no unknown fields.
upb_EncodeStatus status = upb_Encode(
msg, upb_MessageDef_MiniTable(m),
kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena,
&data, &size);
if (status == kUpb_EncodeStatus_Ok) {
uint64_t ret = _upb_Hash(data, size, seed);
upb_Arena_Free(arena);
return ret;
upb_Status status;
upb_Status_Clear(&status);
uint64_t return_value = shared_Message_Hash(msg, m, seed, &status);
if (upb_Status_IsOk(&status)) {
return return_value;
} else {
upb_Arena_Free(arena);
rb_raise(cParseError, "Error calculating hash");
rb_raise(cParseError, upb_Status_ErrorMessage(&status));
}
}

@ -0,0 +1,87 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// -----------------------------------------------------------------------------
// Ruby <-> upb data conversion functions. Strictly free of dependencies on
// Ruby interpreter internals.
#include "shared_convert.h"
bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
upb_CType type, upb_MessageDef* msgdef,
upb_Status* status) {
switch (type) {
case kUpb_CType_Bool:
return memcmp(&val1, &val2, 1) == 0;
case kUpb_CType_Float:
case kUpb_CType_Int32:
case kUpb_CType_UInt32:
case kUpb_CType_Enum:
return memcmp(&val1, &val2, 4) == 0;
case kUpb_CType_Double:
case kUpb_CType_Int64:
case kUpb_CType_UInt64:
return memcmp(&val1, &val2, 8) == 0;
case kUpb_CType_String:
case kUpb_CType_Bytes:
return val1.str_val.size == val2.str_val.size &&
memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) ==
0;
case kUpb_CType_Message:
return shared_Message_Equal(val1.msg_val, val2.msg_val, msgdef, status);
default:
upb_Status_SetErrorMessage(status, "Internal error, unexpected type");
}
}
uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type,
upb_MessageDef* msgdef, uint64_t seed,
upb_Status* status) {
switch (type) {
case kUpb_CType_Bool:
return _upb_Hash(&val, 1, seed);
case kUpb_CType_Float:
case kUpb_CType_Int32:
case kUpb_CType_UInt32:
case kUpb_CType_Enum:
return _upb_Hash(&val, 4, seed);
case kUpb_CType_Double:
case kUpb_CType_Int64:
case kUpb_CType_UInt64:
return _upb_Hash(&val, 8, seed);
case kUpb_CType_String:
case kUpb_CType_Bytes:
return _upb_Hash(val.str_val.data, val.str_val.size, seed);
case kUpb_CType_Message:
return shared_Message_Hash(val.msg_val, msgdef, seed, status);
default:
upb_Status_SetErrorMessage(status, "Internal error, unexpected type");
}
}

@ -0,0 +1,49 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// -----------------------------------------------------------------------------
// Ruby <-> upb data conversion functions. Strictly free of dependencies on
// Ruby interpreter internals.
#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_
#define RUBY_PROTOBUF_SHARED_CONVERT_H_
#include "ruby-upb.h"
#include "shared_message.h"
bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2,
upb_CType type, upb_MessageDef* msgdef,
upb_Status* status);
uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type,
upb_MessageDef* msgdef, uint64_t seed,
upb_Status* status);
#endif // RUBY_PROTOBUF_SHARED_CONVERT_H_

@ -0,0 +1,88 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// -----------------------------------------------------------------------------
// Ruby Message functions. Strictly free of dependencies on
// Ruby interpreter internals.
#include "shared_message.h"
// Support function for Message_Hash. Returns a hash value for the given
// message.
uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
uint64_t seed, upb_Status* status) {
upb_Arena* arena = upb_Arena_New();
char* data;
size_t size;
// Hash a deterministically serialized payloads with no unknown fields.
upb_EncodeStatus encode_status = upb_Encode(
msg, upb_MessageDef_MiniTable(m),
kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena,
&data, &size);
if (encode_status == kUpb_EncodeStatus_Ok) {
uint64_t ret = _upb_Hash(data, size, seed);
upb_Arena_Free(arena);
return ret;
} else {
upb_Arena_Free(arena);
upb_Status_SetErrorMessage(status, "Error calculating hash");
}
}
// Support function for Message_Equal
bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2,
const upb_MessageDef* m, upb_Status* status) {
if (m1 == m2) return true;
size_t size1, size2;
int encode_opts =
kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic;
upb_Arena* arena_tmp = upb_Arena_New();
const upb_MiniTable* layout = upb_MessageDef_MiniTable(m);
// Compare deterministically serialized payloads with no unknown fields.
char* data1;
char* data2;
upb_EncodeStatus status1 =
upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1);
upb_EncodeStatus status2 =
upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2);
if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) {
bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0);
upb_Arena_Free(arena_tmp);
return ret;
} else {
upb_Arena_Free(arena_tmp);
upb_Status_SetErrorMessage(status, "Error comparing messages");
}
}

@ -0,0 +1,48 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// -----------------------------------------------------------------------------
// Ruby Message functions. Strictly free of dependencies on
// Ruby interpreter internals.
#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_
#define RUBY_PROTOBUF_SHARED_MESSAGE_H_
#include "ruby-upb.h"
// Returns a hash value for the given message.
uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m,
uint64_t seed, upb_Status* status);
// Returns true if these two messages are equal.
bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2,
const upb_MessageDef* m, upb_Status* status);
#endif // RUBY_PROTOBUF_SHARED_MESSAGE_H_

@ -10,15 +10,31 @@ Gem::Specification.new do |s|
s.email = "protobuf@googlegroups.com"
s.metadata = { "source_code_uri" => "https://github.com/protocolbuffers/protobuf/tree/#{git_tag}/ruby" }
s.require_paths = ["lib"]
s.files = Dir.glob('lib/**/*.rb')
s.files = Dir.glob('lib/**/*.{rb,rake}')
if RUBY_PLATFORM == "java"
s.platform = "java"
s.files += ["lib/google/protobuf_java.jar"]
s.files += ["lib/google/protobuf_java.jar"] +
Dir.glob('ext/**/*').reject do |file|
File.basename(file) =~ /^((convert|defs|map|repeated_field)\.[ch]|
BUILD\.bazel|extconf\.rb|wrap_memcpy\.c)$/x
end
s.extensions = ["ext/google/protobuf_c/Rakefile"]
s.add_dependency "ffi", "~>1"
s.add_dependency "ffi-compiler", "~>1"
else
s.files += Dir.glob('ext/**/*')
s.extensions= ["ext/google/protobuf_c/extconf.rb"]
s.add_development_dependency "rake-compiler-dock", "= 1.2.1" end
s.files += Dir.glob('ext/**/*').reject do |file|
File.basename(file) =~ /^(BUILD\.bazel)$/
end
s.extensions = %w[
ext/google/protobuf_c/extconf.rb
ext/google/protobuf_c/Rakefile
]
s.add_development_dependency "rake-compiler-dock", "= 1.2.1"
end
s.required_ruby_version = '>= 2.5'
s.add_development_dependency "rake", "~> 13"
s.add_development_dependency "ffi", "~>1"
s.add_development_dependency "ffi-compiler", "~>1"
s.add_development_dependency "rake-compiler", "~> 1.1.0"
s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9'
end

@ -8,9 +8,16 @@ config_setting(
cc_binary(
name = "protobuf_c.so",
linkshared = 1,
tags = ["manual"],
deps = ["//ruby/ext/google/protobuf_c"],
)
cc_binary(
name = "libprotobuf_c_ffi.so",
linkshared = 1,
tags = ["manual"],
deps = ["//ruby/ext/google/protobuf_c:protobuf_c_ffi"],
)
# Move the bundle to the location expected by our Ruby files.
@ -22,13 +29,25 @@ genrule(
tags = ["manual"],
)
# Move the bundle to the location expected by our Ruby files.
genrule(
name = "copy_ffi_bundle",
srcs = ["//ruby/ext/google/protobuf_c:ffi_bundle"],
outs = ["libprotobuf_c_ffi.bundle"],
cmd = "cp $< $@",
tags = ["manual"],
visibility = [
"//ruby:__subpackages__",
],
)
java_binary(
name = "protobuf_java_bin",
create_executable = False,
deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"],
runtime_deps = [
"//ruby/src/main/java:protobuf_java"
"//ruby/src/main/java:protobuf_java",
],
deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"],
create_executable = False,
)
# Move the jar to the location expected by our Ruby files.
@ -46,19 +65,34 @@ ruby_library(
srcs = glob([
"**/*.rb",
]),
deps = ["//:well_known_ruby_protos"],
includes = ["ruby/lib"],
data = select({
# Platform native implementations
"@rules_ruby//ruby/runtime:config_jruby": ["protobuf_java.jar"],
"@platforms//os:osx": ["protobuf_c.bundle"],
"//conditions:default": ["protobuf_c.so"],
}) + select({
# FFI Implementations
"//ruby:macos_ffi_enabled": ["libprotobuf_c_ffi.bundle"],
"//ruby:linux_ffi_enabled": ["libprotobuf_c_ffi.so"],
"//conditions:default": [],
}),
includes = ["ruby/lib"],
visibility = ["//ruby:__pkg__"],
deps = ["//:well_known_ruby_protos"] + select({
"//ruby:ffi_enabled": [
"@protobuf_bundle//:ffi",
"@protobuf_bundle//:ffi-compiler",
],
"//conditions:default": [],
}),
)
pkg_files(
name = "dist_files",
srcs = glob(["**/*.rb"]),
srcs = glob([
"**/*.rb",
"**/*.rake",
]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//ruby:__pkg__"],
)

@ -39,26 +39,16 @@ module Google
class Error < StandardError; end
class ParseError < Error; end
class TypeError < ::TypeError; end
end
end
if RUBY_PLATFORM == "java"
require 'json'
require 'google/protobuf_java'
else
begin
require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c"
rescue LoadError
require 'google/protobuf_c'
end
end
require 'google/protobuf/descriptor_dsl'
require 'google/protobuf/repeated_field'
module Google
module Protobuf
PREFER_FFI = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']
when nil, "", /^native$/i
false
when /^ffi$/i
true
else
warn "Unexpected value `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}` for environment variable `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`. Should be either \"FFI\", \"NATIVE\"."
false
end
def self.encode(msg, options = {})
msg.to_proto(options)
@ -76,5 +66,19 @@ module Google
klass.decode_json(json, options)
end
IMPLEMENTATION = if PREFER_FFI
begin
require 'google/protobuf_ffi'
:FFI
rescue LoadError
warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf."
warn "Falling back to native implementation."
require 'google/protobuf_native'
:NATIVE
end
else
require 'google/protobuf_native'
:NATIVE
end
end
end

@ -0,0 +1,177 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
##
# Message Descriptor - Descriptor for short.
class Descriptor
attr :descriptor_pool, :msg_class
include Enumerable
# FFI Interface methods and setup
extend ::FFI::DataConverter
native_type ::FFI::Type::POINTER
class << self
prepend Google::Protobuf::Internal::TypeSafety
include Google::Protobuf::Internal::PointerHelper
# @param value [Descriptor] Descriptor to convert to an FFI native type
# @param _ [Object] Unused
def to_native(value, _ = nil)
msg_def_ptr = value.nil? ? nil : value.instance_variable_get(:@msg_def)
return ::FFI::Pointer::NULL if msg_def_ptr.nil?
raise "Underlying msg_def was null!" if msg_def_ptr.null?
msg_def_ptr
end
##
# @param msg_def [::FFI::Pointer] MsgDef pointer to be wrapped
# @param _ [Object] Unused
def from_native(msg_def, _ = nil)
return nil if msg_def.nil? or msg_def.null?
file_def = Google::Protobuf::FFI.get_message_file_def msg_def
descriptor_from_file_def(file_def, msg_def)
end
end
def to_native
self.class.to_native(self)
end
##
# Great write up of this strategy:
# See https://blog.appsignal.com/2018/08/07/ruby-magic-changing-the-way-ruby-creates-objects.html
def self.new(*arguments, &block)
raise "Descriptor objects may not be created from Ruby."
end
def to_s
inspect
end
def inspect
"Descriptor - (not the message class) #{name}"
end
def file_descriptor
@descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_message_file_def(@msg_def))
end
def name
@name ||= Google::Protobuf::FFI.get_message_fullname(self)
end
def each_oneof &block
n = Google::Protobuf::FFI.oneof_count(self)
0.upto(n-1) do |i|
yield(Google::Protobuf::FFI.get_oneof_by_index(self, i))
end
nil
end
def each &block
n = Google::Protobuf::FFI.field_count(self)
0.upto(n-1) do |i|
yield(Google::Protobuf::FFI.get_field_by_index(self, i))
end
nil
end
def lookup(name)
Google::Protobuf::FFI.get_field_by_name(self, name, name.size)
end
def lookup_oneof(name)
Google::Protobuf::FFI.get_oneof_by_name(self, name, name.size)
end
def msgclass
@msg_class ||= build_message_class
end
private
extend Google::Protobuf::Internal::Convert
def initialize(msg_def, descriptor_pool)
@msg_def = msg_def
@msg_class = nil
@descriptor_pool = descriptor_pool
end
def self.private_constructor(msg_def, descriptor_pool)
instance = allocate
instance.send(:initialize, msg_def, descriptor_pool)
instance
end
def wrapper?
if defined? @wrapper
@wrapper
else
@wrapper = case Google::Protobuf::FFI.get_well_known_type self
when :DoubleValue, :FloatValue, :Int64Value, :UInt64Value, :Int32Value, :UInt32Value, :StringValue, :BytesValue, :BoolValue
true
else
false
end
end
end
def self.get_message(msg, descriptor, arena)
return nil if msg.nil? or msg.null?
message = OBJECT_CACHE.get(msg.address)
if message.nil?
message = descriptor.msgclass.send(:private_constructor, arena, msg: msg)
end
message
end
def pool
@descriptor_pool
end
end
class FFI
# MessageDef
attach_function :new_message_from_def, :upb_Message_New, [Descriptor, Internal::Arena], :Message
attach_function :field_count, :upb_MessageDef_FieldCount, [Descriptor], :int
attach_function :get_message_file_def, :upb_MessageDef_File, [:pointer], :FileDef
attach_function :get_message_fullname, :upb_MessageDef_FullName, [Descriptor], :string
attach_function :get_mini_table, :upb_MessageDef_MiniTable, [Descriptor], MiniTable.ptr
attach_function :oneof_count, :upb_MessageDef_OneofCount, [Descriptor], :int
attach_function :get_well_known_type, :upb_MessageDef_WellKnownType, [Descriptor], WellKnown
attach_function :message_def_syntax, :upb_MessageDef_Syntax, [Descriptor], Syntax
attach_function :find_msg_def_by_name, :upb_MessageDef_FindByNameWithSize, [Descriptor, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool
end
end
end

@ -0,0 +1,93 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class FFI
# DefPool
attach_function :add_serialized_file, :upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef
attach_function :free_descriptor_pool, :upb_DefPool_Free, [:DefPool], :void
attach_function :create_descriptor_pool,:upb_DefPool_New, [], :DefPool
attach_function :lookup_enum, :upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDescriptor
attach_function :lookup_msg, :upb_DefPool_FindMessageByName, [:DefPool, :string], Descriptor
# FileDescriptorProto
attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto
end
class DescriptorPool
attr :descriptor_pool
attr_accessor :descriptor_class_by_def
def initialize
@descriptor_pool = ::FFI::AutoPointer.new(Google::Protobuf::FFI.create_descriptor_pool, Google::Protobuf::FFI.method(:free_descriptor_pool))
@descriptor_class_by_def = {}
# Should always be the last expression of the initializer to avoid
# leaking references to this object before construction is complete.
Google::Protobuf::OBJECT_CACHE.try_add @descriptor_pool.address, self
end
def add_serialized_file(file_contents)
# Allocate memory sized to file_contents
memBuf = ::FFI::MemoryPointer.new(:char, file_contents.bytesize)
# Insert the data
memBuf.put_bytes(0, file_contents)
file_descriptor_proto = Google::Protobuf::FFI.parse memBuf, file_contents.bytesize
raise ArgumentError.new("Unable to parse FileDescriptorProto") if file_descriptor_proto.null?
status = Google::Protobuf::FFI::Status.new
file_descriptor = Google::Protobuf::FFI.add_serialized_file @descriptor_pool, file_descriptor_proto, status
if file_descriptor.null?
raise TypeError.new("Unable to build file to DescriptorPool: #{Google::Protobuf::FFI.error_message(status)}")
else
@descriptor_class_by_def[file_descriptor.address] = FileDescriptor.new file_descriptor, self
end
end
def lookup name
Google::Protobuf::FFI.lookup_msg(@descriptor_pool, name) ||
Google::Protobuf::FFI.lookup_enum(@descriptor_pool, name)
end
def self.generated_pool
@@generated_pool ||= DescriptorPool.new
end
private
# Implementation details below are subject to breaking changes without
# warning and are intended for use only within the gem.
def get_file_descriptor file_def
return nil if file_def.null?
@descriptor_class_by_def[file_def.address] ||= FileDescriptor.new(file_def, self)
end
end
end
end

@ -0,0 +1,184 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class EnumDescriptor
attr :descriptor_pool, :enum_def
include Enumerable
# FFI Interface methods and setup
extend ::FFI::DataConverter
native_type ::FFI::Type::POINTER
class << self
prepend Google::Protobuf::Internal::TypeSafety
include Google::Protobuf::Internal::PointerHelper
# @param value [Arena] Arena to convert to an FFI native type
# @param _ [Object] Unused
def to_native(value, _)
value.instance_variable_get(:@enum_def) || ::FFI::Pointer::NULL
end
##
# @param enum_def [::FFI::Pointer] EnumDef pointer to be wrapped
# @param _ [Object] Unused
def from_native(enum_def, _)
return nil if enum_def.nil? or enum_def.null?
file_def = Google::Protobuf::FFI.get_message_file_def enum_def
descriptor_from_file_def(file_def, enum_def)
end
end
def self.new(*arguments, &block)
raise "Descriptor objects may not be created from Ruby."
end
def file_descriptor
@descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_enum_file_descriptor(self))
end
def name
Google::Protobuf::FFI.get_enum_fullname(self)
end
def to_s
inspect
end
def inspect
"#{self.class.name}: #{name}"
end
def lookup_name(name)
self.class.send(:lookup_name, self, name)
end
def lookup_value(number)
self.class.send(:lookup_value, self, number)
end
def each &block
n = Google::Protobuf::FFI.enum_value_count(self)
0.upto(n - 1) do |i|
enum_value = Google::Protobuf::FFI.enum_value_by_index(self, i)
yield(Google::Protobuf::FFI.enum_name(enum_value).to_sym, Google::Protobuf::FFI.enum_number(enum_value))
end
nil
end
def enummodule
if @module.nil?
@module = build_enum_module
end
@module
end
private
def initialize(enum_def, descriptor_pool)
@descriptor_pool = descriptor_pool
@enum_def = enum_def
@module = nil
end
def self.private_constructor(enum_def, descriptor_pool)
instance = allocate
instance.send(:initialize, enum_def, descriptor_pool)
instance
end
def self.lookup_value(enum_def, number)
enum_value = Google::Protobuf::FFI.enum_value_by_number(enum_def, number)
if enum_value.null?
nil
else
Google::Protobuf::FFI.enum_name(enum_value).to_sym
end
end
def self.lookup_name(enum_def, name)
enum_value = Google::Protobuf::FFI.enum_value_by_name(enum_def, name.to_s, name.size)
if enum_value.null?
nil
else
Google::Protobuf::FFI.enum_number(enum_value)
end
end
def build_enum_module
descriptor = self
dynamic_module = Module.new do
@descriptor = descriptor
class << self
attr_accessor :descriptor
end
def self.lookup(number)
descriptor.lookup_value number
end
def self.resolve(name)
descriptor.lookup_name name
end
end
self.each do |name, value|
if name[0] < 'A' || name[0] > 'Z'
if name[0] >= 'a' and name[0] <= 'z'
name = name[0].upcase + name[1..-1] # auto capitalize
else
warn(
"Enum value '#{name}' does not start with an uppercase letter " +
"as is required for Ruby constants.")
next
end
end
dynamic_module.const_set(name.to_sym, value)
end
dynamic_module
end
end
class FFI
# EnumDescriptor
attach_function :get_enum_file_descriptor, :upb_EnumDef_File, [EnumDescriptor], :FileDef
attach_function :enum_value_by_name, :upb_EnumDef_FindValueByNameWithSize,[EnumDescriptor, :string, :size_t], :EnumValueDef
attach_function :enum_value_by_number, :upb_EnumDef_FindValueByNumber, [EnumDescriptor, :int], :EnumValueDef
attach_function :get_enum_fullname, :upb_EnumDef_FullName, [EnumDescriptor], :string
attach_function :enum_value_by_index, :upb_EnumDef_Value, [EnumDescriptor, :int], :EnumValueDef
attach_function :enum_value_count, :upb_EnumDef_ValueCount, [EnumDescriptor], :int
attach_function :enum_name, :upb_EnumValueDef_Name, [:EnumValueDef], :string
attach_function :enum_number, :upb_EnumValueDef_Number, [:EnumValueDef], :int
end
end
end

@ -0,0 +1,236 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class FFI
extend ::FFI::Library
# Workaround for Bazel's use of symlinks + JRuby's __FILE__ and `caller`
# that resolves them.
if ENV['BAZEL'] == 'true'
ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c_ffi', ENV['PWD']
else
ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c_ffi'
end
## Map
Upb_Map_Begin = -1
## Encoding Status
Upb_Status_MaxMessage = 127
Upb_Encode_Deterministic = 1
Upb_Encode_SkipUnknown = 2
## JSON Encoding options
# When set, emits 0/default values. TODO(haberman): proto3 only?
Upb_JsonEncode_EmitDefaults = 1
# When set, use normal (snake_case) field names instead of JSON (camelCase) names.
Upb_JsonEncode_UseProtoNames = 2
# When set, emits enums as their integer values instead of as their names.
Upb_JsonEncode_FormatEnumsAsIntegers = 4
## JSON Decoding options
Upb_JsonDecode_IgnoreUnknown = 1
typedef :pointer, :Array
typedef :pointer, :DefPool
typedef :pointer, :EnumValueDef
typedef :pointer, :ExtensionRegistry
typedef :pointer, :FieldDefPointer
typedef :pointer, :FileDef
typedef :pointer, :FileDescriptorProto
typedef :pointer, :Map
typedef :pointer, :Message # Instances of a message
typedef :pointer, :OneofDefPointer
typedef :pointer, :binary_string
if ::FFI::Platform::ARCH == "aarch64"
typedef :u_int8_t, :uint8_t
typedef :u_int16_t, :uint16_t
typedef :u_int32_t, :uint32_t
typedef :u_int64_t, :uint64_t
end
FieldType = enum(
:double, 1,
:float,
:int64,
:uint64,
:int32,
:fixed64,
:fixed32,
:bool,
:string,
:group,
:message,
:bytes,
:uint32,
:enum,
:sfixed32,
:sfixed64,
:sint32,
:sint64
)
CType = enum(
:bool, 1,
:float,
:int32,
:uint32,
:enum,
:message,
:double,
:int64,
:uint64,
:string,
:bytes
)
Label = enum(
:optional, 1,
:required,
:repeated
)
Syntax = enum(
:Proto2, 2,
:Proto3
)
# All the different kind of well known type messages. For simplicity of check,
# number wrappers and string wrappers are grouped together. Make sure the
# order and merber of these groups are not changed.
WellKnown = enum(
:Unspecified,
:Any,
:FieldMask,
:Duration,
:Timestamp,
# number wrappers
:DoubleValue,
:FloatValue,
:Int64Value,
:UInt64Value,
:Int32Value,
:UInt32Value,
# string wrappers
:StringValue,
:BytesValue,
:BoolValue,
:Value,
:ListValue,
:Struct
)
DecodeStatus = enum(
:Ok,
:Malformed, # Wire format was corrupt
:OutOfMemory, # Arena alloc failed
:BadUtf8, # String field had bad UTF-8
:MaxDepthExceeded, # Exceeded UPB_DECODE_MAXDEPTH
# CheckRequired failed, but the parse otherwise succeeded.
:MissingRequired,
)
EncodeStatus = enum(
:Ok,
:OutOfMemory, # Arena alloc failed
:MaxDepthExceeded, # Exceeded UPB_DECODE_MAXDEPTH
# CheckRequired failed, but the parse otherwise succeeded.
:MissingRequired,
)
class StringView < ::FFI::Struct
layout :data, :pointer,
:size, :size_t
end
class MiniTable < ::FFI::Struct
layout :subs, :pointer,
:fields, :pointer,
:size, :uint16_t,
:field_count, :uint16_t,
:ext, :uint8_t, # upb_ExtMode, declared as uint8_t so sizeof(ext) == 1
:dense_below, :uint8_t,
:table_mask, :uint8_t,
:required_count, :uint8_t # Required fields have the lowest hasbits.
# To statically initialize the tables of variable length, we need a flexible
# array member, and we need to compile in gnu99 mode (constant initialization
# of flexible array members is a GNU extension, not in C99 unfortunately. */
# _upb_FastTable_Entry fasttable[];
end
class Status < ::FFI::Struct
layout :ok, :bool,
:msg, [:char, Upb_Status_MaxMessage]
def initialize
super
FFI.clear self
end
end
class MessageValue < ::FFI::Union
layout :bool_val, :bool,
:float_val, :float,
:double_val, :double,
:int32_val, :int32_t,
:int64_val, :int64_t,
:uint32_val, :uint32_t,
:uint64_val,:uint64_t,
:map_val, :pointer,
:msg_val, :pointer,
:array_val,:pointer,
:str_val, StringView
end
class MutableMessageValue < ::FFI::Union
layout :map, :Map,
:msg, :Message,
:array, :Array
end
# Status
attach_function :clear, :upb_Status_Clear, [Status.by_ref], :void
attach_function :error_message, :upb_Status_ErrorMessage, [Status.by_ref], :string
# Generic
attach_function :memcmp, [:pointer, :pointer, :size_t], :int
attach_function :memcpy, [:pointer, :pointer, :size_t], :int
# Alternatives to pre-processor macros
def self.decode_max_depth(i)
i << 16
end
end
end
end

@ -0,0 +1,332 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class FieldDescriptor
attr :field_def, :descriptor_pool
include Google::Protobuf::Internal::Convert
# FFI Interface methods and setup
extend ::FFI::DataConverter
native_type ::FFI::Type::POINTER
class << self
prepend Google::Protobuf::Internal::TypeSafety
include Google::Protobuf::Internal::PointerHelper
# @param value [FieldDescriptor] FieldDescriptor to convert to an FFI native type
# @param _ [Object] Unused
def to_native(value, _)
field_def_ptr = value.instance_variable_get(:@field_def)
warn "Underlying field_def was nil!" if field_def_ptr.nil?
raise "Underlying field_def was null!" if !field_def_ptr.nil? and field_def_ptr.null?
field_def_ptr
end
##
# @param field_def [::FFI::Pointer] FieldDef pointer to be wrapped
# @param _ [Object] Unused
def from_native(field_def, _ = nil)
return nil if field_def.nil? or field_def.null?
file_def = Google::Protobuf::FFI.file_def_by_raw_field_def(field_def)
descriptor_from_file_def(file_def, field_def)
end
end
def self.new(*arguments, &block)
raise "Descriptor objects may not be created from Ruby."
end
def to_s
inspect
end
def inspect
"#{self.class.name}: #{name}"
end
def name
@name ||= Google::Protobuf::FFI.get_full_name(self)
end
def json_name
@json_name ||= Google::Protobuf::FFI.get_json_name(self)
end
def number
@number ||= Google::Protobuf::FFI.get_number(self)
end
def type
@type ||= Google::Protobuf::FFI.get_type(self)
end
def label
@label ||= Google::Protobuf::FFI::Label[Google::Protobuf::FFI.get_label(self)]
end
def default
return nil if Google::Protobuf::FFI.is_sub_message(self)
if Google::Protobuf::FFI.is_repeated(self)
message_value = Google::Protobuf::FFI::MessageValue.new
else
message_value = Google::Protobuf::FFI.get_default(self)
end
enum_def = Google::Protobuf::FFI.get_subtype_as_enum(self)
if enum_def.null?
convert_upb_to_ruby message_value, c_type
else
convert_upb_to_ruby message_value, c_type, enum_def
end
end
def submsg_name
if defined? @submsg_name
@submsg_name
else
@submsg_name = case c_type
when :enum
Google::Protobuf::FFI.get_enum_fullname Google::Protobuf::FFI.get_subtype_as_enum self
when :message
Google::Protobuf::FFI.get_message_fullname Google::Protobuf::FFI.get_subtype_as_message self
else
nil
end
end
end
##
# Tests if this field has been set on the argument message.
#
# @param msg [Google::Protobuf::Message]
# @return [Object] Value of the field on this message.
# @raise [TypeError] If the field is not defined on this message.
def get(msg)
if msg.class.descriptor == Google::Protobuf::FFI.get_containing_message_def(self)
msg.send :get_field, self
else
raise TypeError.new "get method called on wrong message type"
end
end
def subtype
if defined? @subtype
@subtype
else
@subtype = case c_type
when :enum
Google::Protobuf::FFI.get_subtype_as_enum(self)
when :message
Google::Protobuf::FFI.get_subtype_as_message(self)
else
nil
end
end
end
##
# Tests if this field has been set on the argument message.
#
# @param msg [Google::Protobuf::Message]
# @return [Boolean] True iff message has this field set
# @raise [TypeError] If this field does not exist on the message
# @raise [ArgumentError] If this field does not track presence
def has?(msg)
if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self)
raise TypeError.new "has method called on wrong message type"
end
unless has_presence?
raise ArgumentError.new "does not track presence"
end
Google::Protobuf::FFI.get_message_has msg.instance_variable_get(:@msg), self
end
##
# Tests if this field tracks presence.
#
# @return [Boolean] True iff this field tracks presence
def has_presence?
@has_presence ||= Google::Protobuf::FFI.get_has_presence(self)
end
# @param msg [Google::Protobuf::Message]
def clear(msg)
if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self)
raise TypeError.new "clear method called on wrong message type"
end
Google::Protobuf::FFI.clear_message_field msg.instance_variable_get(:@msg), self
nil
end
##
# call-seq:
# FieldDescriptor.set(message, value)
#
# Sets the value corresponding to this field to the given value on the given
# message. Raises an exception if message is of the wrong type. Performs the
# ordinary type-checks for field setting.
#
# @param msg [Google::Protobuf::Message]
# @param value [Object]
def set(msg, value)
if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self)
raise TypeError.new "set method called on wrong message type"
end
unless set_value_on_message value, msg.instance_variable_get(:@msg), msg.instance_variable_get(:@arena)
raise RuntimeError.new "allocation failed"
end
nil
end
def map?
@map ||= Google::Protobuf::FFI.is_map self
end
def repeated?
@repeated ||= Google::Protobuf::FFI.is_repeated self
end
def sub_message?
@sub_message ||= Google::Protobuf::FFI.is_sub_message self
end
def wrapper?
if defined? @wrapper
@wrapper
else
message_descriptor = Google::Protobuf::FFI.get_subtype_as_message(self)
@wrapper = message_descriptor.nil? ? false : message_descriptor.send(:wrapper?)
end
end
private
def initialize(field_def, descriptor_pool)
@field_def = field_def
@descriptor_pool = descriptor_pool
end
def self.private_constructor(field_def, descriptor_pool)
instance = allocate
instance.send(:initialize, field_def, descriptor_pool)
instance
end
# TODO(jatl) Can this be added to the public API?
def real_containing_oneof
@real_containing_oneof ||= Google::Protobuf::FFI.real_containing_oneof self
end
# Implementation details below are subject to breaking changes without
# warning and are intended for use only within the gem.
##
# Sets the field this FieldDescriptor represents to the given value on the given message.
# @param value [Object] Value to be set
# @param msg [::FFI::Pointer] Pointer the the upb_Message
# @param arena [Arena] Arena of the message that owns msg
def set_value_on_message(value, msg, arena, wrap: false)
message_to_alter = msg
field_def_to_set = self
if map?
raise TypeError.new "Expected map" unless value.is_a? Google::Protobuf::Map
message_descriptor = subtype
key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1)
key_field_type = Google::Protobuf::FFI.get_type(key_field_def)
raise TypeError.new "Map key type does not match field's key type" unless key_field_type == value.send(:key_type)
value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2)
value_field_type = Google::Protobuf::FFI.get_type(value_field_def)
raise TypeError.new "Map value type does not match field's value type" unless value_field_type == value.send(:value_type)
raise TypeError.new "Map value type has wrong message/enum class" unless value_field_def.subtype == value.send(:descriptor)
arena.fuse(value.send(:arena))
message_value = Google::Protobuf::FFI::MessageValue.new
message_value[:map_val] = value.send(:map_ptr)
elsif repeated?
raise TypeError.new "Expected repeated field array" unless value.is_a? RepeatedField
raise TypeError.new "Repeated field array has wrong message/enum class" unless value.send(:type) == type
arena.fuse(value.send(:arena))
message_value = Google::Protobuf::FFI::MessageValue.new
message_value[:array_val] = value.send(:array)
else
if value.nil? and (sub_message? or !real_containing_oneof.nil?)
Google::Protobuf::FFI.clear_message_field message_to_alter, field_def_to_set
return true
end
if wrap
value_field_def = Google::Protobuf::FFI.get_field_by_number subtype, 1
type_for_conversion = Google::Protobuf::FFI.get_c_type(value_field_def)
raise RuntimeError.new "Not expecting to get a msg or enum when unwrapping" if [:enum, :message].include? type_for_conversion
message_value = convert_ruby_to_upb(value, arena, type_for_conversion, nil)
message_to_alter = Google::Protobuf::FFI.get_mutable_message(msg, self, arena)[:msg]
field_def_to_set = value_field_def
else
message_value = convert_ruby_to_upb(value, arena, c_type, subtype)
end
end
Google::Protobuf::FFI.set_message_field message_to_alter, field_def_to_set, message_value, arena
end
def c_type
@c_type ||= Google::Protobuf::FFI.get_c_type(self)
end
end
class FFI
# MessageDef
attach_function :get_field_by_index, :upb_MessageDef_Field, [Descriptor, :int], FieldDescriptor
attach_function :get_field_by_name, :upb_MessageDef_FindFieldByNameWithSize,[Descriptor, :string, :size_t], FieldDescriptor
attach_function :get_field_by_number, :upb_MessageDef_FindFieldByNumber, [Descriptor, :uint32_t], FieldDescriptor
# FieldDescriptor
attach_function :get_containing_message_def, :upb_FieldDef_ContainingType, [FieldDescriptor], Descriptor
attach_function :get_c_type, :upb_FieldDef_CType, [FieldDescriptor], CType
attach_function :get_default, :upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value
attach_function :get_subtype_as_enum, :upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDescriptor
attach_function :get_has_presence, :upb_FieldDef_HasPresence, [FieldDescriptor], :bool
attach_function :is_map, :upb_FieldDef_IsMap, [FieldDescriptor], :bool
attach_function :is_repeated, :upb_FieldDef_IsRepeated, [FieldDescriptor], :bool
attach_function :is_sub_message, :upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool
attach_function :get_json_name, :upb_FieldDef_JsonName, [FieldDescriptor], :string
attach_function :get_label, :upb_FieldDef_Label, [FieldDescriptor], Label
attach_function :get_subtype_as_message, :upb_FieldDef_MessageSubDef, [FieldDescriptor], Descriptor
attach_function :get_full_name, :upb_FieldDef_Name, [FieldDescriptor], :string
attach_function :get_number, :upb_FieldDef_Number, [FieldDescriptor], :uint32_t
attach_function :get_type, :upb_FieldDef_Type, [FieldDescriptor], FieldType
attach_function :file_def_by_raw_field_def, :upb_FieldDef_File, [:pointer], :FileDef
end
end
end

@ -0,0 +1,71 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class FFI
# FileDescriptor
attach_function :file_def_name, :upb_FileDef_Name, [:FileDef], :string
attach_function :file_def_syntax, :upb_FileDef_Syntax, [:FileDef], Syntax
attach_function :file_def_pool, :upb_FileDef_Pool, [:FileDef], :DefPool
end
class FileDescriptor
attr :descriptor_pool, :file_def
def initialize(file_def, descriptor_pool)
@descriptor_pool = descriptor_pool
@file_def = file_def
end
def to_s
inspect
end
def inspect
"#{self.class.name}: #{name}"
end
def syntax
case Google::Protobuf::FFI.file_def_syntax(@file_def)
when :Proto3
:proto3
when :Proto2
:proto2
else
nil
end
end
def name
Google::Protobuf::FFI.file_def_name(@file_def)
end
end
end
end

@ -0,0 +1,89 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
##
# Implementation details below are subject to breaking changes without
# warning and are intended for use only within the gem.
module Google
module Protobuf
module Internal
class Arena
attr :pinned_messages
# FFI Interface methods and setup
extend ::FFI::DataConverter
native_type ::FFI::Type::POINTER
class << self
prepend Google::Protobuf::Internal::TypeSafety
# @param value [Arena] Arena to convert to an FFI native type
# @param _ [Object] Unused
def to_native(value, _)
value.instance_variable_get(:@arena) || ::FFI::Pointer::NULL
end
##
# @param value [::FFI::Pointer] Arena pointer to be wrapped
# @param _ [Object] Unused
def from_native(value, _)
new(value)
end
end
def initialize(pointer)
@arena = ::FFI::AutoPointer.new(pointer, Google::Protobuf::FFI.method(:free_arena))
@pinned_messages = []
end
def fuse(other_arena)
return if other_arena == self
unless Google::Protobuf::FFI.fuse_arena(self, other_arena)
raise RuntimeError.new "Unable to fuse arenas. This should never happen since Ruby does not use initial blocks"
end
end
def pin(message)
pinned_messages.push message
end
end
end
class FFI
# Arena
attach_function :create_arena, :Arena_create, [], Internal::Arena
attach_function :fuse_arena, :upb_Arena_Fuse, [Internal::Arena, Internal::Arena], :bool
# Argument takes a :pointer rather than a typed Arena here due to
# implementation details of FFI::AutoPointer.
attach_function :free_arena, :upb_Arena_Free, [:pointer], :void
attach_function :arena_malloc, :upb_Arena_Malloc, [Internal::Arena, :size_t], :pointer
end
end
end

@ -0,0 +1,328 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
##
# Implementation details below are subject to breaking changes without
# warning and are intended for use only within the gem.
module Google
module Protobuf
module Internal
module Convert
# Arena should be the
# @param value [Object] Value to convert
# @param arena [Arena] Arena that owns the Message where the MessageValue
# will be set
# @return [Google::Protobuf::FFI::MessageValue]
def convert_ruby_to_upb(value, arena, c_type, msg_or_enum_def)
raise ArgumentError.new "Expected Descriptor or EnumDescriptor, instead got #{msg_or_enum_def.class}" unless [NilClass, Descriptor, EnumDescriptor].include? msg_or_enum_def.class
return_value = Google::Protobuf::FFI::MessageValue.new
case c_type
when :float
raise TypeError.new "Expected number type for float field '#{name}' (given #{value.class})." unless value.respond_to? :to_f
return_value[:float_val] = value.to_f
when :double
raise TypeError.new "Expected number type for double field '#{name}' (given #{value.class})." unless value.respond_to? :to_f
return_value[:double_val] = value.to_f
when :bool
raise TypeError.new "Invalid argument for boolean field '#{name}' (given #{value.class})." unless [TrueClass, FalseClass].include? value.class
return_value[:bool_val] = value
when :string
raise TypeError.new "Invalid argument for string field '#{name}' (given #{value.class})." unless [Symbol, String].include? value.class
begin
string_value = value.to_s.encode("UTF-8")
rescue Encoding::UndefinedConversionError
# TODO(jatl) - why not include the field name here?
raise Encoding::UndefinedConversionError.new "String is invalid UTF-8"
end
return_value[:str_val][:size] = string_value.bytesize
return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize)
# TODO(jatl) - how important is it to still use arena malloc, versus the following?
# buffer = ::FFI::MemoryPointer.new(:char, string_value.bytesize)
# buffer.put_bytes(0, string_value)
# return_value[:str_val][:data] = buffer
raise NoMemoryError.new "Cannot allocate #{string_value.bytesize} bytes for string on Arena" if return_value[:str_val][:data].nil? || return_value[:str_val][:data].null?
return_value[:str_val][:data].write_string(string_value)
when :bytes
raise TypeError.new "Invalid argument for bytes field '#{name}' (given #{value.class})." unless value.is_a? String
string_value = value.encode("ASCII-8BIT")
return_value[:str_val][:size] = string_value.bytesize
return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize)
raise NoMemoryError.new "Cannot allocate #{string_value.bytesize} bytes for bytes on Arena" if return_value[:str_val][:data].nil? || return_value[:str_val][:data].null?
return_value[:str_val][:data].write_string_length(string_value, string_value.bytesize)
when :message
raise TypeError.new "nil message not allowed here." if value.nil?
if value.is_a? Hash
raise RuntimeError.new "Attempted to initialize message from Hash for field #{name} but have no definition" if msg_or_enum_def.nil?
new_message = msg_or_enum_def.msgclass.
send(:private_constructor, arena, initial_value: value)
return_value[:msg_val] = new_message.instance_variable_get(:@msg)
return return_value
end
descriptor = value.class.respond_to?(:descriptor) ? value.class.descriptor : nil
if descriptor != msg_or_enum_def
wkt = Google::Protobuf::FFI.get_well_known_type(msg_or_enum_def)
case wkt
when :Timestamp
raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Time
new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena
sec = Google::Protobuf::FFI::MessageValue.new
sec[:int64_val] = value.tv_sec
sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1
raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, sec_field_def, sec, arena
nsec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 2
nsec = Google::Protobuf::FFI::MessageValue.new
nsec[:int32_val] = value.tv_nsec
raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, nsec_field_def, nsec, arena
return_value[:msg_val] = new_message
when :Duration
raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Numeric
new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena
sec = Google::Protobuf::FFI::MessageValue.new
sec[:int64_val] = value
sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1
raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, sec_field_def, sec, arena
nsec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 2
nsec = Google::Protobuf::FFI::MessageValue.new
nsec[:int32_val] = ((value.to_f - value.to_i) * 1000000000).round
raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, nsec_field_def, nsec, arena
return_value[:msg_val] = new_message
else
raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'."
end
else
arena.fuse(value.instance_variable_get(:@arena))
return_value[:msg_val] = value.instance_variable_get :@msg
end
when :enum
return_value[:int32_val] = case value
when Numeric
value.to_i
when String, Symbol
enum_number = EnumDescriptor.send(:lookup_name, msg_or_enum_def, value.to_s)
#TODO(jatl) add the bad value to the error message after tests pass
raise RangeError.new "Unknown symbol value for enum field '#{name}'." if enum_number.nil?
enum_number
else
raise TypeError.new "Expected number or symbol type for enum field '#{name}'."
end
#TODO(jatl) After all tests pass, improve error message across integer type by including actual offending value
when :int32
raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric
raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value
raise RangeError.new "Value assigned to int32 field '#{name}' (given #{value.class}) with more than 32-bits." unless value.to_i.bit_length < 32
return_value[:int32_val] = value.to_i
when :uint32
raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric
raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value
raise RangeError.new "Assigning negative value to unsigned integer field '#{name}' (given #{value.class})." if value < 0
raise RangeError.new "Value assigned to uint32 field '#{name}' (given #{value.class}) with more than 32-bits." unless value.to_i.bit_length < 33
return_value[:uint32_val] = value.to_i
when :int64
raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric
raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value
raise RangeError.new "Value assigned to int64 field '#{name}' (given #{value.class}) with more than 64-bits." unless value.to_i.bit_length < 64
return_value[:int64_val] = value.to_i
when :uint64
raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric
raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value
raise RangeError.new "Assigning negative value to unsigned integer field '#{name}' (given #{value.class})." if value < 0
raise RangeError.new "Value assigned to uint64 field '#{name}' (given #{value.class}) with more than 64-bits." unless value.to_i.bit_length < 65
return_value[:uint64_val] = value.to_i
else
raise RuntimeError.new "Unsupported type #{c_type}"
end
return_value
end
##
# Safe to call without an arena if the caller has checked that c_type
# is not :message.
# @param message_value [Google::Protobuf::FFI::MessageValue] Value to be converted.
# @param c_type [Google::Protobuf::FFI::CType] Enum representing the type of message_value
# @param msg_or_enum_def [::FFI::Pointer] Pointer to the MsgDef or EnumDef definition
# @param arena [Google::Protobuf::Internal::Arena] Arena to create Message instances, if needed
def convert_upb_to_ruby(message_value, c_type, msg_or_enum_def = nil, arena = nil)
throw TypeError.new "Expected MessageValue but got #{message_value.class}" unless message_value.is_a? Google::Protobuf::FFI::MessageValue
case c_type
when :bool
message_value[:bool_val]
when :int32
message_value[:int32_val]
when :uint32
message_value[:uint32_val]
when :double
message_value[:double_val]
when :int64
message_value[:int64_val]
when :uint64
message_value[:uint64_val]
when :string
if message_value[:str_val][:size].zero?
""
else
message_value[:str_val][:data].read_string_length(message_value[:str_val][:size]).force_encoding("UTF-8").freeze
end
when :bytes
if message_value[:str_val][:size].zero?
""
else
message_value[:str_val][:data].read_string_length(message_value[:str_val][:size]).force_encoding("ASCII-8BIT").freeze
end
when :float
message_value[:float_val]
when :enum
EnumDescriptor.send(:lookup_value, msg_or_enum_def, message_value[:int32_val]) || message_value[:int32_val]
when :message
raise "Null Arena for message" if arena.nil?
Descriptor.send(:get_message, message_value[:msg_val], msg_or_enum_def, arena)
else
raise RuntimeError.new "Unexpected type #{c_type}"
end
end
def to_h_internal(msg, message_descriptor)
return nil if msg.nil? or msg.null?
hash = {}
is_proto2 = Google::Protobuf::FFI.message_def_syntax(message_descriptor) == :Proto2
message_descriptor.each do |field_descriptor|
# TODO: Legacy behavior, remove when we fix the is_proto2 differences.
if !is_proto2 and
field_descriptor.sub_message? and
!field_descriptor.repeated? and
!Google::Protobuf::FFI.get_message_has(msg, field_descriptor)
hash[field_descriptor.name.to_sym] = nil
next
end
# Do not include fields that are not present (oneof or optional fields).
if is_proto2 and field_descriptor.has_presence? and !Google::Protobuf::FFI.get_message_has(msg, field_descriptor)
next
end
message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor
# Proto2 omits empty map/repeated fields also.
if field_descriptor.map?
hash_entry = map_create_hash(message_value[:map_val], field_descriptor)
elsif field_descriptor.repeated?
array = message_value[:array_val]
if is_proto2 and (array.null? || Google::Protobuf::FFI.array_size(array).zero?)
next
end
hash_entry = repeated_field_create_array(array, field_descriptor, field_descriptor.type)
else
hash_entry = scalar_create_hash(message_value, field_descriptor.type, field_descriptor: field_descriptor)
end
hash[field_descriptor.name.to_sym] = hash_entry
end
hash
end
def map_create_hash(map_ptr, field_descriptor)
return {} if map_ptr.nil? or map_ptr.null?
return_value = {}
message_descriptor = field_descriptor.send(:subtype)
key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1)
key_field_type = Google::Protobuf::FFI.get_type(key_field_def)
value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2)
value_field_type = Google::Protobuf::FFI.get_type(value_field_def)
iter = ::FFI::MemoryPointer.new(:size_t, 1)
iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin)
while Google::Protobuf::FFI.map_next(map_ptr, iter) do
iter_size_t = iter.read(:size_t)
key_message_value = Google::Protobuf::FFI.map_key(map_ptr, iter_size_t)
value_message_value = Google::Protobuf::FFI.map_value(map_ptr, iter_size_t)
hash_key = convert_upb_to_ruby(key_message_value, key_field_type)
hash_value = scalar_create_hash(value_message_value, value_field_type, msg_or_enum_descriptor: value_field_def.subtype)
return_value[hash_key] = hash_value
end
return_value
end
def repeated_field_create_array(array, field_descriptor, type)
return_value = []
n = (array.nil? || array.null?) ? 0 : Google::Protobuf::FFI.array_size(array)
0.upto(n - 1) do |i|
message_value = Google::Protobuf::FFI.get_msgval_at(array, i)
return_value << scalar_create_hash(message_value, type, field_descriptor: field_descriptor)
end
return_value
end
# @param field_descriptor [FieldDescriptor] Descriptor of the field to convert to a hash.
def scalar_create_hash(message_value, type, field_descriptor: nil, msg_or_enum_descriptor: nil)
if [:message, :enum].include? type
if field_descriptor.nil?
if msg_or_enum_descriptor.nil?
raise "scalar_create_hash requires either a FieldDescriptor, MessageDescriptor, or EnumDescriptor as an argument, but received only nil"
end
else
msg_or_enum_descriptor = field_descriptor.subtype
end
if type == :message
to_h_internal(message_value[:msg_val], msg_or_enum_descriptor)
elsif type == :enum
convert_upb_to_ruby message_value, type, msg_or_enum_descriptor
end
else
convert_upb_to_ruby message_value, type
end
end
def message_value_deep_copy(message_value, type, descriptor, arena)
raise unless message_value.is_a? Google::Protobuf::FFI::MessageValue
new_message_value = Google::Protobuf::FFI::MessageValue.new
case type
when :string, :bytes
# TODO(jatl) - how important is it to still use arena malloc, versus using FFI MemoryPointers?
new_message_value[:str_val][:size] = message_value[:str_val][:size]
new_message_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, message_value[:str_val][:size])
raise NoMemoryError.new "Allocation failed" if new_message_value[:str_val][:data].nil? or new_message_value[:str_val][:data].null?
Google::Protobuf::FFI.memcpy(new_message_value[:str_val][:data], message_value[:str_val][:data], message_value[:str_val][:size])
when :message
new_message_value[:msg_val] = descriptor.msgclass.send(:deep_copy, message_value[:msg_val], arena).instance_variable_get(:@msg)
else
Google::Protobuf::FFI.memcpy(new_message_value.to_ptr, message_value.to_ptr, Google::Protobuf::FFI::MessageValue.size)
end
new_message_value
end
end
end
end
end

@ -0,0 +1,58 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
module Internal
module PointerHelper
# Utility code to defensively walk the object graph from a file_def to
# the pool, and either retrieve the wrapper object for the given pointer
# or create one. Assumes that the caller is the wrapper class for the
# given pointer and that it implements `private_constructor`.
def descriptor_from_file_def(file_def, pointer)
raise RuntimeError.new "FileDef is nil" if file_def.nil?
raise RuntimeError.new "FileDef is null" if file_def.null?
pool_def = Google::Protobuf::FFI.file_def_pool file_def
raise RuntimeError.new "PoolDef is nil" if pool_def.nil?
raise RuntimeError.new "PoolDef is null" if pool_def.null?
pool = Google::Protobuf::OBJECT_CACHE.get(pool_def.address)
raise "Cannot find pool in ObjectCache!" if pool.nil?
descriptor = pool.descriptor_class_by_def[pointer.address]
if descriptor.nil?
pool.descriptor_class_by_def[pointer.address] = private_constructor(pointer, pool)
else
descriptor
end
end
end
end
end
end

@ -0,0 +1,48 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# A to_native DataConverter method that raises an error if the value is not of the same type.
# Adapted from to https://www.varvet.com/blog/advanced-topics-in-ruby-ffi/
module Google
module Protobuf
module Internal
module TypeSafety
def to_native(value, ctx = nil)
if value.kind_of?(self) or value.nil?
super
else
raise TypeError.new "Expected a kind of #{name}, was #{value.class}"
end
end
end
end
end
end

@ -0,0 +1,419 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class FFI
# Map
attach_function :map_clear, :upb_Map_Clear, [:Map], :void
attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool
attach_function :map_get, :upb_Map_Get, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool
attach_function :create_map, :upb_Map_New, [Internal::Arena, CType, CType], :Map
attach_function :map_size, :upb_Map_Size, [:Map], :size_t
attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Internal::Arena], :bool
# MapIterator
attach_function :map_next, :upb_MapIterator_Next, [:Map, :pointer], :bool
attach_function :map_done, :upb_MapIterator_Done, [:Map, :size_t], :bool
attach_function :map_key, :upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value
attach_function :map_value, :upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value
end
class Map
include Enumerable
##
# call-seq:
# Map.new(key_type, value_type, value_typeclass = nil, init_hashmap = {})
# => new map
#
# Allocates a new Map container. This constructor may be called with 2, 3, or 4
# arguments. The first two arguments are always present and are symbols (taking
# on the same values as field-type symbols in message descriptors) that
# indicate the type of the map key and value fields.
#
# The supported key types are: :int32, :int64, :uint32, :uint64, :bool,
# :string, :bytes.
#
# The supported value types are: :int32, :int64, :uint32, :uint64, :bool,
# :string, :bytes, :enum, :message.
#
# The third argument, value_typeclass, must be present if value_type is :enum
# or :message. As in RepeatedField#new, this argument must be a message class
# (for :message) or enum module (for :enum).
#
# The last argument, if present, provides initial content for map. Note that
# this may be an ordinary Ruby hashmap or another Map instance with identical
# key and value types. Also note that this argument may be present whether or
# not value_typeclass is present (and it is unambiguously separate from
# value_typeclass because value_typeclass's presence is strictly determined by
# value_type). The contents of this initial hashmap or Map instance are
# shallow-copied into the new Map: the original map is unmodified, but
# references to underlying objects will be shared if the value type is a
# message type.
def self.new(key_type, value_type, value_typeclass = nil, init_hashmap = {})
instance = allocate
# TODO(jatl) This argument mangling doesn't agree with the type signature,
# but does align with the text of the comments and is required to make unit tests pass.
if init_hashmap.empty? and ![:enum, :message].include?(value_type)
init_hashmap = value_typeclass
value_typeclass = nil
end
instance.send(:initialize, key_type, value_type, value_type_class: value_typeclass, initial_values: init_hashmap)
instance
end
##
# call-seq:
# Map.keys => [list_of_keys]
#
# Returns the list of keys contained in the map, in unspecified order.
def keys
return_value = []
internal_iterator do |iterator|
key_message_value = Google::Protobuf::FFI.map_key(@map_ptr, iterator)
return_value << convert_upb_to_ruby(key_message_value, key_type)
end
return_value
end
##
# call-seq:
# Map.values => [list_of_values]
#
# Returns the list of values contained in the map, in unspecified order.
def values
return_value = []
internal_iterator do |iterator|
value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator)
return_value << convert_upb_to_ruby(value_message_value, value_type, descriptor, arena)
end
return_value
end
##
# call-seq:
# Map.[](key) => value
#
# Accesses the element at the given key. Throws an exception if the key type is
# incorrect. Returns nil when the key is not present in the map.
def [](key)
value = Google::Protobuf::FFI::MessageValue.new
key_message_value = convert_ruby_to_upb(key, arena, key_type, nil)
if Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, value)
convert_upb_to_ruby(value, value_type, descriptor, arena)
end
end
##
# call-seq:
# Map.[]=(key, value) => value
#
# Inserts or overwrites the value at the given key with the given new value.
# Throws an exception if the key type is incorrect. Returns the new value that
# was just inserted.
def []=(key, value)
raise FrozenError.new "can't modify frozen #{self.class}" if frozen?
key_message_value = convert_ruby_to_upb(key, arena, key_type, nil)
value_message_value = convert_ruby_to_upb(value, arena, value_type, descriptor)
Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena)
value
end
def has_key?(key)
key_message_value = convert_ruby_to_upb(key, arena, key_type, nil)
Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, nil)
end
##
# call-seq:
# Map.delete(key) => old_value
#
# Deletes the value at the given key, if any, returning either the old value or
# nil if none was present. Throws an exception if the key is of the wrong type.
def delete(key)
raise FrozenError.new "can't modify frozen #{self.class}" if frozen?
value = Google::Protobuf::FFI::MessageValue.new
key_message_value = convert_ruby_to_upb(key, arena, key_type, nil)
if Google::Protobuf::FFI.map_delete(@map_ptr, key_message_value, value)
convert_upb_to_ruby(value, value_type, descriptor, arena)
else
nil
end
end
def clear
raise FrozenError.new "can't modify frozen #{self.class}" if frozen?
Google::Protobuf::FFI.map_clear(@map_ptr)
nil
end
def length
Google::Protobuf::FFI.map_size(@map_ptr)
end
alias size length
##
# call-seq:
# Map.dup => new_map
#
# Duplicates this map with a shallow copy. References to all non-primitive
# element objects (e.g., submessages) are shared.
def dup
internal_dup
end
alias clone dup
##
# call-seq:
# Map.==(other) => boolean
#
# Compares this map to another. Maps are equal if they have identical key sets,
# and for each key, the values in both maps compare equal. Elements are
# compared as per normal Ruby semantics, by calling their :== methods (or
# performing a more efficient comparison for primitive types).
#
# Maps with dissimilar key types or value types/typeclasses are never equal,
# even if value comparison (for example, between integers and floats) would
# have otherwise indicated that every element has equal value.
def ==(other)
if other.is_a? Hash
other = self.class.send(:private_constructor, key_type, value_type, descriptor, initial_values: other)
elsif !other.is_a? Google::Protobuf::Map
return false
end
return true if object_id == other.object_id
return false if key_type != other.send(:key_type) or value_type != other.send(:value_type) or descriptor != other.send(:descriptor) or length != other.length
other_map_ptr = other.send(:map_ptr)
each_msg_val do |key_message_value, value_message_value|
other_value = Google::Protobuf::FFI::MessageValue.new
return false unless Google::Protobuf::FFI.map_get(other_map_ptr, key_message_value, other_value)
return false unless Google::Protobuf::FFI.message_value_equal(value_message_value, other_value, value_type, descriptor)
end
true
end
def hash
return_value = 0
each_msg_val do |key_message_value, value_message_value|
return_value = Google::Protobuf::FFI.message_value_hash(key_message_value, key_type, nil, return_value)
return_value = Google::Protobuf::FFI.message_value_hash(value_message_value, value_type, descriptor, return_value)
end
return_value
end
##
# call-seq:
# Map.to_h => {}
#
# Returns a Ruby Hash object containing all the values within the map
def to_h
return {} if map_ptr.nil? or map_ptr.null?
return_value = {}
each_msg_val do |key_message_value, value_message_value|
hash_key = convert_upb_to_ruby(key_message_value, key_type)
hash_value = scalar_create_hash(value_message_value, value_type, msg_or_enum_descriptor: descriptor)
return_value[hash_key] = hash_value
end
return_value
end
def inspect
key_value_pairs = []
each_msg_val do |key_message_value, value_message_value|
key_string = convert_upb_to_ruby(key_message_value, key_type).inspect
if value_type == :message
sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(descriptor)
value_string = sub_msg_descriptor.msgclass.send(:inspect_internal, value_message_value[:msg_val])
else
value_string = convert_upb_to_ruby(value_message_value, value_type, descriptor).inspect
end
key_value_pairs << "#{key_string}=>#{value_string}"
end
"{#{key_value_pairs.join(", ")}}"
end
##
# call-seq:
# Map.merge(other_map) => map
#
# Copies key/value pairs from other_map into a copy of this map. If a key is
# set in other_map and this map, the value from other_map overwrites the value
# in the new copy of this map. Returns the new copy of this map with merged
# contents.
def merge(other)
internal_merge(other)
end
##
# call-seq:
# Map.each(&block)
#
# Invokes &block on each |key, value| pair in the map, in unspecified order.
# Note that Map also includes Enumerable; map thus acts like a normal Ruby
# sequence.
def each &block
each_msg_val do |key_message_value, value_message_value|
key_value = convert_upb_to_ruby(key_message_value, key_type)
value_value = convert_upb_to_ruby(value_message_value, value_type, descriptor, arena)
yield key_value, value_value
end
nil
end
private
attr :arena, :map_ptr, :key_type, :value_type, :descriptor, :name
include Google::Protobuf::Internal::Convert
def internal_iterator
iter = ::FFI::MemoryPointer.new(:size_t, 1)
iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin)
while Google::Protobuf::FFI.map_next(@map_ptr, iter) do
iter_size_t = iter.read(:size_t)
yield iter_size_t
end
end
def each_msg_val &block
internal_iterator do |iterator|
key_message_value = Google::Protobuf::FFI.map_key(@map_ptr, iterator)
value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator)
yield key_message_value, value_message_value
end
end
def internal_dup
instance = self.class.send(:private_constructor, key_type, value_type, descriptor, arena: arena)
new_map_ptr = instance.send(:map_ptr)
each_msg_val do |key_message_value, value_message_value|
Google::Protobuf::FFI.map_set(new_map_ptr, key_message_value, value_message_value, arena)
end
instance
end
def internal_merge_into_self(other)
case other
when Hash
other.each do |key, value|
key_message_value = convert_ruby_to_upb(key, arena, key_type, nil)
value_message_value = convert_ruby_to_upb(value, arena, value_type, descriptor)
Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena)
end
when Google::Protobuf::Map
unless key_type == other.send(:key_type) and value_type == other.send(:value_type) and descriptor == other.descriptor
raise ArgumentError.new "Attempt to merge Map with mismatching types" #TODO(jatl) Improve error message by adding type information
end
arena.fuse(other.send(:arena))
iter = ::FFI::MemoryPointer.new(:size_t, 1)
iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin)
other.send(:each_msg_val) do |key_message_value, value_message_value|
Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena)
end
else
raise ArgumentError.new "Unknown type merging into Map" #TODO(jatl) improve this error message by including type information
end
self
end
def internal_merge(other)
internal_dup.internal_merge_into_self(other)
end
def initialize(key_type, value_type, value_type_class: nil, initial_values: nil, arena: nil, map: nil, descriptor: nil, name: nil)
@name = name || 'Map'
unless [:int32, :int64, :uint32, :uint64, :bool, :string, :bytes].include? key_type
raise ArgumentError.new "Invalid key type for map." #TODO(jatl) improve error message to include what type was passed
end
@key_type = key_type
unless [:int32, :int64, :uint32, :uint64, :bool, :string, :bytes, :enum, :message].include? value_type
raise ArgumentError.new "Invalid value type for map." #TODO(jatl) improve error message to include what type was passed
end
@value_type = value_type
if !descriptor.nil?
raise ArgumentError "Expected descriptor to be a Descriptor or EnumDescriptor" unless [EnumDescriptor, Descriptor].include? descriptor.class
@descriptor = descriptor
elsif [:message, :enum].include? value_type
raise ArgumentError.new "Expected at least 3 arguments for message/enum." if value_type_class.nil?
descriptor = value_type_class.respond_to?(:descriptor) ? value_type_class.descriptor : nil
raise ArgumentError.new "Type class #{value_type_class} has no descriptor. Please pass a class or enum as returned by the DescriptorPool." if descriptor.nil?
@descriptor = descriptor
else
@descriptor = nil
end
@arena = arena || Google::Protobuf::FFI.create_arena
@map_ptr = map || Google::Protobuf::FFI.create_map(@arena, @key_type, @value_type)
internal_merge_into_self(initial_values) unless initial_values.nil?
# Should always be the last expression of the initializer to avoid
# leaking references to this object before construction is complete.
OBJECT_CACHE.try_add(@map_ptr.address, self)
end
# @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned
# @param values [Hash|Map] Initial value; may be nil or empty
# @param arena [Arena] Owning message's arena
def self.construct_for_field(field, arena, value: nil, map: nil)
raise ArgumentError.new "Expected Hash object as initializer value for map field '#{field.name}' (given #{value.class})." unless value.nil? or value.is_a? Hash
instance = allocate
raise ArgumentError.new "Expected field with type :message, instead got #{field.class}" unless field.type == :message
message_descriptor = field.send(:subtype)
key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1)
key_field_type = Google::Protobuf::FFI.get_type(key_field_def)
value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2)
value_field_type = Google::Protobuf::FFI.get_type(value_field_def)
instance.send(:initialize, key_field_type, value_field_type, initial_values: value, name: field.name, arena: arena, map: map, descriptor: value_field_def.subtype)
instance
end
def self.private_constructor(key_type, value_type, descriptor, initial_values: nil, arena: nil)
instance = allocate
instance.send(:initialize, key_type, value_type, descriptor: descriptor, initial_values: initial_values, arena: arena)
instance
end
extend Google::Protobuf::Internal::Convert
def self.deep_copy(map)
instance = allocate
instance.send(:initialize, map.send(:key_type), map.send(:value_type), descriptor: map.send(:descriptor))
map.send(:each_msg_val) do |key_message_value, value_message_value|
Google::Protobuf::FFI.map_set(instance.send(:map_ptr), key_message_value, message_value_deep_copy(value_message_value, map.send(:value_type), map.send(:descriptor), instance.send(:arena)), instance.send(:arena))
end
instance
end
end
end
end

@ -0,0 +1,664 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Decorates Descriptor with the `build_message_class` method that defines
# Message classes.
module Google
module Protobuf
class FFI
# Message
attach_function :clear_message_field, :upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void
attach_function :get_message_value, :upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value
attach_function :get_message_has, :upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool
attach_function :set_message_field, :upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Internal::Arena], :bool
attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Internal::Arena, :pointer, :pointer], EncodeStatus
attach_function :json_decode_message, :upb_JsonDecode, [:binary_string, :size_t, :Message, Descriptor, :DefPool, :int, Internal::Arena, Status.by_ref], :bool
attach_function :json_encode_message, :upb_JsonEncode, [:Message, Descriptor, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t
attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Internal::Arena], DecodeStatus
attach_function :get_mutable_message, :upb_Message_Mutable, [:Message, FieldDescriptor, Internal::Arena], MutableMessageValue.by_value
attach_function :get_message_which_oneof, :upb_Message_WhichOneof, [:Message, OneofDescriptor], FieldDescriptor
attach_function :message_discard_unknown, :upb_Message_DiscardUnknown, [:Message, Descriptor, :int], :bool
# MessageValue
attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, Descriptor], :bool
attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, Descriptor, :uint64_t], :uint64_t
end
class Descriptor
def build_message_class
descriptor = self
Class.new(Google::Protobuf::const_get(:AbstractMessage)) do
@descriptor = descriptor
class << self
attr_accessor :descriptor
private
attr_accessor :oneof_field_names
include ::Google::Protobuf::Internal::Convert
end
alias original_method_missing method_missing
def method_missing(method_name, *args)
method_missing_internal method_name, *args, mode: :method_missing
end
def respond_to_missing?(method_name, include_private = false)
method_missing_internal(method_name, mode: :respond_to_missing?) || super
end
##
# Public constructor. Automatically allocates from a new Arena.
def self.new(initial_value = nil)
instance = allocate
instance.send(:initialize, initial_value)
instance
end
def freeze
super
@arena.pin self
self
end
def dup
duplicate = self.class.private_constructor(@arena)
mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor)
size = mini_table[:size]
duplicate.instance_variable_get(:@msg).write_string_length(@msg.read_string_length(size), size)
duplicate
end
alias clone dup
def eql?(other)
return false unless self.class === other
encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown
temporary_arena = Google::Protobuf::FFI.create_arena
mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor)
size_one = ::FFI::MemoryPointer.new(:size_t, 1)
encoding_one = ::FFI::MemoryPointer.new(:pointer, 1)
encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one)
raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok
size_two = ::FFI::MemoryPointer.new(:size_t, 1)
encoding_two = ::FFI::MemoryPointer.new(:pointer, 1)
encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two)
raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok
if encoding_one.null? or encoding_two.null?
raise ParseError.new "Error comparing messages"
end
size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero?
end
alias == eql?
def hash
encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown
temporary_arena = Google::Protobuf::FFI.create_arena
mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor)
size_ptr = ::FFI::MemoryPointer.new(:size_t, 1)
encoding = ::FFI::MemoryPointer.new(:pointer, 1)
encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr)
if encoding_status != :Ok or encoding.null?
raise ParseError.new "Error calculating hash"
end
encoding.read(:pointer).read_string(size_ptr.read(:size_t)).hash
end
def to_h
to_h_internal @msg, self.class.descriptor
end
##
# call-seq:
# Message.inspect => string
#
# Returns a human-readable string representing this message. It will be
# formatted as "<MessageType: field1: value1, field2: value2, ...>". Each
# field's value is represented according to its own #inspect method.
def inspect
self.class.inspect_internal @msg
end
def to_s
self.inspect
end
##
# call-seq:
# Message.[](index) => value
# Accesses a field's value by field name. The provided field name
# should be a string.
def [](name)
raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String
index_internal name
end
##
# call-seq:
# Message.[]=(index, value)
# Sets a field's value by field name. The provided field name should
# be a string.
# @param name [String] Name of the field to be set
# @param value [Object] Value to set the field to
def []=(name, value)
raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String
index_assign_internal(value, name: name)
end
##
# call-seq:
# MessageClass.decode(data, options) => message
#
# Decodes the given data (as a string containing bytes in protocol buffers wire
# format) under the interpretation given by this message class's definition
# and returns a message object with the corresponding field values.
# @param data [String] Binary string in Protobuf wire format to decode
# @param options [Hash] options for the decoder
# @option options [Integer] :recursion_limit Set to maximum decoding depth for message (default is 64)
def self.decode(data, options = {})
raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash
raise ArgumentError.new "Expected string for binary protobuf data." unless data.is_a? String
decoding_options = 0
depth = options[:recursion_limit]
if depth.is_a? Numeric
decoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i)
end
message = new
mini_table_ptr = Google::Protobuf::FFI.get_mini_table(message.class.descriptor)
status = Google::Protobuf::FFI.decode_message(data, data.bytesize, message.instance_variable_get(:@msg), mini_table_ptr, nil, decoding_options, message.instance_variable_get(:@arena))
raise ParseError.new "Error occurred during parsing" unless status == :Ok
message
end
##
# call-seq:
# MessageClass.encode(msg, options) => bytes
#
# Encodes the given message object to its serialized form in protocol buffers
# wire format.
# @param options [Hash] options for the encoder
# @option options [Integer] :recursion_limit Set to maximum encoding depth for message (default is 64)
def self.encode(message, options = {})
raise ArgumentError.new "Message of wrong type." unless message.is_a? self
raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash
encoding_options = 0
depth = options[:recursion_limit]
if depth.is_a? Numeric
encoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i)
end
encode_internal(message.instance_variable_get(:@msg), encoding_options) do |encoding, size, _|
if encoding.nil? or encoding.null?
raise RuntimeError.new "Exceeded maximum depth (possibly cycle)"
else
encoding.read_string_length(size).force_encoding("ASCII-8BIT").freeze
end
end
end
##
# all-seq:
# MessageClass.decode_json(data, options = {}) => message
#
# Decodes the given data (as a string containing bytes in protocol buffers wire
# format) under the interpretation given by this message class's definition
# and returns a message object with the corresponding field values.
#
# @param options [Hash] options for the decoder
# @option options [Boolean] :ignore_unknown_fields Set true to ignore unknown fields (default is to raise an error)
# @return [Message]
def self.decode_json(data, options = {})
decoding_options = 0
unless options.is_a? Hash
if options.respond_to? :to_h
options options.to_h
else
#TODO(jatl) can this error message be improve to include what was received?
raise ArgumentError.new "Expected hash arguments"
end
end
raise ArgumentError.new "Expected string for JSON data." unless data.is_a? String
raise RuntimeError.new "Cannot parse a wrapper directly" if descriptor.send(:wrapper?)
if options[:ignore_unknown_fields]
decoding_options |= Google::Protobuf::FFI::Upb_JsonDecode_IgnoreUnknown
end
message = new
pool_def = message.class.descriptor.instance_variable_get(:@descriptor_pool).descriptor_pool
status = Google::Protobuf::FFI::Status.new
unless Google::Protobuf::FFI.json_decode_message(data, data.bytesize, message.instance_variable_get(:@msg), message.class.descriptor, pool_def, decoding_options, message.instance_variable_get(:@arena), status)
raise ParseError.new "Error occurred during parsing: #{Google::Protobuf::FFI.error_message(status)}"
end
message
end
def self.encode_json(message, options = {})
encoding_options = 0
unless options.is_a? Hash
if options.respond_to? :to_h
options = options.to_h
else
#TODO(jatl) can this error message be improve to include what was received?
raise ArgumentError.new "Expected hash arguments"
end
end
if options[:preserve_proto_fieldnames]
encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_UseProtoNames
end
if options[:emit_defaults]
encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_EmitDefaults
end
if options[:format_enums_as_integers]
encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_FormatEnumsAsIntegers
end
buffer_size = 1024
buffer = ::FFI::MemoryPointer.new(:char, buffer_size)
status = Google::Protobuf::FFI::Status.new
msg = message.instance_variable_get(:@msg)
pool_def = message.class.descriptor.instance_variable_get(:@descriptor_pool).descriptor_pool
size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status)
unless status[:ok]
raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}"
end
if size >= buffer_size
buffer_size = size + 1
buffer = ::FFI::MemoryPointer.new(:char, buffer_size)
status.clear
size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status)
unless status[:ok]
raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}"
end
if size >= buffer_size
raise ParseError.new "Inconsistent JSON encoding sizes - was #{buffer_size - 1}, now #{size}"
end
end
buffer.read_string_length(size).force_encoding("UTF-8").freeze
end
private
# Implementation details below are subject to breaking changes without
# warning and are intended for use only within the gem.
include Google::Protobuf::Internal::Convert
def self.setup_accessors!
@descriptor.each do |field_descriptor|
field_name = field_descriptor.name
unless instance_methods(true).include?(field_name.to_sym)
#TODO(jatl) - at a high level, dispatching to either
# index_internal or get_field would be logically correct, but slightly slower.
if field_descriptor.map?
define_method(field_name) do
mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena
get_map_field(mutable_message_value[:map], field_descriptor)
end
elsif field_descriptor.repeated?
define_method(field_name) do
mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena
get_repeated_field(mutable_message_value[:array], field_descriptor)
end
elsif field_descriptor.sub_message?
define_method(field_name) do
return nil unless Google::Protobuf::FFI.get_message_has @msg, field_descriptor
mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena
sub_message = mutable_message[:msg]
sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor)
Descriptor.send(:get_message, sub_message, sub_message_def, @arena)
end
else
c_type = field_descriptor.send(:c_type)
if c_type == :enum
define_method(field_name) do
message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor
convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field_descriptor)
end
else
define_method(field_name) do
message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor
convert_upb_to_ruby message_value, c_type
end
end
end
define_method("#{field_name}=") do |value|
index_assign_internal(value, field_descriptor: field_descriptor)
end
define_method("clear_#{field_name}") do
clear_internal(field_descriptor)
end
if field_descriptor.type == :enum
define_method("#{field_name}_const") do
if field_descriptor.repeated?
return_value = []
get_field(field_descriptor).send(:each_msg_val) do |msg_val|
return_value << msg_val[:int32_val]
end
return_value
else
message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor
message_value[:int32_val]
end
end
end
if !field_descriptor.repeated? and field_descriptor.wrapper?
define_method("#{field_name}_as_value") do
get_field(field_descriptor, unwrap: true)
end
define_method("#{field_name}_as_value=") do |value|
if value.nil?
clear_internal(field_descriptor)
else
index_assign_internal(value, field_descriptor: field_descriptor, wrap: true)
end
end
end
if field_descriptor.has_presence?
define_method("has_#{field_name}?") do
Google::Protobuf::FFI.get_message_has(@msg, field_descriptor)
end
end
end
end
end
def self.setup_oneof_accessors!
@oneof_field_names = []
@descriptor.each_oneof do |oneof_descriptor|
self.add_oneof_accessors_for! oneof_descriptor
end
end
def self.add_oneof_accessors_for!(oneof_descriptor)
field_name = oneof_descriptor.name.to_sym
@oneof_field_names << field_name
unless instance_methods(true).include?(field_name)
define_method(field_name) do
field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor)
if field_descriptor.nil?
return
else
return field_descriptor.name.to_sym
end
end
define_method("clear_#{field_name}") do
field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor)
unless field_descriptor.nil?
clear_internal(field_descriptor)
end
end
define_method("has_#{field_name}?") do
!Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor).nil?
end
end
end
setup_accessors!
setup_oneof_accessors!
def self.private_constructor(arena, msg: nil, initial_value: nil)
instance = allocate
instance.send(:initialize, initial_value, arena, msg)
instance
end
def self.inspect_field(field_descriptor, c_type, message_value)
if field_descriptor.sub_message?
sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor)
sub_msg_descriptor.msgclass.send(:inspect_internal, message_value[:msg_val])
else
convert_upb_to_ruby(message_value, c_type, field_descriptor.subtype).inspect
end
end
# @param msg [::FFI::Pointer] Pointer to the Message
def self.inspect_internal(msg)
field_output = []
descriptor.each do |field_descriptor|
next if field_descriptor.has_presence? && !Google::Protobuf::FFI.get_message_has(msg, field_descriptor)
if field_descriptor.map?
# TODO(jatl) Adapted - from map#each_msg_val and map#inspect- can this be refactored to reduce echo without introducing a arena allocation?
message_descriptor = field_descriptor.subtype
key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1)
key_field_type = Google::Protobuf::FFI.get_type(key_field_def)
value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2)
value_field_type = Google::Protobuf::FFI.get_type(value_field_def)
message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor)
iter = ::FFI::MemoryPointer.new(:size_t, 1)
iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin)
key_value_pairs = []
while Google::Protobuf::FFI.map_next(message_value[:map_val], iter) do
iter_size_t = iter.read(:size_t)
key_message_value = Google::Protobuf::FFI.map_key(message_value[:map_val], iter_size_t)
value_message_value = Google::Protobuf::FFI.map_value(message_value[:map_val], iter_size_t)
key_string = convert_upb_to_ruby(key_message_value, key_field_type).inspect
value_string = inspect_field(value_field_def, value_field_type, value_message_value)
key_value_pairs << "#{key_string}=>#{value_string}"
end
field_output << "#{field_descriptor.name}: {#{key_value_pairs.join(", ")}}"
elsif field_descriptor.repeated?
# TODO(jatl) Adapted - from repeated_field#each - can this be refactored to reduce echo?
repeated_field_output = []
message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor)
array = message_value[:array_val]
n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array)
0.upto(n - 1) do |i|
element = Google::Protobuf::FFI.get_msgval_at(array, i)
repeated_field_output << inspect_field(field_descriptor, field_descriptor.send(:c_type), element)
end
field_output << "#{field_descriptor.name}: [#{repeated_field_output.join(", ")}]"
else
message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor
rendered_value = inspect_field(field_descriptor, field_descriptor.send(:c_type), message_value)
field_output << "#{field_descriptor.name}: #{rendered_value}"
end
end
"<#{name}: #{field_output.join(', ')}>"
end
def self.deep_copy(msg, arena = nil)
arena ||= Google::Protobuf::FFI.create_arena
encode_internal(msg) do |encoding, size, mini_table_ptr|
message = private_constructor(arena)
if encoding.nil? or encoding.null? or Google::Protobuf::FFI.decode_message(encoding, size, message.instance_variable_get(:@msg), mini_table_ptr, nil, 0, arena) != :Ok
raise ParseError.new "Error occurred copying proto"
end
message
end
end
def self.encode_internal(msg, encoding_options = 0)
temporary_arena = Google::Protobuf::FFI.create_arena
mini_table_ptr = Google::Protobuf::FFI.get_mini_table(descriptor)
size_ptr = ::FFI::MemoryPointer.new(:size_t, 1)
pointer_ptr = ::FFI::MemoryPointer.new(:pointer, 1)
encoding_status = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, pointer_ptr.to_ptr, size_ptr)
raise "Encoding failed due to #{encoding_status}" unless encoding_status == :Ok
yield pointer_ptr.read(:pointer), size_ptr.read(:size_t), mini_table_ptr
end
def method_missing_internal(method_name, *args, mode: nil)
raise ArgumentError.new "method_missing_internal called with invalid mode #{mode.inspect}" unless [:respond_to_missing?, :method_missing].include? mode
#TODO(jatl) not being allowed is not the same thing as not responding, but this is needed to pass tests
if method_name.to_s.end_with? '='
if self.class.send(:oneof_field_names).include? method_name.to_s[0..-2].to_sym
return false if mode == :respond_to_missing?
raise RuntimeError.new "Oneof accessors are read-only."
end
end
original_method_missing(method_name, *args) if mode == :method_missing
end
def clear_internal(field_def)
raise FrozenError.new "can't modify frozen #{self.class}" if frozen?
Google::Protobuf::FFI.clear_message_field(@msg, field_def)
end
def index_internal(name)
field_descriptor = self.class.descriptor.lookup(name)
get_field field_descriptor unless field_descriptor.nil?
end
#TODO(jatl) - well known types keeps us on our toes by overloading methods.
# How much of the public API needs to be defended?
def index_assign_internal(value, name: nil, field_descriptor: nil, wrap: false)
raise FrozenError.new "can't modify frozen #{self.class}" if frozen?
if field_descriptor.nil?
field_descriptor = self.class.descriptor.lookup(name)
if field_descriptor.nil?
raise ArgumentError.new "Unknown field: #{name}"
end
end
unless field_descriptor.send :set_value_on_message, value, @msg, @arena, wrap: wrap
raise RuntimeError.new "allocation failed"
end
end
##
# @param initial_value [Object] initial value of this Message
# @param arena [Arena] Optional; Arena where this message will be allocated
# @param msg [::FFI::Pointer] Optional; Message to initialize; creates
# one if omitted or nil.
def initialize(initial_value = nil, arena = nil, msg = nil)
@arena = arena || Google::Protobuf::FFI.create_arena
@msg = msg || Google::Protobuf::FFI.new_message_from_def(self.class.descriptor, @arena)
unless initial_value.nil?
raise ArgumentError.new "Expected hash arguments or message, not #{initial_value.class}" unless initial_value.respond_to? :each
field_def_ptr = ::FFI::MemoryPointer.new :pointer
oneof_def_ptr = ::FFI::MemoryPointer.new :pointer
initial_value.each do |key, value|
raise ArgumentError.new "Expected string or symbols as hash keys when initializing proto from hash." unless [String, Symbol].include? key.class
unless Google::Protobuf::FFI.find_msg_def_by_name self.class.descriptor, key.to_s, key.to_s.bytesize, field_def_ptr, oneof_def_ptr
raise ArgumentError.new "Unknown field name '#{key}' in initialization map entry."
end
raise NotImplementedError.new "Haven't added oneofsupport yet" unless oneof_def_ptr.get_pointer(0).null?
raise NotImplementedError.new "Expected a field def" if field_def_ptr.get_pointer(0).null?
field_descriptor = FieldDescriptor.from_native field_def_ptr.get_pointer(0)
next if value.nil?
if field_descriptor.map?
index_assign_internal(Google::Protobuf::Map.send(:construct_for_field, field_descriptor, @arena, value: value), name: key.to_s)
elsif field_descriptor.repeated?
index_assign_internal(RepeatedField.send(:construct_for_field, field_descriptor, @arena, values: value), name: key.to_s)
else
index_assign_internal(value, name: key.to_s)
end
end
end
# Should always be the last expression of the initializer to avoid
# leaking references to this object before construction is complete.
Google::Protobuf::OBJECT_CACHE.try_add @msg.address, self
end
##
# Gets a field of this message identified by the argument definition.
#
# @param field [FieldDescriptor] Descriptor of the field to get
def get_field(field, unwrap: false)
if field.map?
mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena
get_map_field(mutable_message_value[:map], field)
elsif field.repeated?
mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena
get_repeated_field(mutable_message_value[:array], field)
elsif field.sub_message?
return nil unless Google::Protobuf::FFI.get_message_has @msg, field
sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field)
if unwrap
if field.has?(self)
wrapper_message_value = Google::Protobuf::FFI.get_message_value @msg, field
fields = Google::Protobuf::FFI.field_count(sub_message_def)
raise "Sub message has #{fields} fields! Expected exactly 1." unless fields == 1
value_field_def = Google::Protobuf::FFI.get_field_by_number sub_message_def, 1
message_value = Google::Protobuf::FFI.get_message_value wrapper_message_value[:msg_val], value_field_def
convert_upb_to_ruby message_value, Google::Protobuf::FFI.get_c_type(value_field_def)
else
nil
end
else
mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena
sub_message = mutable_message[:msg]
Descriptor.send(:get_message, sub_message, sub_message_def, @arena)
end
else
c_type = field.send(:c_type)
message_value = Google::Protobuf::FFI.get_message_value @msg, field
if c_type == :enum
convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field)
else
convert_upb_to_ruby message_value, c_type
end
end
end
##
# @param array [::FFI::Pointer] Pointer to the Array
# @param field [Google::Protobuf::FieldDescriptor] Type of the repeated field
def get_repeated_field(array, field)
return nil if array.nil? or array.null?
repeated_field = OBJECT_CACHE.get(array.address)
if repeated_field.nil?
repeated_field = RepeatedField.send(:construct_for_field, field, @arena, array: array)
end
repeated_field
end
##
# @param map [::FFI::Pointer] Pointer to the Map
# @param field [Google::Protobuf::FieldDescriptor] Type of the map field
def get_map_field(map, field)
return nil if map.nil? or map.null?
map_field = OBJECT_CACHE.get(map.address)
if map_field.nil?
map_field = Google::Protobuf::Map.send(:construct_for_field, field, @arena, map: map)
end
map_field
end
end
end
end
end
end

@ -0,0 +1,53 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
private
SIZEOF_LONG = ::FFI::MemoryPointer.new(:long).size
SIZEOF_VALUE = ::FFI::Pointer::SIZE
def self.interpreter_supports_non_finalized_keys_in_weak_map?
! defined? JRUBY_VERSION
end
def self.cache_implementation
if interpreter_supports_non_finalized_keys_in_weak_map? and SIZEOF_LONG >= SIZEOF_VALUE
Google::Protobuf::ObjectCache
else
Google::Protobuf::LegacyObjectCache
end
end
public
OBJECT_CACHE = cache_implementation.new
end
end

@ -0,0 +1,111 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2022 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
module Google
module Protobuf
class OneofDescriptor
attr :descriptor_pool, :oneof_def
include Enumerable
# FFI Interface methods and setup
extend ::FFI::DataConverter
native_type ::FFI::Type::POINTER
class << self
prepend Google::Protobuf::Internal::TypeSafety
include Google::Protobuf::Internal::PointerHelper
# @param value [OneofDescriptor] FieldDescriptor to convert to an FFI native type
# @param _ [Object] Unused
def to_native(value, _ = nil)
oneof_def_ptr = value.instance_variable_get(:@oneof_def)
warn "Underlying oneof_def was nil!" if oneof_def_ptr.nil?
raise "Underlying oneof_def was null!" if !oneof_def_ptr.nil? and oneof_def_ptr.null?
oneof_def_ptr
end
##
# @param oneof_def [::FFI::Pointer] OneofDef pointer to be wrapped
# @param _ [Object] Unused
def from_native(oneof_def, _ = nil)
return nil if oneof_def.nil? or oneof_def.null?
message_descriptor = Google::Protobuf::FFI.get_oneof_containing_type oneof_def
raise RuntimeError.new "Message Descriptor is nil" if message_descriptor.nil?
file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor.to_native
descriptor_from_file_def(file_def, oneof_def)
end
end
def self.new(*arguments, &block)
raise "OneofDescriptor objects may not be created from Ruby."
end
def name
Google::Protobuf::FFI.get_oneof_name(self)
end
def each &block
n = Google::Protobuf::FFI.get_oneof_field_count(self)
0.upto(n-1) do |i|
yield(Google::Protobuf::FFI.get_oneof_field_by_index(self, i))
end
nil
end
private
def initialize(oneof_def, descriptor_pool)
@descriptor_pool = descriptor_pool
@oneof_def = oneof_def
end
def self.private_constructor(oneof_def, descriptor_pool)
instance = allocate
instance.send(:initialize, oneof_def, descriptor_pool)
instance
end
end
class FFI
# MessageDef
attach_function :get_oneof_by_name, :upb_MessageDef_FindOneofByNameWithSize, [Descriptor, :string, :size_t], OneofDescriptor
attach_function :get_oneof_by_index, :upb_MessageDef_Oneof, [Descriptor, :int], OneofDescriptor
# OneofDescriptor
attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDescriptor], :string
attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDescriptor], :int
attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDescriptor, :int], FieldDescriptor
attach_function :get_oneof_containing_type,:upb_OneofDef_ContainingType,[:pointer], Descriptor
# FieldDescriptor
attach_function :real_containing_oneof, :upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDescriptor
end
end
end

@ -0,0 +1,526 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
require 'forwardable'
#
# This class makes RepeatedField act (almost-) like a Ruby Array.
# It has convenience methods that extend the core C or Java based
# methods.
#
# This is a best-effort to mirror Array behavior. Two comments:
# 1) patches always welcome :)
# 2) if performance is an issue, feel free to rewrite the method
# in jruby and C. The source code has plenty of examples
#
# KNOWN ISSUES
# - #[]= doesn't allow less used approaches such as `arr[1, 2] = 'fizz'`
# - #concat should return the orig array
# - #push should accept multiple arguments and push them all at the same time
#
module Google
module Protobuf
class FFI
# Array
attach_function :append_array, :upb_Array_Append, [:Array, MessageValue.by_value, Internal::Arena], :bool
attach_function :get_msgval_at,:upb_Array_Get, [:Array, :size_t], MessageValue.by_value
attach_function :create_array, :upb_Array_New, [Internal::Arena, CType], :Array
attach_function :array_resize, :upb_Array_Resize, [:Array, :size_t, Internal::Arena], :bool
attach_function :array_set, :upb_Array_Set, [:Array, :size_t, MessageValue.by_value], :void
attach_function :array_size, :upb_Array_Size, [:Array], :size_t
end
class RepeatedField
extend Forwardable
# NOTE: using delegators rather than method_missing to make the
# relationship explicit instead of implicit
def_delegators :to_ary,
:&, :*, :-, :'<=>',
:assoc, :bsearch, :bsearch_index, :combination, :compact, :count,
:cycle, :dig, :drop, :drop_while, :eql?, :fetch, :find_index, :flatten,
:include?, :index, :inspect, :join,
:pack, :permutation, :product, :pretty_print, :pretty_print_cycle,
:rassoc, :repeated_combination, :repeated_permutation, :reverse,
:rindex, :rotate, :sample, :shuffle, :shelljoin,
:to_s, :transpose, :uniq, :|
include Enumerable
##
# call-seq:
# RepeatedField.new(type, type_class = nil, initial_values = [])
#
# Creates a new repeated field. The provided type must be a Ruby symbol, and
# an take on the same values as those accepted by FieldDescriptor#type=. If
# the type is :message or :enum, type_class must be non-nil, and must be the
# Ruby class or module returned by Descriptor#msgclass or
# EnumDescriptor#enummodule, respectively. An initial list of elements may also
# be provided.
def self.new(type, type_class = nil, initial_values = [])
instance = allocate
# TODO(jatl) This argument mangling doesn't agree with the type signature in the comments
# but is required to make unit tests pass;
if type_class.is_a?(Enumerable) and initial_values.empty? and ![:enum, :message].include?(type)
initial_values = type_class
type_class = nil
end
instance.send(:initialize, type, type_class: type_class, initial_values: initial_values)
instance
end
##
# call-seq:
# RepeatedField.each(&block)
#
# Invokes the block once for each element of the repeated field. RepeatedField
# also includes Enumerable; combined with this method, the repeated field thus
# acts like an ordinary Ruby sequence.
def each &block
each_msg_val do |element|
yield(convert_upb_to_ruby(element, type, descriptor, arena))
end
self
end
def [](*args)
count = length
if args.size < 1
raise ArgumentError.new "Index or range is a required argument."
end
if args[0].is_a? Range
if args.size > 1
raise ArgumentError.new "Expected 1 when passing Range argument, but got #{args.size}"
end
range = args[0]
# Handle begin-less and/or endless ranges, when supported.
index_of_first = range.respond_to?(:begin) ? range.begin : range.last
index_of_first = 0 if index_of_first.nil?
end_of_range = range.respond_to?(:end) ? range.end : range.last
index_of_last = end_of_range.nil? ? -1 : end_of_range
if index_of_last < 0
index_of_last += count
end
unless range.exclude_end? and !end_of_range.nil?
index_of_last += 1
end
index_of_first += count if index_of_first < 0
length = index_of_last - index_of_first
return [] if length.zero?
elsif args[0].is_a? Integer
index_of_first = args[0]
index_of_first += count if index_of_first < 0
if args.size > 2
raise ArgumentError.new "Expected 1 or 2 arguments, but got #{args.size}"
end
if args.size == 1 # No length specified, return one element
if array.null? or index_of_first < 0 or index_of_first >= count
return nil
else
return convert_upb_to_ruby(Google::Protobuf::FFI.get_msgval_at(array, index_of_first), type, descriptor, arena)
end
else
length = [args[1],count].min
end
else
raise NotImplementedError
end
if array.null? or index_of_first < 0 or index_of_first >= count
nil
else
if index_of_first + length > count
length = count - index_of_first
end
if length < 0
nil
else
subarray(index_of_first, length)
end
end
end
alias at []
def []=(index, value)
raise FrozenError if frozen?
count = length
index += count if index < 0
return nil if index < 0
if index >= count
resize(index+1)
empty_message_value = Google::Protobuf::FFI::MessageValue.new # Implicitly clear
count.upto(index-1) do |i|
Google::Protobuf::FFI.array_set(array, i, empty_message_value)
end
end
Google::Protobuf::FFI.array_set(array, index, convert_ruby_to_upb(value, arena, type, descriptor))
nil
end
def push(*elements)
raise FrozenError if frozen?
internal_push(*elements)
end
def <<(element)
raise FrozenError if frozen?
push element
end
def replace(replacements)
raise FrozenError if frozen?
clear
push(*replacements)
end
def clear
raise FrozenError if frozen?
resize 0
self
end
def length
array.null? ? 0 : Google::Protobuf::FFI.array_size(array)
end
alias size :length
def dup
instance = self.class.allocate
instance.send(:initialize, type, descriptor: descriptor, arena: arena)
each_msg_val do |element|
instance.send(:append_msg_val, element)
end
instance
end
alias clone dup
def ==(other)
return true if other.object_id == object_id
if other.is_a? RepeatedField
return false unless other.length == length
each_msg_val_with_index do |msg_val, i|
other_msg_val = Google::Protobuf::FFI.get_msgval_at(other.send(:array), i)
unless Google::Protobuf::FFI.message_value_equal(msg_val, other_msg_val, type, descriptor)
return false
end
end
return true
elsif other.is_a? Enumerable
return to_ary == other.to_a
end
false
end
##
# call-seq:
# RepeatedField.to_ary => array
#
# Used when converted implicitly into array, e.g. compared to an Array.
# Also called as a fallback of Object#to_a
def to_ary
return_value = []
each do |element|
return_value << element
end
return_value
end
def hash
return_value = 0
each_msg_val do |msg_val|
return_value = Google::Protobuf::FFI.message_value_hash(msg_val, type, descriptor, return_value)
end
return_value
end
def +(other)
if other.is_a? RepeatedField
if type != other.instance_variable_get(:@type) or descriptor != other.instance_variable_get(:@descriptor)
raise ArgumentError.new "Attempt to append RepeatedField with different element type."
end
fuse_arena(other.send(:arena))
super_set = dup
other.send(:each_msg_val) do |msg_val|
super_set.send(:append_msg_val, msg_val)
end
super_set
elsif other.is_a? Enumerable
super_set = dup
super_set.push(*other.to_a)
else
raise ArgumentError.new "Unknown type appending to RepeatedField"
end
end
def concat(other)
raise ArgumentError.new "Expected Enumerable, but got #{other.class}" unless other.is_a? Enumerable
push(*other.to_a)
end
def first(n=nil)
if n.nil?
return self[0]
elsif n < 0
raise ArgumentError, "negative array size"
else
return self[0...n]
end
end
def last(n=nil)
if n.nil?
return self[-1]
elsif n < 0
raise ArgumentError, "negative array size"
else
start = [self.size-n, 0].max
return self[start...self.size]
end
end
def pop(n=nil)
if n
results = []
n.times{ results << pop_one }
return results
else
return pop_one
end
end
def empty?
self.size == 0
end
# array aliases into enumerable
alias_method :each_index, :each_with_index
alias_method :slice, :[]
alias_method :values_at, :select
alias_method :map, :collect
class << self
def define_array_wrapper_method(method_name)
define_method(method_name) do |*args, &block|
arr = self.to_a
result = arr.send(method_name, *args)
self.replace(arr)
return result if result
return block ? block.call : result
end
end
private :define_array_wrapper_method
def define_array_wrapper_with_result_method(method_name)
define_method(method_name) do |*args, &block|
# result can be an Enumerator, Array, or nil
# Enumerator can sometimes be returned if a block is an optional argument and it is not passed in
# nil usually specifies that no change was made
result = self.to_a.send(method_name, *args, &block)
if result
new_arr = result.to_a
self.replace(new_arr)
if result.is_a?(Enumerator)
# generate a fresh enum; rewinding the exiting one, in Ruby 2.2, will
# reset the enum with the same length, but all the #next calls will
# return nil
result = new_arr.to_enum
# generate a wrapper enum so any changes which occur by a chained
# enum can be captured
ie = ProxyingEnumerator.new(self, result)
result = ie.to_enum
end
end
result
end
end
private :define_array_wrapper_with_result_method
end
%w(delete delete_at shift slice! unshift).each do |method_name|
define_array_wrapper_method(method_name)
end
%w(collect! compact! delete_if fill flatten! insert reverse!
rotate! select! shuffle! sort! sort_by! uniq!).each do |method_name|
define_array_wrapper_with_result_method(method_name)
end
alias_method :keep_if, :select!
alias_method :map!, :collect!
alias_method :reject!, :delete_if
# propagates changes made by user of enumerator back to the original repeated field.
# This only applies in cases where the calling function which created the enumerator,
# such as #sort!, modifies itself rather than a new array, such as #sort
class ProxyingEnumerator < Struct.new(:repeated_field, :external_enumerator)
def each(*args, &block)
results = []
external_enumerator.each_with_index do |val, i|
result = yield(val)
results << result
#nil means no change occurred from yield; usually occurs when #to_a is called
if result
repeated_field[i] = result if result != val
end
end
results
end
end
private
include Google::Protobuf::Internal::Convert
attr :name, :arena, :array, :type, :descriptor
def internal_push(*elements)
elements.each do |element|
append_msg_val convert_ruby_to_upb(element, arena, type, descriptor)
end
self
end
def pop_one
raise FrozenError if frozen?
count = length
return nil if length.zero?
last_element = Google::Protobuf::FFI.get_msgval_at(array, count-1)
return_value = convert_upb_to_ruby(last_element, type, descriptor, arena)
resize(count-1)
return_value
end
def subarray(start, length)
return_result = []
(start..(start + length - 1)).each do |i|
element = Google::Protobuf::FFI.get_msgval_at(array, i)
return_result << convert_upb_to_ruby(element, type, descriptor, arena)
end
return_result
end
def each_msg_val_with_index &block
n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array)
0.upto(n-1) do |i|
yield Google::Protobuf::FFI.get_msgval_at(array, i), i
end
end
def each_msg_val &block
each_msg_val_with_index do |msg_val, _|
yield msg_val
end
end
# @param msg_val [Google::Protobuf::FFI::MessageValue] Value to append
def append_msg_val(msg_val)
unless Google::Protobuf::FFI.append_array(array, msg_val, arena)
raise NoMemoryError.new "Could not allocate room for #{msg_val} in Arena"
end
end
# @param new_size [Integer] New size of the array
def resize(new_size)
unless Google::Protobuf::FFI.array_resize(array, new_size, arena)
raise NoMemoryError.new "Array resize to #{new_size} failed!"
end
end
def initialize(type, type_class: nil, initial_values: nil, name: nil, arena: nil, array: nil, descriptor: nil)
@name = name || 'RepeatedField'
raise ArgumentError.new "Expected argument type to be a Symbol" unless type.is_a? Symbol
field_number = Google::Protobuf::FFI::FieldType[type]
raise ArgumentError.new "Unsupported type '#{type}'" if field_number.nil?
if !descriptor.nil?
@descriptor = descriptor
elsif [:message, :enum].include? type
raise ArgumentError.new "Expected at least 2 arguments for message/enum." if type_class.nil?
descriptor = type_class.respond_to?(:descriptor) ? type_class.descriptor : nil
raise ArgumentError.new "Type class #{type_class} has no descriptor. Please pass a class or enum as returned by the DescriptorPool." if descriptor.nil?
@descriptor = descriptor
else
@descriptor = nil
end
@type = type
@arena = arena || Google::Protobuf::FFI.create_arena
@array = array || Google::Protobuf::FFI.create_array(@arena, @type)
unless initial_values.nil?
unless initial_values.is_a? Enumerable
raise ArgumentError.new "Expected array as initializer value for repeated field '#{name}' (given #{initial_values.class})."
end
internal_push(*initial_values)
end
# Should always be the last expression of the initializer to avoid
# leaking references to this object before construction is complete.
OBJECT_CACHE.try_add(@array.address, self)
end
# @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned
# @param values [Enumerable] Initial values; may be nil or empty
# @param arena [Arena] Owning message's arena
def self.construct_for_field(field, arena, values: nil, array: nil)
instance = allocate
options = {initial_values: values, name: field.name, arena: arena, array: array}
if [:enum, :message].include? field.type
options[:descriptor] = field.subtype
end
instance.send(:initialize, field.type, **options)
instance
end
def fuse_arena(arena)
arena.fuse(arena)
end
extend Google::Protobuf::Internal::Convert
def self.deep_copy(repeated_field)
instance = allocate
instance.send(:initialize, repeated_field.send(:type), descriptor: repeated_field.send(:descriptor))
instance.send(:resize, repeated_field.length)
new_array = instance.send(:array)
repeated_field.send(:each_msg_val_with_index) do |element, i|
Google::Protobuf::FFI.array_set(new_array, i, message_value_deep_copy(element, repeated_field.send(:type), repeated_field.send(:descriptor), instance.send(:arena)))
end
instance
end
end
end
end

@ -0,0 +1,73 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
require 'ffi-compiler/loader'
require 'google/protobuf/ffi/ffi'
require 'google/protobuf/ffi/internal/type_safety'
require 'google/protobuf/ffi/internal/pointer_helper'
require 'google/protobuf/ffi/internal/arena'
require 'google/protobuf/ffi/internal/convert'
require 'google/protobuf/ffi/descriptor'
require 'google/protobuf/ffi/enum_descriptor'
require 'google/protobuf/ffi/field_descriptor'
require 'google/protobuf/ffi/oneof_descriptor'
require 'google/protobuf/ffi/descriptor_pool'
require 'google/protobuf/ffi/file_descriptor'
require 'google/protobuf/ffi/map'
require 'google/protobuf/ffi/object_cache'
require 'google/protobuf/ffi/repeated_field'
require 'google/protobuf/ffi/message'
require 'google/protobuf/descriptor_dsl'
module Google
module Protobuf
def self.deep_copy(object)
case object
when RepeatedField
RepeatedField.send(:deep_copy, object)
when Google::Protobuf::Map
Google::Protobuf::Map.deep_copy(object)
when Google::Protobuf::MessageExts
object.class.send(:deep_copy, object.instance_variable_get(:@msg))
else
raise NotImplementedError
end
end
def self.discard_unknown(message)
raise FrozenError if message.frozen?
raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil?
unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128)
raise RuntimeError.new "Messages nested too deeply."
end
nil
end
end
end

@ -0,0 +1,43 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
if RUBY_PLATFORM == "java"
require 'json'
require 'google/protobuf_java'
else
begin
require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c"
rescue LoadError
require 'google/protobuf_c'
end
end
require 'google/protobuf/descriptor_dsl'
require 'google/protobuf/repeated_field'

@ -0,0 +1,94 @@
require "ffi-compiler/compile_task"
# # @param task [FFI::Compiler::CompileTask] task to configure
def configure_common_compile_task(task)
if FileUtils.pwd.include? 'ext'
src_dir = '.'
third_party_path = 'third_party/utf8_range'
else
src_dir = 'ext/google/protobuf_c'
third_party_path = 'ext/google/protobuf_c/third_party/utf8_range'
end
task.add_include_path third_party_path
task.add_define 'NDEBUG'
task.cflags << "-std=gnu99 -O3"
[
:convert, :defs, :map, :message, :protobuf, :repeated_field, :wrap_memcpy
].each { |file| task.exclude << "/#{file}.c" }
task.ext_dir = src_dir
task.source_dirs = [src_dir]
if RbConfig::CONFIG['target_os'] =~ /darwin|linux/
task.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement"
end
end
# FFI::CompilerTask's constructor walks the filesystem at task definition time
# to create subtasks for each source file, so files from third_party must be
# copied into place before the task is defined for it to work correctly.
# TODO(jatl) Is there a sane way to check for generated protos under lib too?
def with_generated_files
expected_path = FileUtils.pwd.include?('ext') ? 'third_party/utf8_range' : 'ext/google/protobuf_c/third_party/utf8_range'
if File.directory?(expected_path)
yield
else
task :default do
# It is possible, especially in cases like the first invocation of
# `rake test` following `rake clean` or a fresh checkout that the
# `copy_third_party` task has been executed since initial task definition.
# If so, run the task definition block now and invoke it explicitly.
if File.directory?(expected_path)
yield
Rake::Task[:default].invoke
else
raise "Missing directory #{File.absolute_path(expected_path)}." +
" Did you forget to run `rake copy_third_party` before building" +
" native extensions?"
end
end
end
end
desc "Compile Protobuf library for FFI"
namespace "ffi-protobuf" do
with_generated_files do
# Compile Ruby UPB separately in order to limit use of -DUPB_BUILD_API to one
# compilation unit.
desc "Compile UPB library for FFI"
namespace "ffi-upb" do
with_generated_files do
FFI::Compiler::CompileTask.new('ruby-upb') do |c|
configure_common_compile_task c
c.add_define "UPB_BUILD_API"
c.exclude << "/glue.c"
c.exclude << "/shared_message.c"
c.exclude << "/shared_convert.c"
if RbConfig::CONFIG['target_os'] =~ /darwin|linux/
c.cflags << "-fvisibility=hidden"
end
end
end
end
FFI::Compiler::CompileTask.new 'protobuf_c_ffi' do |c|
configure_common_compile_task c
# Ruby UPB was already compiled with different flags.
c.exclude << "/range2-neon.c"
c.exclude << "/range2-sse.c"
c.exclude << "/naive.c"
c.exclude << "/ruby-upb.c"
end
# Setup dependencies so that the .o files generated by building ffi-upb are
# available to link here.
# TODO(jatl) Can this be simplified? Can the single shared library be used
# instead of the object files?
protobuf_c_task = Rake::Task[:default]
protobuf_c_shared_lib_task = Rake::Task[protobuf_c_task.prereqs.last]
ruby_upb_shared_lib_task = Rake::Task[:"ffi-upb:default"].prereqs.first
Rake::Task[ruby_upb_shared_lib_task].prereqs.each do |dependency|
protobuf_c_shared_lib_task.prereqs.prepend dependency
end
end
end

@ -14,6 +14,15 @@ filegroup(
],
)
ruby_test(
name = "implementation",
srcs = ["implementation.rb"],
deps = [
"//ruby:protobuf",
"@protobuf_bundle//:test-unit",
],
)
ruby_test(
name = "basic",
srcs = ["basic.rb"],

@ -0,0 +1,37 @@
require 'google/protobuf'
require 'test/unit'
class BackendTest < Test::Unit::TestCase
# Verifies the implementation of Protobuf is the preferred one.
# See protobuf.rb for the logic that defines PREFER_FFI.
def test_prefer_ffi_aligns_with_implementation
expected = Google::Protobuf::PREFER_FFI ? :FFI : :NATIVE
assert_equal expected, Google::Protobuf::IMPLEMENTATION
end
def test_prefer_ffi
unless ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] =~ /ffi/i
omit"FFI implementation requires environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI to activate."
end
assert_equal true, Google::Protobuf::PREFER_FFI
end
def test_ffi_implementation
unless ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] =~ /ffi/i
omit "FFI implementation requires environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI to activate."
end
assert_equal :FFI, Google::Protobuf::IMPLEMENTATION
end
def test_prefer_native
if ENV.include?('PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION') and ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] !~ /native/i
omit"Native implementation requires omitting environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION or setting it to `NATIVE` to activate."
end
assert_equal false, Google::Protobuf::PREFER_FFI
end
def test_native_implementation
if ENV.include?('PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION') and ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] !~ /native/i
omit"Native implementation requires omitting environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION or setting it to `NATIVE` to activate."
end
assert_equal :NATIVE, Google::Protobuf::IMPLEMENTATION
end
end

@ -4,7 +4,7 @@ require 'test/unit'
class PlatformTest < Test::Unit::TestCase
def test_correct_implementation_for_platform
omit('OBJECT_CACHE not defined') unless defined? Google::Protobuf::OBJECT_CACHE
if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7.0') and Google::Protobuf::SIZEOF_LONG >= Google::Protobuf::SIZEOF_VALUE
if Google::Protobuf::SIZEOF_LONG >= Google::Protobuf::SIZEOF_VALUE and not defined? JRUBY_VERSION
assert_instance_of Google::Protobuf::ObjectCache, Google::Protobuf::OBJECT_CACHE
else
assert_instance_of Google::Protobuf::LegacyObjectCache, Google::Protobuf::OBJECT_CACHE

@ -9,6 +9,8 @@ package(
default_visibility = ["//src/google/protobuf:__subpackages__"],
)
licenses(["notice"])
# The current Rust Protobuf runtime for the build. Depending on the value of
# `:rust_proto_library_kernel` build setting it forwards to the cpp or upb kernels. This is the
# target that users are expected to depend on.
@ -62,7 +64,10 @@ rust_library(
"//src/google/protobuf:__subpackages__",
"//rust:__subpackages__",
],
deps = ["//rust/upb_kernel:upb_c_api"],
deps = [
":utf8",
"//rust/upb_kernel:upb_c_api",
],
)
rust_test(
@ -98,6 +103,7 @@ rust_library(
"//src/google/protobuf:__subpackages__",
"//rust:__subpackages__",
],
deps = [":utf8"],
)
rust_test(
@ -110,6 +116,12 @@ rust_test(
],
)
rust_library(
name = "utf8",
srcs = ["utf8.rs"],
visibility = ["//visibility:private"],
)
proto_lang_toolchain(
name = "proto_rust_upb_toolchain",
command_line = "--rust_out=experimental-codegen=enabled,kernel=upb:$(OUT)",

@ -47,6 +47,7 @@ use std::ptr::{self, NonNull};
/// dropped.
///
/// Note that this type is neither `Sync` nor `Send`.
#[derive(Debug)]
pub struct Arena {
#[allow(dead_code)]
ptr: NonNull<u8>,

@ -41,6 +41,7 @@ use std::fmt;
use std::hash::{Hash, Hasher};
use std::iter;
use std::ops::{Deref, DerefMut};
use utf8::Utf8Chunks;
/// This type will be replaced by something else in a future revision.
// TODO(b/285309330): remove this and any `impl`s using it.
@ -251,7 +252,7 @@ impl<'msg> Ord for BytesMut<'msg> {
}
/// The bytes were not valid UTF-8.
#[derive(Debug)]
#[derive(Debug, PartialEq)]
pub struct Utf8Error(pub(crate) ());
impl From<std::str::Utf8Error> for Utf8Error {
@ -355,16 +356,34 @@ impl ProtoStr {
///
/// [`U+FFFD REPLACEMENT CHARACTER`]: std::char::REPLACEMENT_CHARACTER
pub fn chars(&self) -> impl Iterator<Item = char> + '_ {
todo!("b/285309330: requires UTF-8 chunk splitting");
['a'].into_iter() // necessary for `impl Trait` to compile
Utf8Chunks::new(self.as_bytes()).flat_map(|chunk| {
let mut yield_replacement_char = !chunk.invalid().is_empty();
chunk.valid().chars().chain(iter::from_fn(move || {
// Yield a single replacement character for every
// non-empty invalid sequence.
yield_replacement_char.then(|| {
yield_replacement_char = false;
char::REPLACEMENT_CHARACTER
})
}))
})
}
/// Returns an iterator over chunks of UTF-8 data in the string.
///
/// An `Ok(&str)` is yielded for every valid UTF-8 chunk, and an
/// `Err(&[u8])` for non-UTF-8 chunks.
pub fn utf8_chunks(&self) -> Todo<'_> {
todo!("b/285309330: requires UTF-8 chunk splitting");
/// `Err(&[u8])` for each non-UTF-8 chunk. An `Err` will be emitted
/// multiple times in a row for contiguous invalid chunks. Each invalid
/// chunk in an `Err` has a maximum length of 3 bytes.
pub fn utf8_chunks(&self) -> impl Iterator<Item = Result<&str, &[u8]>> + '_ {
Utf8Chunks::new(self.as_bytes()).flat_map(|chunk| {
let valid = chunk.valid();
let invalid = chunk.invalid();
(!valid.is_empty())
.then_some(Ok(valid))
.into_iter()
.chain((!invalid.is_empty()).then_some(Err(invalid)))
})
}
/// Converts known-UTF-8 bytes to a `ProtoStr` without a check.
@ -407,14 +426,22 @@ impl<'msg> TryFrom<&'msg ProtoStr> for &'msg str {
}
impl fmt::Debug for ProtoStr {
fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
todo!("b/285309330: requires UTF-8 chunk splitting")
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&Utf8Chunks::new(self.as_bytes()).debug(), f)
}
}
impl fmt::Display for ProtoStr {
fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
todo!("b/285309330: requires UTF-8 chunk splitting")
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use std::fmt::Write as _;
for chunk in Utf8Chunks::new(self.as_bytes()) {
fmt::Display::fmt(chunk.valid(), f)?;
if !chunk.invalid().is_empty() {
// One invalid chunk is emitted per detected invalid sequence.
f.write_char(char::REPLACEMENT_CHARACTER)?;
}
}
Ok(())
}
}
@ -466,5 +493,207 @@ impl_bytes_partial_cmp!(
#[cfg(test)]
mod tests {
use super::*;
// TODO(b/285309330): Add unit tests
// Shorter and safe utility function to construct `ProtoStr` from bytes for
// testing.
fn test_proto_str(bytes: &[u8]) -> &ProtoStr {
// SAFETY: The runtime that this test executes under does not elide UTF-8 checks
// inside of `ProtoStr`.
unsafe { ProtoStr::from_utf8_unchecked(bytes) }
}
// UTF-8 test cases copied from:
// https://github.com/rust-lang/rust/blob/e8ee0b7/library/core/tests/str_lossy.rs
#[test]
fn proto_str_debug() {
assert_eq!(&format!("{:?}", test_proto_str(b"Hello There")), "\"Hello There\"");
assert_eq!(
&format!(
"{:?}",
test_proto_str(b"Hello\xC0\x80 There\xE6\x83 Goodbye\xf4\x8d\x93\xaa"),
),
"\"Hello\\xC0\\x80 There\\xE6\\x83 Goodbye\\u{10d4ea}\"",
);
}
#[test]
fn proto_str_display() {
assert_eq!(&test_proto_str(b"Hello There").to_string(), "Hello There");
assert_eq!(
&test_proto_str(b"Hello\xC0\x80 There\xE6\x83 Goodbye\xf4\x8d\x93\xaa").to_string(),
"Hello<EFBFBD><EFBFBD> There<EFBFBD> Goodbye\u{10d4ea}",
);
}
#[test]
fn proto_str_to_rust_str() {
assert_eq!(test_proto_str(b"hello").to_str(), Ok("hello"));
assert_eq!(test_proto_str("ศไทย中华Việt Nam".as_bytes()).to_str(), Ok("ศไทย中华Việt Nam"));
for expect_fail in [
&b"Hello\xC2 There\xFF Goodbye"[..],
b"Hello\xC0\x80 There\xE6\x83 Goodbye",
b"\xF5foo\xF5\x80bar",
b"\xF1foo\xF1\x80bar\xF1\x80\x80baz",
b"\xF4foo\xF4\x80bar\xF4\xBFbaz",
b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar",
b"\xED\xA0\x80foo\xED\xBF\xBFbar",
] {
assert_eq!(test_proto_str(expect_fail).to_str(), Err(Utf8Error(())), "{expect_fail:?}");
}
}
#[test]
fn proto_str_to_cow() {
assert_eq!(test_proto_str(b"hello").to_cow_lossy(), Cow::Borrowed("hello"));
assert_eq!(
test_proto_str("ศไทย中华Việt Nam".as_bytes()).to_cow_lossy(),
Cow::Borrowed("ศไทย中华Việt Nam")
);
for (bytes, lossy_str) in [
(&b"Hello\xC2 There\xFF Goodbye"[..], "Hello<EFBFBD> There<EFBFBD> Goodbye"),
(b"Hello\xC0\x80 There\xE6\x83 Goodbye", "Hello<EFBFBD><EFBFBD> There<EFBFBD> Goodbye"),
(b"\xF5foo\xF5\x80bar", "<EFBFBD>foo<EFBFBD><EFBFBD>bar"),
(b"\xF1foo\xF1\x80bar\xF1\x80\x80baz", "<EFBFBD>foo<EFBFBD>bar<EFBFBD>baz"),
(b"\xF4foo\xF4\x80bar\xF4\xBFbaz", "<EFBFBD>foo<EFBFBD>bar<EFBFBD><EFBFBD>baz"),
(b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar", "<EFBFBD><EFBFBD><EFBFBD><EFBFBD>foo\u{10000}bar"),
(b"\xED\xA0\x80foo\xED\xBF\xBFbar", "<EFBFBD><EFBFBD><EFBFBD>foo<EFBFBD><EFBFBD><EFBFBD>bar"),
] {
let cow = test_proto_str(bytes).to_cow_lossy();
assert!(matches!(cow, Cow::Owned(_)));
assert_eq!(&*cow, lossy_str, "{bytes:?}");
}
}
#[test]
fn proto_str_utf8_chunks() {
macro_rules! assert_chunks {
($bytes:expr, $($chunks:expr),* $(,)?) => {
let bytes = $bytes;
let chunks: &[Result<&str, &[u8]>] = &[$($chunks),*];
let s = test_proto_str(bytes);
let mut got_chunks = s.utf8_chunks();
let mut expected_chars = chunks.iter().copied();
assert!(got_chunks.eq(expected_chars), "{bytes:?} -> {chunks:?}");
};
}
assert_chunks!(b"hello", Ok("hello"));
assert_chunks!("ศไทย中华Việt Nam".as_bytes(), Ok("ศไทย中华Việt Nam"));
assert_chunks!(
b"Hello\xC2 There\xFF Goodbye",
Ok("Hello"),
Err(b"\xC2"),
Ok(" There"),
Err(b"\xFF"),
Ok(" Goodbye"),
);
assert_chunks!(
b"Hello\xC0\x80 There\xE6\x83 Goodbye",
Ok("Hello"),
Err(b"\xC0"),
Err(b"\x80"),
Ok(" There"),
Err(b"\xE6\x83"),
Ok(" Goodbye"),
);
assert_chunks!(
b"\xF5foo\xF5\x80bar",
Err(b"\xF5"),
Ok("foo"),
Err(b"\xF5"),
Err(b"\x80"),
Ok("bar"),
);
assert_chunks!(
b"\xF1foo\xF1\x80bar\xF1\x80\x80baz",
Err(b"\xF1"),
Ok("foo"),
Err(b"\xF1\x80"),
Ok("bar"),
Err(b"\xF1\x80\x80"),
Ok("baz"),
);
assert_chunks!(
b"\xF4foo\xF4\x80bar\xF4\xBFbaz",
Err(b"\xF4"),
Ok("foo"),
Err(b"\xF4\x80"),
Ok("bar"),
Err(b"\xF4"),
Err(b"\xBF"),
Ok("baz"),
);
assert_chunks!(
b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar",
Err(b"\xF0"),
Err(b"\x80"),
Err(b"\x80"),
Err(b"\x80"),
Ok("foo\u{10000}bar"),
);
assert_chunks!(
b"\xED\xA0\x80foo\xED\xBF\xBFbar",
Err(b"\xED"),
Err(b"\xA0"),
Err(b"\x80"),
Ok("foo"),
Err(b"\xED"),
Err(b"\xBF"),
Err(b"\xBF"),
Ok("bar"),
);
}
#[test]
fn proto_str_chars() {
macro_rules! assert_chars {
($bytes:expr, $chars:expr) => {
let bytes = $bytes;
let chars = $chars;
let s = test_proto_str(bytes);
let mut got_chars = s.chars();
let mut expected_chars = chars.into_iter();
assert!(got_chars.eq(expected_chars), "{bytes:?} -> {chars:?}");
};
}
assert_chars!(b"hello", ['h', 'e', 'l', 'l', 'o']);
assert_chars!(
"ศไทย中华Việt Nam".as_bytes(),
['ศ', 'ไ', 'ท', 'ย', '中', '华', 'V', 'i', 'ệ', 't', ' ', 'N', 'a', 'm']
);
assert_chars!(
b"Hello\xC2 There\xFF Goodbye",
[
'H', 'e', 'l', 'l', 'o', '<EFBFBD>', ' ', 'T', 'h', 'e', 'r', 'e', '<EFBFBD>', ' ', 'G', 'o',
'o', 'd', 'b', 'y', 'e'
]
);
assert_chars!(
b"Hello\xC0\x80 There\xE6\x83 Goodbye",
[
'H', 'e', 'l', 'l', 'o', '<EFBFBD>', '<EFBFBD>', ' ', 'T', 'h', 'e', 'r', 'e', '<EFBFBD>', ' ', 'G',
'o', 'o', 'd', 'b', 'y', 'e'
]
);
assert_chars!(b"\xF5foo\xF5\x80bar", ['<EFBFBD>', 'f', 'o', 'o', '<EFBFBD>', '<EFBFBD>', 'b', 'a', 'r']);
assert_chars!(
b"\xF1foo\xF1\x80bar\xF1\x80\x80baz",
['<EFBFBD>', 'f', 'o', 'o', '<EFBFBD>', 'b', 'a', 'r', '<EFBFBD>', 'b', 'a', 'z']
);
assert_chars!(
b"\xF4foo\xF4\x80bar\xF4\xBFbaz",
['<EFBFBD>', 'f', 'o', 'o', '<EFBFBD>', 'b', 'a', 'r', '<EFBFBD>', '<EFBFBD>', 'b', 'a', 'z']
);
assert_chars!(
b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar",
['<EFBFBD>', '<EFBFBD>', '<EFBFBD>', '<EFBFBD>', 'f', 'o', 'o', '\u{10000}', 'b', 'a', 'r']
);
assert_chars!(
b"\xED\xA0\x80foo\xED\xBF\xBFbar",
['<EFBFBD>', '<EFBFBD>', '<EFBFBD>', 'f', 'o', 'o', '<EFBFBD>', '<EFBFBD>', '<EFBFBD>', 'b', 'a', 'r']
);
}
}

@ -23,7 +23,9 @@ alias(
rust_proto_library(
name = "unittest_rust_proto",
testonly = True,
visibility = ["//rust/test/shared:__subpackages__"],
visibility = [
"//visibility:private", # Only private by automation, not intent. Owner may accept CLs adding visibility. See go/scheuklappen#explicit-private.
],
deps = [UNITTEST_PROTO_TARGET],
)
@ -63,20 +65,14 @@ proto_library(
rust_upb_proto_library(
name = "parent_upb_rust_proto",
testonly = True,
visibility = [
"//rust/test/shared:__subpackages__",
"//rust/test/upb:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":parent_proto"],
)
rust_upb_proto_library(
name = "child_upb_rust_proto",
testonly = True,
visibility = [
"//rust/test/shared:__subpackages__",
"//rust/test/upb:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":child_proto"],
)
@ -95,20 +91,14 @@ cc_proto_library(
rust_cc_proto_library(
name = "parent_cc_rust_proto",
testonly = True,
visibility = [
"//rust/test/cpp:__subpackages__",
"//rust/test/shared:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":parent_cc_proto"],
)
rust_cc_proto_library(
name = "child_cc_rust_proto",
testonly = True,
visibility = [
"//rust/test/cpp:__subpackages__",
"//rust/test/shared:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":child_cc_proto"],
)
@ -127,20 +117,14 @@ cc_proto_library(
rust_cc_proto_library(
name = "dots_in_package_cc_rust_proto",
testonly = True,
visibility = [
"//rust/test/cpp:__subpackages__",
"//rust/test/shared:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":dots_in_package_cc_proto"],
)
rust_upb_proto_library(
name = "dots_in_package_upb_rust_proto",
testonly = True,
visibility = [
"//rust/test/shared:__subpackages__",
"//rust/test/upb:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":dots_in_package_proto"],
)
@ -170,20 +154,14 @@ cc_proto_library(
rust_cc_proto_library(
name = "no_package_cc_rust_proto",
testonly = True,
visibility = [
"//rust/test/cpp:__subpackages__",
"//rust/test/shared:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":no_package_cc_proto"],
)
rust_upb_proto_library(
name = "no_package_upb_rust_proto",
testonly = True,
visibility = [
"//rust/test/shared:__subpackages__",
"//rust/test/upb:__subpackages__",
],
visibility = ["//rust/test/shared:__subpackages__"],
deps = [":no_package_proto"],
)

@ -225,3 +225,13 @@ fn test_optional_bytes_accessors() {
msg.optional_bytes_set(Some(b""));
assert_eq!(msg.optional_bytes().unwrap(), b"");
}
#[test]
#[should_panic = "b/285309454"]
#[allow(unreachable_code)]
fn test_singular_msg_field() {
let msg = TestAllTypes::new();
// TODO("b/285309454"): fetch the inner integer `bb`
// call should look like msg.optional_nested_message().bb()
match msg.optional_nested_message() {}
}

@ -61,6 +61,7 @@ pub struct RawArenaData {
/// dropped.
///
/// Note that this type is neither `Sync` nor `Send`.
#[derive(Debug)]
pub struct Arena {
// Safety invariant: this must always be a valid arena
raw: RawArena,

@ -0,0 +1,328 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google LLC. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google LLC. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//! Lossy UTF-8 processing utilities.
#![deny(unsafe_op_in_unsafe_fn)]
// TODO(b/291781742): Replace this with the `std` versions once stable.
// This is adapted from https://github.com/rust-lang/rust/blob/e8ee0b7/library/core/src/str/lossy.rs
// The adaptations:
// - remove `#[unstable]` attributes.
// - replace `crate`/`super` paths with their `std` equivalents in code and
// examples.
// - include `UTF8_CHAR_WIDTH`/`utf8_char_width` from `core::str::validations`.
// - use a custom `split_at_unchecked` instead of the nightly one
use std::fmt;
use std::fmt::Formatter;
use std::fmt::Write;
use std::iter::FusedIterator;
use std::str::from_utf8_unchecked;
/// An item returned by the [`Utf8Chunks`] iterator.
///
/// A `Utf8Chunk` stores a sequence of [`u8`] up to the first broken character
/// when decoding a UTF-8 string.
///
/// # Examples
///
/// ```
/// use utf8::Utf8Chunks;
///
/// // An invalid UTF-8 string
/// let bytes = b"foo\xF1\x80bar";
///
/// // Decode the first `Utf8Chunk`
/// let chunk = Utf8Chunks::new(bytes).next().unwrap();
///
/// // The first three characters are valid UTF-8
/// assert_eq!("foo", chunk.valid());
///
/// // The fourth character is broken
/// assert_eq!(b"\xF1\x80", chunk.invalid());
/// ```
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Utf8Chunk<'a> {
valid: &'a str,
invalid: &'a [u8],
}
impl<'a> Utf8Chunk<'a> {
/// Returns the next validated UTF-8 substring.
///
/// This substring can be empty at the start of the string or between
/// broken UTF-8 characters.
#[must_use]
pub fn valid(&self) -> &'a str {
self.valid
}
/// Returns the invalid sequence that caused a failure.
///
/// The returned slice will have a maximum length of 3 and starts after the
/// substring given by [`valid`]. Decoding will resume after this sequence.
///
/// If empty, this is the last chunk in the string. If non-empty, an
/// unexpected byte was encountered or the end of the input was reached
/// unexpectedly.
///
/// Lossy decoding would replace this sequence with [`U+FFFD REPLACEMENT
/// CHARACTER`].
///
/// [`valid`]: Self::valid
/// [`U+FFFD REPLACEMENT CHARACTER`]: std::char::REPLACEMENT_CHARACTER
#[must_use]
pub fn invalid(&self) -> &'a [u8] {
self.invalid
}
}
#[must_use]
pub struct Debug<'a>(&'a [u8]);
impl fmt::Debug for Debug<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.write_char('"')?;
for chunk in Utf8Chunks::new(self.0) {
// Valid part.
// Here we partially parse UTF-8 again which is suboptimal.
{
let valid = chunk.valid();
let mut from = 0;
for (i, c) in valid.char_indices() {
let esc = c.escape_debug();
// If char needs escaping, flush backlog so far and write, else skip
if esc.len() != 1 {
f.write_str(&valid[from..i])?;
for c in esc {
f.write_char(c)?;
}
from = i + c.len_utf8();
}
}
f.write_str(&valid[from..])?;
}
// Broken parts of string as hex escape.
for &b in chunk.invalid() {
write!(f, "\\x{:02X}", b)?;
}
}
f.write_char('"')
}
}
/// An iterator used to decode a slice of mostly UTF-8 bytes to string slices
/// ([`&str`]) and byte slices ([`&[u8]`][byteslice]).
///
/// If you want a simple conversion from UTF-8 byte slices to string slices,
/// [`from_utf8`] is easier to use.
///
/// [byteslice]: slice
/// [`from_utf8`]: std::str::from_utf8
///
/// # Examples
///
/// This can be used to create functionality similar to
/// [`String::from_utf8_lossy`] without allocating heap memory:
///
/// ```
/// use utf8::Utf8Chunks;
///
/// fn from_utf8_lossy<F>(input: &[u8], mut push: F) where F: FnMut(&str) {
/// for chunk in Utf8Chunks::new(input) {
/// push(chunk.valid());
///
/// if !chunk.invalid().is_empty() {
/// push("\u{FFFD}");
/// }
/// }
/// }
/// ```
#[must_use = "iterators are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct Utf8Chunks<'a> {
source: &'a [u8],
}
impl<'a> Utf8Chunks<'a> {
/// Creates a new iterator to decode the bytes.
pub fn new(bytes: &'a [u8]) -> Self {
Self { source: bytes }
}
#[doc(hidden)]
pub fn debug(&self) -> Debug<'_> {
Debug(self.source)
}
}
impl<'a> Iterator for Utf8Chunks<'a> {
type Item = Utf8Chunk<'a>;
fn next(&mut self) -> Option<Utf8Chunk<'a>> {
if self.source.is_empty() {
return None;
}
const TAG_CONT_U8: u8 = 128;
fn safe_get(xs: &[u8], i: usize) -> u8 {
*xs.get(i).unwrap_or(&0)
}
let mut i = 0;
let mut valid_up_to = 0;
while i < self.source.len() {
// SAFETY: `i < self.source.len()` per previous line.
// For some reason the following are both significantly slower:
// while let Some(&byte) = self.source.get(i) {
// while let Some(byte) = self.source.get(i).copied() {
let byte = unsafe { *self.source.get_unchecked(i) };
i += 1;
if byte < 128 {
// This could be a `1 => ...` case in the match below, but for
// the common case of all-ASCII inputs, we bypass loading the
// sizeable UTF8_CHAR_WIDTH table into cache.
} else {
let w = utf8_char_width(byte);
match w {
2 => {
if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
break;
}
i += 1;
}
3 => {
match (byte, safe_get(self.source, i)) {
(0xE0, 0xA0..=0xBF) => (),
(0xE1..=0xEC, 0x80..=0xBF) => (),
(0xED, 0x80..=0x9F) => (),
(0xEE..=0xEF, 0x80..=0xBF) => (),
_ => break,
}
i += 1;
if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
break;
}
i += 1;
}
4 => {
match (byte, safe_get(self.source, i)) {
(0xF0, 0x90..=0xBF) => (),
(0xF1..=0xF3, 0x80..=0xBF) => (),
(0xF4, 0x80..=0x8F) => (),
_ => break,
}
i += 1;
if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
break;
}
i += 1;
if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
break;
}
i += 1;
}
_ => break,
}
}
valid_up_to = i;
}
/// # Safety
/// `index` must be in-bounds for `x`
unsafe fn split_at_unchecked(x: &[u8], index: usize) -> (&[u8], &[u8]) {
// SAFTEY: in-bounds as promised by the caller
unsafe { (x.get_unchecked(..index), x.get_unchecked(index..)) }
}
// SAFETY: `i <= self.source.len()` because it is only ever incremented
// via `i += 1` and in between every single one of those increments, `i`
// is compared against `self.source.len()`. That happens either
// literally by `i < self.source.len()` in the while-loop's condition,
// or indirectly by `safe_get(self.source, i) & 192 != TAG_CONT_U8`. The
// loop is terminated as soon as the latest `i += 1` has made `i` no
// longer less than `self.source.len()`, which means it'll be at most
// equal to `self.source.len()`.
let (inspected, remaining) = unsafe { split_at_unchecked(self.source, i) };
self.source = remaining;
// SAFETY: `valid_up_to <= i` because it is only ever assigned via
// `valid_up_to = i` and `i` only increases.
let (valid, invalid) = unsafe { split_at_unchecked(inspected, valid_up_to) };
Some(Utf8Chunk {
// SAFETY: All bytes up to `valid_up_to` are valid UTF-8.
valid: unsafe { from_utf8_unchecked(valid) },
invalid,
})
}
}
impl FusedIterator for Utf8Chunks<'_> {}
impl fmt::Debug for Utf8Chunks<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("Utf8Chunks").field("source", &self.debug()).finish()
}
}
// https://tools.ietf.org/html/rfc3629
const UTF8_CHAR_WIDTH: &[u8; 256] = &[
// 1 2 3 4 5 6 7 8 9 A B C D E F
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // A
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // B
0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E
4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // F
];
/// Given a first byte, determines how many bytes are in this UTF-8 character.
#[must_use]
#[inline]
const fn utf8_char_width(b: u8) -> usize {
UTF8_CHAR_WIDTH[b as usize] as usize
}

@ -385,6 +385,7 @@ set(libprotoc_srcs
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/accessors.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_bytes.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_message.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/context.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/generator.cc

@ -139,7 +139,7 @@ genrule(
--proto_path=$$(dirname $$(dirname $$(dirname $(location any.proto)))) \
$(SRCS)
""",
exec_tools = ["//:protoc"],
tools = ["//:protoc"],
visibility = ["//visibility:private"],
)

@ -380,10 +380,6 @@ void Any::InternalSwap(Any* other) {
} // namespace google
namespace google {
namespace protobuf {
template<> PROTOBUF_NOINLINE ::google::protobuf::Any*
Arena::CreateMaybeMessage< ::google::protobuf::Any >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::Any >(arena);
}
} // namespace protobuf
} // namespace google
// @@protoc_insertion_point(global_scope)

@ -58,8 +58,6 @@ namespace protobuf {
class Any;
struct AnyDefaultTypeInternal;
PROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_;
template <>
PROTOBUF_EXPORT ::google::protobuf::Any* Arena::CreateMaybeMessage<::google::protobuf::Any>(Arena*);
} // namespace protobuf
} // namespace google

@ -587,9 +587,12 @@ void Api::MergeImpl(::google::protobuf::Message& to_msg, const ::google::protobu
::uint32_t cached_has_bits = 0;
(void) cached_has_bits;
_this->_internal_mutable_methods()->MergeFrom(from._internal_methods());
_this->_internal_mutable_options()->MergeFrom(from._internal_options());
_this->_internal_mutable_mixins()->MergeFrom(from._internal_mixins());
_this->_internal_mutable_methods()->MergeFrom(
from._internal_methods());
_this->_internal_mutable_options()->MergeFrom(
from._internal_options());
_this->_internal_mutable_mixins()->MergeFrom(
from._internal_mixins());
if (!from._internal_name().empty()) {
_this->_internal_set_name(from._internal_name());
}
@ -970,7 +973,8 @@ void Method::MergeImpl(::google::protobuf::Message& to_msg, const ::google::prot
::uint32_t cached_has_bits = 0;
(void) cached_has_bits;
_this->_internal_mutable_options()->MergeFrom(from._internal_options());
_this->_internal_mutable_options()->MergeFrom(
from._internal_options());
if (!from._internal_name().empty()) {
_this->_internal_set_name(from._internal_name());
}
@ -1265,18 +1269,6 @@ void Mixin::InternalSwap(Mixin* other) {
} // namespace google
namespace google {
namespace protobuf {
template<> PROTOBUF_NOINLINE ::google::protobuf::Api*
Arena::CreateMaybeMessage< ::google::protobuf::Api >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::Api >(arena);
}
template<> PROTOBUF_NOINLINE ::google::protobuf::Method*
Arena::CreateMaybeMessage< ::google::protobuf::Method >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::Method >(arena);
}
template<> PROTOBUF_NOINLINE ::google::protobuf::Mixin*
Arena::CreateMaybeMessage< ::google::protobuf::Mixin >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::Mixin >(arena);
}
} // namespace protobuf
} // namespace google
// @@protoc_insertion_point(global_scope)

@ -66,12 +66,6 @@ PROTOBUF_EXPORT extern MethodDefaultTypeInternal _Method_default_instance_;
class Mixin;
struct MixinDefaultTypeInternal;
PROTOBUF_EXPORT extern MixinDefaultTypeInternal _Mixin_default_instance_;
template <>
PROTOBUF_EXPORT ::google::protobuf::Api* Arena::CreateMaybeMessage<::google::protobuf::Api>(Arena*);
template <>
PROTOBUF_EXPORT ::google::protobuf::Method* Arena::CreateMaybeMessage<::google::protobuf::Method>(Arena*);
template <>
PROTOBUF_EXPORT ::google::protobuf::Mixin* Arena::CreateMaybeMessage<::google::protobuf::Mixin>(Arena*);
} // namespace protobuf
} // namespace google

@ -12,6 +12,7 @@ load(
load("@rules_proto//proto:defs.bzl", "proto_library")
load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test")
load("//build_defs:cpp_opts.bzl", "COPTS")
load("test_plugin_injection.bzl", "inject_plugin_paths")
proto_library(
name = "plugin_proto",
@ -267,6 +268,8 @@ cc_test(
],
)
inject_plugin_paths("test_plugin_paths")
cc_test(
name = "command_line_interface_unittest",
srcs = ["command_line_interface_unittest.cc"],
@ -278,8 +281,10 @@ cc_test(
],
}) + [
# Note: This only works on Windows with symlinks and runfiles enabled.
"-DGOOGLE_PROTOBUF_FAKE_PLUGIN_PATH=\\\"$(rootpath :fake_plugin)\\\"",
"-DGOOGLE_PROTOBUF_TEST_PLUGIN_PATH=\\\"$(rootpath :test_plugin)\\\"",
# "-DGOOGLE_PROTOBUF_FAKE_PLUGIN_PATH=\\\"$(rootpath :fake_plugin)\\\"",
# "-DGOOGLE_PROTOBUF_TEST_PLUGIN_PATH=\\\"$(rootpath :test_plugin)\\\"",
# Workaround for https://github.com/bazelbuild/bazel/issues/19124.
"-DGOOGLE_PROTOBUF_USE_BAZEL_GENERATED_PLUGIN_PATHS=1",
],
data = [
":fake_plugin",
@ -293,6 +298,7 @@ cc_test(
":command_line_interface",
":command_line_interface_tester",
":mock_code_generator",
":test_plugin_paths",
"//:protobuf",
"//src/google/protobuf:cc_test_protos",
"//src/google/protobuf:test_textproto",

@ -73,6 +73,11 @@
#include "google/protobuf/unittest.pb.h"
#include "google/protobuf/unittest_custom_options.pb.h"
#ifdef GOOGLE_PROTOBUF_USE_BAZEL_GENERATED_PLUGIN_PATHS
// This is needed because of https://github.com/bazelbuild/bazel/issues/19124.
#include "google/protobuf/compiler/test_plugin_paths.h"
#endif // GOOGLE_PROTOBUF_USE_BAZEL_GENERATED_PLUGIN_PATHS
// Must be included last.
#include "google/protobuf/port_def.inc"

@ -117,6 +117,43 @@ std::vector<Sub> FieldVars(const FieldDescriptor* field, const Options& opts) {
return vars;
}
FieldGeneratorBase::FieldGeneratorBase(const FieldDescriptor* descriptor,
const Options& options,
MessageSCCAnalyzer* scc)
: descriptor_(descriptor), options_(options) {
should_split_ = ShouldSplit(descriptor, options);
is_oneof_ = descriptor->real_containing_oneof() != nullptr;
switch (descriptor->cpp_type()) {
case FieldDescriptor::CPPTYPE_ENUM:
case FieldDescriptor::CPPTYPE_INT32:
case FieldDescriptor::CPPTYPE_INT64:
case FieldDescriptor::CPPTYPE_UINT32:
case FieldDescriptor::CPPTYPE_UINT64:
case FieldDescriptor::CPPTYPE_FLOAT:
case FieldDescriptor::CPPTYPE_DOUBLE:
case FieldDescriptor::CPPTYPE_BOOL:
is_trivial_ = !(descriptor->is_repeated() || descriptor->is_map());
has_trivial_value_ = is_trivial_;
break;
case FieldDescriptor::CPPTYPE_STRING:
is_string_ = true;
string_type_ = descriptor->options().ctype();
is_inlined_ = IsStringInlined(descriptor, options);
is_bytes_ = descriptor->type() == FieldDescriptor::TYPE_BYTES;
break;
case FieldDescriptor::CPPTYPE_MESSAGE:
is_message_ = true;
is_group_ = descriptor->type() == FieldDescriptor::TYPE_GROUP;
is_foreign_ = IsCrossFileMessage(descriptor);
is_lazy_ = IsLazy(descriptor, options, scc);
is_weak_ = IsImplicitWeakField(descriptor, options, scc);
if (!(descriptor->is_repeated() || descriptor->is_map())) {
has_trivial_value_ = !is_lazy_;
}
break;
}
}
void FieldGeneratorBase::GenerateAggregateInitializer(io::Printer* p) const {
if (ShouldSplit(descriptor_, options_)) {
p->Emit(R"cc(
@ -144,7 +181,7 @@ void FieldGeneratorBase::GenerateCopyAggregateInitializer(
}
void FieldGeneratorBase::GenerateCopyConstructorCode(io::Printer* p) const {
if (ShouldSplit(descriptor_, options_)) {
if (should_split()) {
// There is no copy constructor for the `Split` struct, so we need to copy
// the value here.
Formatter format(p, variables_);

@ -62,14 +62,57 @@ namespace cpp {
// matter of clean composability.
class FieldGeneratorBase {
public:
FieldGeneratorBase(const FieldDescriptor* descriptor, const Options& options)
: descriptor_(descriptor), options_(options) {}
FieldGeneratorBase(const FieldDescriptor* descriptor, const Options& options,
MessageSCCAnalyzer* scc_analyzer);
FieldGeneratorBase(const FieldGeneratorBase&) = delete;
FieldGeneratorBase& operator=(const FieldGeneratorBase&) = delete;
virtual ~FieldGeneratorBase() = 0;
// Returns true if this field should be placed in the cold 'Split' section.
bool should_split() const { return should_split_; }
// Returns true if this field is trivial. (int, float, double, enum, bool)
bool is_trivial() const { return is_trivial_; }
// Returns true if the field value itself is trivial, i.e., the field is
// trivial, or a (raw) pointer value to a singular, non lazy message.
bool has_trivial_value() const { return has_trivial_value_; }
// Returns true if the field is a singular or repeated message.
// This includes group message types. To explicitly check if a message
// type is a group type, use the `is_group()` function,
bool is_message() const { return is_message_; }
// Returns true if the field is a group message field (TYPE_GROUP).
bool is_group() const { return is_group_; }
// Returns true if the field is a weak message
bool is_weak() const { return is_weak_; }
// Returns true if the field is a lazy message.
bool is_lazy() const { return is_lazy_; }
// Returns true if the field is a foreign message field.
bool is_foreign() const { return is_foreign_; }
// Returns true if the field is a string field.
bool is_string() const { return is_string_; }
// Returns true if the field API uses bytes (void) instead of chars.
bool is_bytes() const { return is_bytes_; }
// Returns the public API string type for string fields.
FieldOptions::CType string_type() const { return string_type_; }
// Returns true if this field is part of a oneof field.
bool is_oneof() const { return is_oneof_; }
// Returns true if the field should be inlined instead of dynamically
// allocated. Applies to string and message value.
bool is_inlined() const { return is_inlined_; }
virtual std::vector<io::Printer::Sub> MakeVars() const { return {}; }
virtual void GeneratePrivateMembers(io::Printer* p) const = 0;
@ -131,6 +174,21 @@ class FieldGeneratorBase {
const FieldDescriptor* descriptor_;
const Options& options_;
absl::flat_hash_map<absl::string_view, std::string> variables_;
private:
bool should_split_ = false;
bool is_trivial_ = false;
bool has_trivial_value_ = false;
bool is_message_ = false;
bool is_group_ = false;
bool is_string_ = false;
bool is_bytes_ = false;
bool is_inlined_ = false;
bool is_foreign_ = false;
bool is_lazy_ = false;
bool is_weak_ = false;
bool is_oneof_ = false;
FieldOptions::CType string_type_ = FieldOptions::STRING;
};
inline FieldGeneratorBase::~FieldGeneratorBase() = default;

@ -77,7 +77,8 @@ void SetCordVariables(
class CordFieldGenerator : public FieldGeneratorBase {
public:
CordFieldGenerator(const FieldDescriptor* descriptor, const Options& options);
CordFieldGenerator(const FieldDescriptor* descriptor, const Options& options,
MessageSCCAnalyzer* scc);
~CordFieldGenerator() override = default;
void GeneratePrivateMembers(io::Printer* printer) const override;
@ -103,7 +104,7 @@ class CordFieldGenerator : public FieldGeneratorBase {
class CordOneofFieldGenerator : public CordFieldGenerator {
public:
CordOneofFieldGenerator(const FieldDescriptor* descriptor,
const Options& options);
const Options& options, MessageSCCAnalyzer* scc);
~CordOneofFieldGenerator() override = default;
void GeneratePrivateMembers(io::Printer* printer) const override;
@ -123,8 +124,9 @@ class CordOneofFieldGenerator : public CordFieldGenerator {
CordFieldGenerator::CordFieldGenerator(const FieldDescriptor* descriptor,
const Options& options)
: FieldGeneratorBase(descriptor, options) {
const Options& options,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(descriptor, options, scc) {
SetCordVariables(descriptor, &variables_, options);
}
@ -215,7 +217,7 @@ void CordFieldGenerator::GenerateSwappingCode(io::Printer* printer) const {
}
void CordFieldGenerator::GenerateConstructorCode(io::Printer* printer) const {
ABSL_CHECK(!ShouldSplit(descriptor_, options_));
ABSL_CHECK(!should_split());
Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) {
format("$field$ = ::absl::string_view($default$, $default_length$);\n");
@ -224,7 +226,7 @@ void CordFieldGenerator::GenerateConstructorCode(io::Printer* printer) const {
void CordFieldGenerator::GenerateDestructorCode(io::Printer* printer) const {
Formatter format(printer, variables_);
if (ShouldSplit(descriptor_, options_)) {
if (should_split()) {
// A cord field in the `Split` struct is automatically destroyed when the
// split pointer is deleted and should not be explicitly destroyed here.
return;
@ -270,7 +272,7 @@ void CordFieldGenerator::GenerateConstexprAggregateInitializer(
)cc");
} else {
p->Emit(
{{"Split", ShouldSplit(descriptor_, options_) ? "Split::" : ""}},
{{"Split", should_split() ? "Split::" : ""}},
R"cc(
/*decltype($field$)*/ {::absl::strings_internal::MakeStringConstant(
$classname$::Impl_::$Split$_default_$name$_func_{})},
@ -279,7 +281,7 @@ void CordFieldGenerator::GenerateConstexprAggregateInitializer(
}
void CordFieldGenerator::GenerateAggregateInitializer(io::Printer* p) const {
if (ShouldSplit(descriptor_, options_)) {
if (should_split()) {
p->Emit(R"cc(
decltype(Impl_::Split::$name$_){},
)cc");
@ -293,8 +295,9 @@ void CordFieldGenerator::GenerateAggregateInitializer(io::Printer* p) const {
// ===================================================================
CordOneofFieldGenerator::CordOneofFieldGenerator(
const FieldDescriptor* descriptor, const Options& options)
: CordFieldGenerator(descriptor, options) {}
const FieldDescriptor* descriptor, const Options& options,
MessageSCCAnalyzer* scc)
: CordFieldGenerator(descriptor, options, scc) {}
void CordOneofFieldGenerator::GeneratePrivateMembers(
io::Printer* printer) const {
@ -410,14 +413,14 @@ void CordOneofFieldGenerator::GenerateArenaDestructorCode(
std::unique_ptr<FieldGeneratorBase> MakeSingularCordGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<CordFieldGenerator>(desc, options);
return absl::make_unique<CordFieldGenerator>(desc, options, scc);
}
std::unique_ptr<FieldGeneratorBase> MakeOneofCordGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<CordOneofFieldGenerator>(desc, options);
return absl::make_unique<CordOneofFieldGenerator>(desc, options, scc);
}
} // namespace cpp

@ -75,11 +75,9 @@ std::vector<Sub> Vars(const FieldDescriptor* field, const Options& opts) {
class SingularEnum : public FieldGeneratorBase {
public:
SingularEnum(const FieldDescriptor* field, const Options& opts)
: FieldGeneratorBase(field, opts),
field_(field),
opts_(&opts),
is_oneof_(field->real_containing_oneof() != nullptr) {}
SingularEnum(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {}
~SingularEnum() override = default;
std::vector<Sub> MakeVars() const override { return Vars(field_, *opts_); }
@ -103,7 +101,7 @@ class SingularEnum : public FieldGeneratorBase {
}
void GenerateSwappingCode(io::Printer* p) const override {
if (is_oneof_) return;
if (is_oneof()) return;
p->Emit(R"cc(
swap($field_$, other->$field_$);
@ -111,7 +109,7 @@ class SingularEnum : public FieldGeneratorBase {
}
void GenerateConstructorCode(io::Printer* p) const override {
if (!is_oneof_) return;
if (!is_oneof()) return;
p->Emit(R"cc(
$ns$::_$Msg$_default_instance_.$field_$ = $kDefault$;
)cc");
@ -145,7 +143,7 @@ class SingularEnum : public FieldGeneratorBase {
}
void GenerateAggregateInitializer(io::Printer* p) const override {
if (ShouldSplit(descriptor_, options_)) {
if (should_split()) {
p->Emit(R"cc(
decltype(Impl_::Split::$name$_){$kDefault$},
)cc");
@ -168,7 +166,6 @@ class SingularEnum : public FieldGeneratorBase {
private:
const FieldDescriptor* field_;
const Options* opts_;
bool is_oneof_;
};
void SingularEnum::GenerateAccessorDeclarations(io::Printer* p) const {
@ -202,7 +199,7 @@ void SingularEnum::GenerateInlineAccessorDefinitions(io::Printer* p) const {
}
)cc");
if (is_oneof_) {
if (is_oneof()) {
p->Emit(R"cc(
inline $Enum$ $Msg$::_internal_$name$() const {
if ($has_field$) {
@ -237,20 +234,28 @@ void SingularEnum::GenerateInlineAccessorDefinitions(io::Printer* p) const {
class RepeatedEnum : public FieldGeneratorBase {
public:
RepeatedEnum(const FieldDescriptor* field, const Options& opts)
: FieldGeneratorBase(field, opts),
RepeatedEnum(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts, scc),
field_(field),
opts_(&opts),
has_cached_size_(field_->is_packed() &&
HasGeneratedMethods(field_->file(), opts)) {}
HasGeneratedMethods(field_->file(), opts) &&
!should_split()) {}
~RepeatedEnum() override = default;
std::vector<Sub> MakeVars() const override { return Vars(field_, *opts_); }
void GeneratePrivateMembers(io::Printer* p) const override {
p->Emit(R"cc(
$pb$::RepeatedField<int> $name$_;
)cc");
if (should_split()) {
p->Emit(R"cc(
$pbi$::RawPtr<$pb$::RepeatedField<int>> $name$_;
)cc");
} else {
p->Emit(R"cc(
$pb$::RepeatedField<int> $name$_;
)cc");
}
if (has_cached_size_) {
p->Emit(R"cc(
@ -266,22 +271,41 @@ class RepeatedEnum : public FieldGeneratorBase {
}
void GenerateMergingCode(io::Printer* p) const override {
p->Emit(R"cc(
_this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
)cc");
// TODO(b/239716377): experiment with simplifying this to be
// `if (!from.empty()) { body(); }` for both split and non-split cases.
auto body = [&] {
p->Emit(R"cc(
_this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
)cc");
};
if (!should_split()) {
body();
} else {
p->Emit({{"body", body}}, R"cc(
if (!from.$field_$.IsDefault()) {
$body$;
}
)cc");
}
}
void GenerateSwappingCode(io::Printer* p) const override {
ABSL_CHECK(!ShouldSplit(descriptor_, options_));
ABSL_CHECK(!should_split());
p->Emit(R"cc(
$field_$.InternalSwap(&other->$field_$);
)cc");
}
void GenerateDestructorCode(io::Printer* p) const override {
p->Emit(R"cc(
_internal_mutable_$name$()->~RepeatedField();
)cc");
if (should_split()) {
p->Emit(R"cc(
$field_$.DeleteIfNotDefault();
)cc");
} else {
p->Emit(R"cc(
_internal_mutable_$name$()->~RepeatedField();
)cc");
}
}
void GenerateConstexprAggregateInitializer(io::Printer* p) const override {
@ -321,7 +345,13 @@ class RepeatedEnum : public FieldGeneratorBase {
}
void GenerateCopyConstructorCode(io::Printer* p) const override {
ABSL_CHECK(!ShouldSplit(field_, *opts_));
if (should_split()) {
p->Emit(R"cc(
if (!from._internal_$name$().empty()) {
_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
}
)cc");
}
}
void GenerateConstructorCode(io::Printer* p) const override {}
@ -392,29 +422,69 @@ void RepeatedEnum::GenerateInlineAccessorDefinitions(io::Printer* p) const {
$TsanDetectConcurrentMutation$;
return _internal_mutable_$name$();
}
inline const $pb$::RepeatedField<int>& $Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline $pb$::RepeatedField<int>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
if (should_split()) {
p->Emit(R"cc(
inline const $pb$::RepeatedField<int>& $Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return *$field_$;
}
inline $pb$::RepeatedField<int>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
$PrepareSplitMessageForWrite$;
if ($field_$.IsDefault()) {
$field_$.Set($pb$::Arena::CreateMessage<$pb$::RepeatedField<int>>(
GetArenaForAllocation()));
}
return $field_$.Get();
}
)cc");
} else {
p->Emit(R"cc(
inline const $pb$::RepeatedField<int>& $Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline $pb$::RepeatedField<int>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
}
}
void RepeatedEnum::GenerateSerializeWithCachedSizesToArray(
io::Printer* p) const {
if (field_->is_packed()) {
p->Emit(R"cc(
{
int byte_size = $cached_size_$.Get();
if (byte_size > 0) {
target = stream->WriteEnumPacked($number$, _internal_$name$(),
byte_size, target);
}
}
)cc");
p->Emit(
{
{"byte_size",
[&] {
if (has_cached_size_) {
p->Emit(
R"cc(std::size_t byte_size = $cached_size_$.Get();)cc");
} else {
p->Emit(R"cc(
std::size_t byte_size = 0;
auto count = static_cast<std::size_t>(this->_internal_$name$_size());
for (std::size_t i = 0; i < count; ++i) {
byte_size += ::_pbi::WireFormatLite::EnumSize(
this->_internal_$name$().Get(static_cast<int>(i)));
}
)cc");
}
}},
},
R"cc(
{
$byte_size$;
if (byte_size > 0) {
target = stream->WriteEnumPacked($number$, _internal_$name$(),
byte_size, target);
}
}
)cc");
return;
}
p->Emit(R"cc(
@ -445,8 +515,12 @@ void RepeatedEnum::GenerateByteSize(io::Printer* p) const {
total_size += ::_pbi::WireFormatLite::Int32Size(
static_cast<int32_t>(data_size));
}
$cached_size_$.Set(::_pbi::ToCachedSize(data_size));
)cc");
if (has_cached_size_) {
p->Emit(R"cc(
$cached_size_$.Set(::_pbi::ToCachedSize(data_size));
)cc");
}
}},
},
R"cc(
@ -468,13 +542,13 @@ void RepeatedEnum::GenerateByteSize(io::Printer* p) const {
std::unique_ptr<FieldGeneratorBase> MakeSinguarEnumGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<SingularEnum>(desc, options);
return absl::make_unique<SingularEnum>(desc, options, scc);
}
std::unique_ptr<FieldGeneratorBase> MakeRepeatedEnumGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<RepeatedEnum>(desc, options);
return absl::make_unique<RepeatedEnum>(desc, options, scc);
}
} // namespace cpp

@ -80,7 +80,7 @@ class Map : public FieldGeneratorBase {
public:
Map(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts),
: FieldGeneratorBase(field, opts, scc),
field_(field),
key_(field->message_type()->map_key()),
val_(field->message_type()->map_value()),
@ -141,7 +141,7 @@ class Map : public FieldGeneratorBase {
}
void GenerateAggregateInitializer(io::Printer* p) const override {
if (ShouldSplit(field_, *opts_)) {
if (should_split()) {
p->Emit(R"cc(
/* decltype($Msg$::Split::$name$_) */ {
$pbi$::ArenaInitialized(),
@ -158,7 +158,7 @@ class Map : public FieldGeneratorBase {
void GenerateConstructorCode(io::Printer* p) const override {}
void GenerateDestructorCode(io::Printer* p) const override {
if (ShouldSplit(field_, *opts_)) {
if (should_split()) {
p->Emit(R"cc(
$cached_split_ptr$->$name$_.~$MapField$();
)cc");

@ -99,19 +99,16 @@ class SingularMessage : public FieldGeneratorBase {
public:
SingularMessage(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts),
: FieldGeneratorBase(field, opts, scc),
field_(field),
opts_(&opts),
weak_(IsImplicitWeakField(field, opts, scc)),
has_required_(scc->HasRequiredFields(field->message_type())),
has_hasbit_(HasHasbit(field)),
is_oneof_(field_->real_containing_oneof() != nullptr),
is_foreign_(IsCrossFileMessage(field)) {}
has_hasbit_(HasHasbit(field)) {}
~SingularMessage() override = default;
std::vector<Sub> MakeVars() const override {
return Vars(field_, *opts_, weak_);
return Vars(field_, *opts_, is_weak());
}
void GeneratePrivateMembers(io::Printer* p) const override {
@ -145,11 +142,8 @@ class SingularMessage : public FieldGeneratorBase {
const FieldDescriptor* field_;
const Options* opts_;
bool weak_;
bool has_required_;
bool has_hasbit_;
bool is_oneof_;
bool is_foreign_;
};
void SingularMessage::GenerateAccessorDeclarations(io::Printer* p) const {
@ -310,7 +304,7 @@ void SingularMessage::GenerateInlineAccessorDefinitions(io::Printer* p) const {
void SingularMessage::GenerateInternalAccessorDeclarations(
io::Printer* p) const {
if (!weak_) {
if (!is_weak()) {
p->Emit(R"cc(
static const $Submsg$& $name$(const $Msg$* msg);
)cc");
@ -329,7 +323,7 @@ void SingularMessage::GenerateInternalAccessorDefinitions(
// practice, the linker is then not able to throw them out making implicit
// weak dependencies not work at all.
if (!weak_) {
if (!is_weak()) {
// This inline accessor directly returns member field and is used in
// Serialize such that AFDO profile correctly captures access information to
// message fields under serialize.
@ -355,7 +349,7 @@ void SingularMessage::GenerateInternalAccessorDefinitions(
}},
{"is_already_set",
[&] {
if (!is_oneof_) {
if (!is_oneof()) {
p->Emit("msg->$field_$ == nullptr");
} else {
p->Emit("msg->$not_has_field$");
@ -363,7 +357,7 @@ void SingularMessage::GenerateInternalAccessorDefinitions(
}},
{"clear_oneof",
[&] {
if (!is_oneof_) return;
if (!is_oneof()) return;
p->Emit(R"cc(
msg->clear_$oneof_name$();
msg->set_has_$name$();
@ -420,7 +414,7 @@ void SingularMessage::GenerateMessageClearingCode(io::Printer* p) const {
}
void SingularMessage::GenerateMergingCode(io::Printer* p) const {
if (weak_) {
if (is_weak()) {
p->Emit(
"_Internal::mutable_$name$(_this)->CheckTypeAndMergeFrom(\n"
" _Internal::$name$(&from));\n");
@ -444,7 +438,7 @@ void SingularMessage::GenerateDestructorCode(io::Printer* p) const {
// care when handling them.
p->Emit("if (this != internal_default_instance()) ");
}
if (ShouldSplit(field_, *opts_)) {
if (should_split()) {
p->Emit("delete $cached_split_ptr$->$name$_;\n");
return;
}
@ -469,7 +463,7 @@ void SingularMessage::GenerateCopyConstructorCode(io::Printer* p) const {
void SingularMessage::GenerateSerializeWithCachedSizesToArray(
io::Printer* p) const {
if (field_->type() == FieldDescriptor::TYPE_MESSAGE) {
if (!is_group()) {
p->Emit(
"target = $pbi$::WireFormatLite::\n"
" InternalWrite$declared_type$($number$, _Internal::$name$(this),\n"
@ -520,7 +514,7 @@ void SingularMessage::GenerateCopyAggregateInitializer(io::Printer* p) const {
}
void SingularMessage::GenerateAggregateInitializer(io::Printer* p) const {
if (ShouldSplit(field_, *opts_)) {
if (should_split()) {
p->Emit(R"cc(
decltype(Impl_::Split::$name$_){nullptr},
)cc");
@ -638,7 +632,7 @@ void OneofMessage::GenerateInlineAccessorDefinitions(io::Printer* p) const {
" clear_$oneof_name$();\n"
" if ($name$) {\n"
" set_has_$name$();\n");
if (weak_) {
if (is_weak()) {
p->Emit(
" $field_$ = "
"reinterpret_cast<$pb$::MessageLite*>($name$);\n");
@ -656,7 +650,7 @@ void OneofMessage::GenerateInlineAccessorDefinitions(io::Printer* p) const {
" if ($not_has_field$) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n");
if (weak_) {
if (is_weak()) {
p->Emit(
" $field_$ = "
"reinterpret_cast<$pb$::MessageLite*>(CreateMaybeMessage< "
@ -716,16 +710,15 @@ class RepeatedMessage : public FieldGeneratorBase {
public:
RepeatedMessage(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts),
: FieldGeneratorBase(field, opts, scc),
field_(field),
opts_(&opts),
weak_(IsImplicitWeakField(field, opts, scc)),
has_required_(scc->HasRequiredFields(field->message_type())) {}
~RepeatedMessage() override = default;
std::vector<Sub> MakeVars() const override {
return Vars(field_, *opts_, weak_);
return Vars(field_, *opts_, is_weak());
}
void GeneratePrivateMembers(io::Printer* p) const override;
@ -735,7 +728,7 @@ class RepeatedMessage : public FieldGeneratorBase {
void GenerateMergingCode(io::Printer* p) const override;
void GenerateSwappingCode(io::Printer* p) const override;
void GenerateConstructorCode(io::Printer* p) const override;
void GenerateCopyConstructorCode(io::Printer* p) const override {}
void GenerateCopyConstructorCode(io::Printer* p) const override;
void GenerateDestructorCode(io::Printer* p) const override;
void GenerateSerializeWithCachedSizesToArray(io::Printer* p) const override;
void GenerateByteSize(io::Printer* p) const override;
@ -744,12 +737,17 @@ class RepeatedMessage : public FieldGeneratorBase {
private:
const FieldDescriptor* field_;
const Options* opts_;
bool weak_;
bool has_required_;
};
void RepeatedMessage::GeneratePrivateMembers(io::Printer* p) const {
p->Emit("$pb$::$Weak$RepeatedPtrField< $Submsg$ > $name$_;\n");
if (should_split()) {
p->Emit(R"cc(
$pbi$::RawPtr<$pb$::$Weak$RepeatedPtrField<$Submsg$>> $name$_;
)cc");
} else {
p->Emit("$pb$::$Weak$RepeatedPtrField< $Submsg$ > $name$_;\n");
}
}
void RepeatedMessage::GenerateAccessorDeclarations(io::Printer* p) const {
@ -764,7 +762,7 @@ void RepeatedMessage::GenerateAccessorDeclarations(io::Printer* p) const {
"private:\n"
"const $pb$::RepeatedPtrField<$Submsg$>& _internal_$name$() const;\n"
"$pb$::RepeatedPtrField<$Submsg$>* _internal_mutable_$name$();\n");
if (weak_) {
if (is_weak()) {
format(
"const $pb$::WeakRepeatedPtrField<$Submsg$>& _internal_weak_$name$() "
"const;\n"
@ -831,19 +829,40 @@ void RepeatedMessage::GenerateInlineAccessorDefinitions(io::Printer* p) const {
" return _internal_$name$();\n"
"}\n");
p->Emit(R"cc(
inline const $pb$::$Weak$RepeatedPtrField<$Submsg$>&
$Msg$::_internal$_weak$_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline $pb$::$Weak$RepeatedPtrField<$Submsg$>*
$Msg$::_internal_mutable$_weak$_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
if (weak_) {
if (should_split()) {
p->Emit(R"cc(
inline const $pb$::$Weak$RepeatedPtrField<$Submsg$>&
$Msg$::_internal$_weak$_$name$() const {
$TsanDetectConcurrentRead$;
return *$field_$;
}
inline $pb$::$Weak$RepeatedPtrField<$Submsg$>*
$Msg$::_internal_mutable$_weak$_$name$() {
$TsanDetectConcurrentRead$;
$PrepareSplitMessageForWrite$;
if ($field_$.IsDefault()) {
$field_$.Set(
CreateMaybeMessage<$pb$::$Weak$RepeatedPtrField<$Submsg$>>(
GetArenaForAllocation()));
}
return $field_$.Get();
}
)cc");
} else {
p->Emit(R"cc(
inline const $pb$::$Weak$RepeatedPtrField<$Submsg$>&
$Msg$::_internal$_weak$_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline $pb$::$Weak$RepeatedPtrField<$Submsg$>*
$Msg$::_internal_mutable$_weak$_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
}
if (is_weak()) {
p->Emit(R"cc(
inline const $pb$::RepeatedPtrField<$Submsg$>& $Msg$::_internal_$name$()
const {
@ -861,13 +880,27 @@ void RepeatedMessage::GenerateClearingCode(io::Printer* p) const {
}
void RepeatedMessage::GenerateMergingCode(io::Printer* p) const {
p->Emit(
"_this->_internal_mutable$_weak$_$name$()->MergeFrom(from._internal"
"$_weak$_$name$());\n");
// TODO(b/239716377): experiment with simplifying this to be
// `if (!from.empty()) { body(); }` for both split and non-split cases.
auto body = [&] {
p->Emit(R"cc(
_this->_internal_mutable$_weak$_$name$()->MergeFrom(
from._internal$_weak$_$name$());
)cc");
};
if (!should_split()) {
body();
} else {
p->Emit({{"body", body}}, R"cc(
if (!from.$field_$.IsDefault()) {
$body$;
}
)cc");
}
}
void RepeatedMessage::GenerateSwappingCode(io::Printer* p) const {
ABSL_CHECK(!ShouldSplit(descriptor_, options_));
ABSL_CHECK(!should_split());
p->Emit(R"cc(
$field_$.InternalSwap(&other->$field_$);
)cc");
@ -877,13 +910,31 @@ void RepeatedMessage::GenerateConstructorCode(io::Printer* p) const {
// Not needed for repeated fields.
}
void RepeatedMessage::GenerateCopyConstructorCode(io::Printer* p) const {
// TODO(b/291633281): For split repeated fields we might want to use type
// erasure to reduce binary size costs.
if (should_split()) {
p->Emit(R"cc(
if (!from._internal$_weak$_$name$().empty()) {
_internal_mutable$_weak$_$name$()->MergeFrom(from._internal$_weak$_$name$());
}
)cc");
}
}
void RepeatedMessage::GenerateDestructorCode(io::Printer* p) const {
p->Emit("$field_$.~$Weak$RepeatedPtrField();\n");
if (should_split()) {
p->Emit(R"cc(
$field_$.DeleteIfNotDefault();
)cc");
} else {
p->Emit("$field_$.~$Weak$RepeatedPtrField();\n");
}
}
void RepeatedMessage::GenerateSerializeWithCachedSizesToArray(
io::Printer* p) const {
if (weak_) {
if (is_weak()) {
p->Emit(
"for (auto it = this->$field_$.pointer_begin(),\n"
" end = this->$field_$.pointer_end(); it < end; ++it) {\n");
@ -935,7 +986,7 @@ void RepeatedMessage::GenerateByteSize(io::Printer* p) const {
void RepeatedMessage::GenerateIsInitialized(io::Printer* p) const {
if (!has_required_) return;
if (weak_) {
if (is_weak()) {
p->Emit(
"if (!$pbi$::AllAreInitializedWeak($field_$.weak))\n"
" return false;\n");

@ -110,11 +110,9 @@ std::vector<Sub> Vars(const FieldDescriptor* field, const Options& options) {
class SingularPrimitive final : public FieldGeneratorBase {
public:
SingularPrimitive(const FieldDescriptor* field, const Options& opts)
: FieldGeneratorBase(field, opts),
field_(field),
opts_(&opts),
is_oneof_(field_->real_containing_oneof() != nullptr) {}
SingularPrimitive(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {}
~SingularPrimitive() override = default;
std::vector<Sub> MakeVars() const override { return Vars(field_, *opts_); }
@ -138,7 +136,7 @@ class SingularPrimitive final : public FieldGeneratorBase {
}
void GenerateSwappingCode(io::Printer* p) const override {
if (is_oneof_) {
if (is_oneof()) {
// Don't print any swapping code. Swapping the union will swap this field.
return;
}
@ -150,7 +148,7 @@ class SingularPrimitive final : public FieldGeneratorBase {
}
void GenerateConstructorCode(io::Printer* p) const override {
if (!is_oneof_) {
if (!is_oneof()) {
return;
}
@ -191,7 +189,6 @@ class SingularPrimitive final : public FieldGeneratorBase {
private:
const FieldDescriptor* field_;
const Options* opts_;
bool is_oneof_;
};
void SingularPrimitive::GenerateAccessorDeclarations(io::Printer* p) const {
@ -226,7 +223,7 @@ void SingularPrimitive::GenerateInlineAccessorDefinitions(
}
)cc");
if (is_oneof_) {
if (is_oneof()) {
p->Emit(R"cc(
inline $Type$ $Msg$::_internal_$name$() const {
if ($has_field$) {
@ -309,8 +306,9 @@ void SingularPrimitive::GenerateByteSize(io::Printer* p) const {
class RepeatedPrimitive final : public FieldGeneratorBase {
public:
RepeatedPrimitive(const FieldDescriptor* field, const Options& opts)
: FieldGeneratorBase(field, opts), field_(field), opts_(&opts) {}
RepeatedPrimitive(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {}
~RepeatedPrimitive() override = default;
std::vector<Sub> MakeVars() const override { return Vars(field_, *opts_); }
@ -322,27 +320,54 @@ class RepeatedPrimitive final : public FieldGeneratorBase {
}
void GenerateMergingCode(io::Printer* p) const override {
p->Emit(R"cc(
_this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
)cc");
// TODO(b/239716377): experiment with simplifying this to be
// `if (!from.empty()) { body(); }` for both split and non-split cases.
auto body = [&] {
p->Emit(R"cc(
_this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
)cc");
};
if (!should_split()) {
body();
} else {
p->Emit({{"body", body}}, R"cc(
if (!from.$field_$.IsDefault()) {
$body$;
}
)cc");
}
}
void GenerateSwappingCode(io::Printer* p) const override {
ABSL_CHECK(!ShouldSplit(descriptor_, options_));
ABSL_CHECK(!should_split());
p->Emit(R"cc(
$field_$.InternalSwap(&other->$field_$);
)cc");
}
void GenerateDestructorCode(io::Printer* p) const override {
p->Emit(R"cc(
$field_$.~RepeatedField();
)cc");
if (should_split()) {
p->Emit(R"cc(
$field_$.DeleteIfNotDefault();
)cc");
} else {
p->Emit(R"cc(
$field_$.~RepeatedField();
)cc");
}
}
void GenerateConstructorCode(io::Printer* p) const override {}
void GenerateCopyConstructorCode(io::Printer* p) const override {}
void GenerateCopyConstructorCode(io::Printer* p) const override {
if (should_split()) {
p->Emit(R"cc(
if (!from._internal_$name$().empty()) {
_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
}
)cc");
}
}
void GenerateConstexprAggregateInitializer(io::Printer* p) const override {
p->Emit(R"cc(
@ -352,7 +377,7 @@ class RepeatedPrimitive final : public FieldGeneratorBase {
}
void GenerateAggregateInitializer(io::Printer* p) const override {
ABSL_CHECK(!ShouldSplit(descriptor_, options_));
ABSL_CHECK(!should_split());
p->Emit(R"cc(
decltype($field_$){arena},
)cc");
@ -360,7 +385,7 @@ class RepeatedPrimitive final : public FieldGeneratorBase {
}
void GenerateCopyAggregateInitializer(io::Printer* p) const override {
ABSL_CHECK(!ShouldSplit(descriptor_, options_));
ABSL_CHECK(!should_split());
p->Emit(R"cc(
decltype($field_$){from.$field_$},
)cc");
@ -377,7 +402,8 @@ class RepeatedPrimitive final : public FieldGeneratorBase {
bool HasCachedSize() const {
bool is_packed_varint =
field_->is_packed() && !FixedSize(field_->type()).has_value();
return is_packed_varint && HasGeneratedMethods(field_->file(), *opts_);
return is_packed_varint && HasGeneratedMethods(field_->file(), *opts_) &&
!should_split();
}
void GenerateCacheSizeInitializer(io::Printer* p) const {
@ -394,9 +420,15 @@ class RepeatedPrimitive final : public FieldGeneratorBase {
};
void RepeatedPrimitive::GeneratePrivateMembers(io::Printer* p) const {
p->Emit(R"cc(
$pb$::RepeatedField<$Type$> $name$_;
)cc");
if (should_split()) {
p->Emit(R"cc(
$pbi$::RawPtr<$pb$::RepeatedField<$Type$>> $name$_;
)cc");
} else {
p->Emit(R"cc(
$pb$::RepeatedField<$Type$> $name$_;
)cc");
}
if (HasCachedSize()) {
p->Emit({{"_cached_size_", MakeVarintCachedSizeName(field_)}},
@ -459,15 +491,37 @@ void RepeatedPrimitive::GenerateInlineAccessorDefinitions(
return _internal_mutable_$name$();
}
inline const $pb$::RepeatedField<$Type$>& $Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline $pb$::RepeatedField<$Type$>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
if (should_split()) {
p->Emit(R"cc(
inline const $pb$::RepeatedField<$Type$>& $Msg$::_internal_$name$()
const {
$TsanDetectConcurrentRead$;
return *$field_$;
}
inline $pb$::RepeatedField<$Type$>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
$PrepareSplitMessageForWrite$;
if ($field_$.IsDefault()) {
$field_$.Set($pb$::Arena::CreateMessage<$pb$::RepeatedField<$Type$>>(
GetArenaForAllocation()));
}
return $field_$.Get();
}
)cc");
} else {
p->Emit(R"cc(
inline const $pb$::RepeatedField<$Type$>& $Msg$::_internal_$name$()
const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline $pb$::RepeatedField<$Type$>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
}
}
void RepeatedPrimitive::GenerateSerializeWithCachedSizesToArray(
@ -492,15 +546,29 @@ void RepeatedPrimitive::GenerateSerializeWithCachedSizesToArray(
return;
}
p->Emit(R"cc(
{
int byte_size = $_field_cached_byte_size_$.Get();
if (byte_size > 0) {
target = stream->Write$DeclaredType$Packed($number$, _internal_$name$(),
byte_size, target);
}
}
)cc");
p->Emit(
{
{"byte_size",
[&] {
if (HasCachedSize()) {
p->Emit(R"cc($_field_cached_byte_size_$.Get();)cc");
} else {
p->Emit(R"cc(
::_pbi::WireFormatLite::$DeclaredType$Size(
this->_internal_$name$());
)cc");
}
}},
},
R"cc(
{
int byte_size = $byte_size$;
if (byte_size > 0) {
target = stream->Write$DeclaredType$Packed(
$number$, _internal_$name$(), byte_size, target);
}
}
)cc");
}
void RepeatedPrimitive::GenerateByteSize(io::Printer* p) const {
@ -562,13 +630,13 @@ void RepeatedPrimitive::GenerateByteSize(io::Printer* p) const {
std::unique_ptr<FieldGeneratorBase> MakeSinguarPrimitiveGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<SingularPrimitive>(desc, options);
return absl::make_unique<SingularPrimitive>(desc, options, scc);
}
std::unique_ptr<FieldGeneratorBase> MakeRepeatedPrimitiveGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<RepeatedPrimitive>(desc, options);
return absl::make_unique<RepeatedPrimitive>(desc, options, scc);
}
} // namespace cpp

@ -65,7 +65,7 @@ std::vector<Sub> Vars(const FieldDescriptor* field, const Options& opts) {
"::", MakeDefaultFieldName(field));
bool empty_default = field->default_value_string().empty();
bool is_bytes = field->type() == FieldDescriptor::TYPE_BYTES;
bool bytes = field->type() == FieldDescriptor::TYPE_BYTES;
return {
{"kDefault", DefaultValue(opts, field)},
@ -82,35 +82,33 @@ std::vector<Sub> Vars(const FieldDescriptor* field, const Options& opts) {
Sub{"lazy_args", !empty_default ? absl::StrCat(lazy_var, ",") : ""}
.WithSuffix(","),
{"byte", is_bytes ? "void" : "char"},
{"Set", is_bytes ? "SetBytes" : "Set"},
{"byte", bytes ? "void" : "char"},
{"Set", bytes ? "SetBytes" : "Set"},
};
}
class SingularString : public FieldGeneratorBase {
public:
SingularString(const FieldDescriptor* field, const Options& opts)
: FieldGeneratorBase(field, opts),
field_(field),
opts_(&opts),
is_oneof_(field->real_containing_oneof() != nullptr),
inlined_(IsStringInlined(field, opts)) {}
SingularString(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {}
~SingularString() override = default;
std::vector<Sub> MakeVars() const override { return Vars(field_, *opts_); }
bool IsInlined() const override { return inlined_; }
bool IsInlined() const override { return is_inlined(); }
ArenaDtorNeeds NeedsArenaDestructor() const override {
return inlined_ ? ArenaDtorNeeds::kOnDemand : ArenaDtorNeeds::kNone;
return is_inlined() ? ArenaDtorNeeds::kOnDemand : ArenaDtorNeeds::kNone;
}
void GeneratePrivateMembers(io::Printer* p) const override {
// Skips the automatic destruction if inlined; rather calls it explicitly if
// allocating arena is null.
p->Emit({{"Str", inlined_ ? "InlinedStringField" : "ArenaStringPtr"}}, R"cc(
$pbi$::$Str$ $name$_;
)cc");
p->Emit({{"Str", is_inlined() ? "InlinedStringField" : "ArenaStringPtr"}},
R"cc(
$pbi$::$Str$ $name$_;
)cc");
}
void GenerateMergingCode(io::Printer* p) const override {
@ -120,7 +118,7 @@ class SingularString : public FieldGeneratorBase {
}
void GenerateArenaDestructorCode(io::Printer* p) const override {
if (!inlined_) return;
if (!is_inlined()) return;
p->Emit(R"cc(
if (!_this->_internal_$name$_donated()) {
@ -172,8 +170,6 @@ class SingularString : public FieldGeneratorBase {
const FieldDescriptor* field_;
const Options* opts_;
bool is_oneof_;
bool inlined_;
};
void SingularString::GenerateStaticMembers(io::Printer* p) const {
@ -182,7 +178,7 @@ void SingularString::GenerateStaticMembers(io::Printer* p) const {
static const $pbi$::LazyString $default_variable_name$;
)cc");
}
if (inlined_) {
if (is_inlined()) {
// `_init_inline_xxx` is used for initializing default instances.
p->Emit(R"cc(
static std::true_type _init_inline_$name$_;
@ -231,7 +227,7 @@ void SingularString::GenerateAccessorDeclarations(io::Printer* p) const {
p->Emit(
{{"donated",
[&] {
if (!inlined_) return;
if (!is_inlined()) return;
p->Emit(R"cc(
inline PROTOBUF_ALWAYS_INLINE bool _internal_$name$_donated() const;
)cc");
@ -288,7 +284,7 @@ void ArgsForSetter(io::Printer* p, bool inlined) {
}
void SingularString::ReleaseImpl(io::Printer* p) const {
if (is_oneof_) {
if (is_oneof()) {
p->Emit(R"cc(
if ($not_has_field$) {
return nullptr;
@ -306,7 +302,7 @@ void SingularString::ReleaseImpl(io::Printer* p) const {
return;
}
if (inlined_) {
if (is_inlined()) {
p->Emit(R"cc(
if (($has_hasbit$) == 0) {
return nullptr;
@ -342,7 +338,7 @@ void SingularString::ReleaseImpl(io::Printer* p) const {
}
void SingularString::SetAllocatedImpl(io::Printer* p) const {
if (is_oneof_) {
if (is_oneof()) {
p->Emit(R"cc(
if (has_$oneof_name$()) {
clear_$oneof_name$();
@ -365,7 +361,7 @@ void SingularString::SetAllocatedImpl(io::Printer* p) const {
)cc");
}
if (inlined_) {
if (is_inlined()) {
// Currently, string fields with default value can't be inlined.
p->Emit(R"cc(
$field_$.SetAllocated(nullptr, value, $set_args$);
@ -393,18 +389,18 @@ void SingularString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
{
{"if_IsDefault",
[&] {
if (EmptyDefault() || is_oneof_) return;
if (EmptyDefault() || is_oneof()) return;
p->Emit(R"cc(
if ($field_$.IsDefault()) {
return $default_variable_field$.get();
}
)cc");
}},
{"update_hasbit", [&] { UpdateHasbitSet(p, is_oneof_); }},
{"set_args", [&] { ArgsForSetter(p, inlined_); }},
{"update_hasbit", [&] { UpdateHasbitSet(p, is_oneof()); }},
{"set_args", [&] { ArgsForSetter(p, is_inlined()); }},
{"check_hasbit",
[&] {
if (!is_oneof_) return;
if (!is_oneof()) return;
p->Emit(R"cc(
if ($not_has_field$) {
return $kDefaultStr$;
@ -473,7 +469,7 @@ void SingularString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
}
)cc");
if (inlined_) {
if (is_inlined()) {
p->Emit(R"cc(
inline bool $Msg$::_internal_$name$_donated() const {
return $inlined_string_donated$;
@ -483,7 +479,7 @@ void SingularString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
}
void SingularString::GenerateClearingCode(io::Printer* p) const {
if (is_oneof_) {
if (is_oneof()) {
p->Emit(R"cc(
$field_$.Destroy();
)cc");
@ -497,14 +493,14 @@ void SingularString::GenerateClearingCode(io::Printer* p) const {
return;
}
ABSL_DCHECK(!inlined_);
ABSL_DCHECK(!is_inlined());
p->Emit(R"cc(
$field_$.ClearToDefault($lazy_var$, GetArenaForAllocation());
)cc");
}
void SingularString::GenerateMessageClearingCode(io::Printer* p) const {
if (is_oneof_) {
if (is_oneof()) {
p->Emit(R"cc(
$field_$.Destroy();
)cc");
@ -520,7 +516,7 @@ void SingularString::GenerateMessageClearingCode(io::Printer* p) const {
// will have checked that this field is set. If so, we can avoid redundant
// checks against the default variable.
if (inlined_ && HasHasbit(field_)) {
if (is_inlined() && HasHasbit(field_)) {
// Calling mutable_$name$() gives us a string reference and sets the has bit
// for $name$ (in proto2). We may get here when the string field is inlined
// but the string's contents have not been changed by the user, so we cannot
@ -551,12 +547,12 @@ void SingularString::GenerateMessageClearingCode(io::Printer* p) const {
}
void SingularString::GenerateSwappingCode(io::Printer* p) const {
if (is_oneof_) {
if (is_oneof()) {
// Don't print any swapping code. Swapping the union will swap this field.
return;
}
if (!inlined_) {
if (!is_inlined()) {
p->Emit(R"cc(
::_pbi::ArenaStringPtr::InternalSwap(&$field_$, lhs_arena,
&other->$field_$, rhs_arena);
@ -577,8 +573,8 @@ void SingularString::GenerateSwappingCode(io::Printer* p) const {
}
void SingularString::GenerateConstructorCode(io::Printer* p) const {
if ((inlined_ && EmptyDefault()) || is_oneof_) return;
ABSL_DCHECK(!inlined_);
if ((is_inlined() && EmptyDefault()) || is_oneof()) return;
ABSL_DCHECK(!is_inlined());
p->Emit(R"cc(
$field_$.InitDefault();
@ -596,7 +592,7 @@ void SingularString::GenerateConstructorCode(io::Printer* p) const {
void SingularString::GenerateCopyConstructorCode(io::Printer* p) const {
GenerateConstructorCode(p);
if (inlined_) {
if (is_inlined()) {
p->Emit(R"cc(
new (&_this->$field_$)::_pbi::InlinedStringField;
)cc");
@ -613,7 +609,7 @@ void SingularString::GenerateCopyConstructorCode(io::Printer* p) const {
}},
{"set_args",
[&] {
if (!inlined_) {
if (!is_inlined()) {
p->Emit("_this->GetArenaForAllocation()");
} else {
p->Emit(
@ -630,17 +626,17 @@ void SingularString::GenerateCopyConstructorCode(io::Printer* p) const {
}
void SingularString::GenerateDestructorCode(io::Printer* p) const {
if (inlined_) {
if (is_inlined()) {
// Explicitly calls ~InlinedStringField as its automatic call is disabled.
// Destructor has been implicitly skipped as a union.
ABSL_DCHECK(!ShouldSplit(field_, *opts_));
ABSL_DCHECK(!should_split());
p->Emit(R"cc(
$field_$.~InlinedStringField();
)cc");
return;
}
if (ShouldSplit(field_, *opts_)) {
if (should_split()) {
p->Emit(R"cc(
$cached_split_ptr$->$name$_.Destroy();
)cc");
@ -669,7 +665,7 @@ void SingularString::GenerateSerializeWithCachedSizesToArray(
void SingularString::GenerateConstexprAggregateInitializer(
io::Printer* p) const {
if (inlined_) {
if (is_inlined()) {
p->Emit(R"cc(
/*decltype($field_$)*/ {nullptr, false},
)cc");
@ -684,12 +680,12 @@ void SingularString::GenerateConstexprAggregateInitializer(
}
void SingularString::GenerateAggregateInitializer(io::Printer* p) const {
if (ShouldSplit(field_, options_)) {
ABSL_CHECK(!inlined_);
if (should_split()) {
ABSL_CHECK(!is_inlined());
p->Emit(R"cc(
decltype(Impl_::Split::$name$_){},
)cc");
} else if (!inlined_) {
} else if (!is_inlined()) {
p->Emit(R"cc(
decltype($field_$){},
)cc");
@ -702,16 +698,23 @@ void SingularString::GenerateAggregateInitializer(io::Printer* p) const {
class RepeatedString : public FieldGeneratorBase {
public:
RepeatedString(const FieldDescriptor* field, const Options& opts)
: FieldGeneratorBase(field, opts), field_(field), opts_(&opts) {}
RepeatedString(const FieldDescriptor* field, const Options& opts,
MessageSCCAnalyzer* scc)
: FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {}
~RepeatedString() override = default;
std::vector<Sub> MakeVars() const override { return Vars(field_, *opts_); }
void GeneratePrivateMembers(io::Printer* p) const override {
p->Emit(R"cc(
$pb$::RepeatedPtrField<std::string> $name$_;
)cc");
if (ShouldSplit(descriptor_, options_)) {
p->Emit(R"cc(
$pbi$::RawPtr<$pb$::RepeatedPtrField<std::string>> $name$_;
)cc");
} else {
p->Emit(R"cc(
$pb$::RepeatedPtrField<std::string> $name$_;
)cc");
}
}
void GenerateClearingCode(io::Printer* p) const override {
@ -721,9 +724,22 @@ class RepeatedString : public FieldGeneratorBase {
}
void GenerateMergingCode(io::Printer* p) const override {
p->Emit(R"cc(
_this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
)cc");
// TODO(b/239716377): experiment with simplifying this to be
// `if (!from.empty()) { body(); }` for both split and non-split cases.
auto body = [&] {
p->Emit(R"cc(
_this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
)cc");
};
if (!ShouldSplit(descriptor_, options_)) {
body();
} else {
p->Emit({{"body", body}}, R"cc(
if (!from.$field_$.IsDefault()) {
$body$;
}
)cc");
}
}
void GenerateSwappingCode(io::Printer* p) const override {
@ -734,15 +750,27 @@ class RepeatedString : public FieldGeneratorBase {
}
void GenerateDestructorCode(io::Printer* p) const override {
p->Emit(R"cc(
_internal_mutable_$name$()->~RepeatedPtrField();
)cc");
if (ShouldSplit(descriptor_, options_)) {
p->Emit(R"cc(
$field_$.DeleteIfNotDefault();
)cc");
} else {
p->Emit(R"cc(
_internal_mutable_$name$()->~RepeatedPtrField();
)cc");
}
}
void GenerateConstructorCode(io::Printer* p) const override {}
void GenerateCopyConstructorCode(io::Printer* p) const override {
ABSL_CHECK(!ShouldSplit(field_, options_));
if (ShouldSplit(descriptor_, options_)) {
p->Emit(R"cc(
if (!from._internal_$name$().empty()) {
_internal_mutable_$name$()->MergeFrom(from._internal_$name$());
}
)cc");
}
}
void GenerateByteSize(io::Printer* p) const override {
@ -904,17 +932,39 @@ void RepeatedString::GenerateInlineAccessorDefinitions(io::Printer* p) const {
$TsanDetectConcurrentMutation$;
return _internal_mutable_$name$();
}
inline const ::$proto_ns$::RepeatedPtrField<std::string>&
$Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline ::$proto_ns$::RepeatedPtrField<std::string>*
$Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
if (ShouldSplit(descriptor_, options_)) {
p->Emit(R"cc(
inline const $pb$::RepeatedPtrField<std::string>&
$Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return *$field_$;
}
inline $pb$::RepeatedPtrField<std::string>* $Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
$PrepareSplitMessageForWrite$;
if ($field_$.IsDefault()) {
$field_$.Set(
$pb$::Arena::CreateMessage<$pb$::RepeatedPtrField<std::string>>(
GetArenaForAllocation()));
}
return $field_$.Get();
}
)cc");
} else {
p->Emit(R"cc(
inline const ::$proto_ns$::RepeatedPtrField<std::string>&
$Msg$::_internal_$name$() const {
$TsanDetectConcurrentRead$;
return $field_$;
}
inline ::$proto_ns$::RepeatedPtrField<std::string>*
$Msg$::_internal_mutable_$name$() {
$TsanDetectConcurrentRead$;
return &$field_$;
}
)cc");
}
}
void RepeatedString::GenerateSerializeWithCachedSizesToArray(
@ -938,13 +988,13 @@ void RepeatedString::GenerateSerializeWithCachedSizesToArray(
std::unique_ptr<FieldGeneratorBase> MakeSinguarStringGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<SingularString>(desc, options);
return absl::make_unique<SingularString>(desc, options, scc);
}
std::unique_ptr<FieldGeneratorBase> MakeRepeatedStringGenerator(
const FieldDescriptor* desc, const Options& options,
MessageSCCAnalyzer* scc) {
return absl::make_unique<RepeatedString>(desc, options);
return absl::make_unique<RepeatedString>(desc, options, scc);
}
} // namespace cpp

@ -1242,11 +1242,18 @@ class FileGenerator::ForwardDeclarations {
}
void PrintTopLevelDecl(io::Printer* p, const Options& options) const {
for (const auto& c : classes_) {
p->Emit({{"class", QualifiedClassName(c.second, options)}}, R"cc(
template <>
$dllexport_decl $$class$* Arena::CreateMaybeMessage<$class$>(Arena*);
)cc");
if (ShouldGenerateExternSpecializations(options)) {
for (const auto& c : classes_) {
// To reduce total linker input size in large binaries we make these
// functions extern and define then in the pb.cc file. This avoids bloat
// in callers by having duplicate definitions of the template.
// However, it increases the size of the pb.cc translation units so it
// is a tradeoff.
p->Emit({{"class", QualifiedClassName(c.second, options)}}, R"cc(
template <>
$dllexport_decl $$class$* Arena::CreateMaybeMessage<$class$>(Arena*);
)cc");
}
}
}

@ -189,8 +189,6 @@ bool CppGenerator::Generate(const FileDescriptor* file,
.emplace(value.substr(pos, next_pos - pos));
pos = next_pos + 1;
} while (pos < value.size());
} else if (key == "unverified_lazy_message_sets") {
file_options.unverified_lazy_message_sets = true;
} else if (key == "force_eagerly_verified_lazy") {
file_options.force_eagerly_verified_lazy = true;
} else if (key == "experimental_tail_call_table_mode") {

@ -599,6 +599,40 @@ int EstimateAlignmentSize(const FieldDescriptor* field) {
return -1; // Make compiler happy.
}
int EstimateSize(const FieldDescriptor* field) {
if (field == nullptr) return 0;
if (field->is_repeated()) {
if (field->is_map()) {
return sizeof(google::protobuf::Map<int32_t, int32_t>);
}
return field->cpp_type() < FieldDescriptor::CPPTYPE_STRING || IsCord(field)
? sizeof(RepeatedField<int32_t>)
: sizeof(internal::RepeatedPtrFieldBase);
}
switch (field->cpp_type()) {
case FieldDescriptor::CPPTYPE_BOOL:
return 1;
case FieldDescriptor::CPPTYPE_INT32:
case FieldDescriptor::CPPTYPE_UINT32:
case FieldDescriptor::CPPTYPE_ENUM:
case FieldDescriptor::CPPTYPE_FLOAT:
return 4;
case FieldDescriptor::CPPTYPE_INT64:
case FieldDescriptor::CPPTYPE_UINT64:
case FieldDescriptor::CPPTYPE_DOUBLE:
case FieldDescriptor::CPPTYPE_MESSAGE:
return 8;
case FieldDescriptor::CPPTYPE_STRING:
if (IsCord(field)) return sizeof(absl::Cord);
return sizeof(internal::ArenaStringPtr);
}
ABSL_LOG(FATAL) << "Can't get here.";
return -1; // Make compiler happy.
}
std::string FieldConstantName(const FieldDescriptor* field) {
std::string field_name = UnderscoresToCamelCase(field->name(), true);
std::string result = absl::StrCat("k", field_name, "FieldNumber");

@ -229,6 +229,12 @@ std::string FieldMemberName(const FieldDescriptor* field, bool split);
// 64-bit pointers.
int EstimateAlignmentSize(const FieldDescriptor* field);
// Returns an estimate of the size of the field. This
// can't guarantee to be correct because the generated code could be compiled on
// different systems with different alignment rules. The estimates below assume
// 64-bit pointers.
int EstimateSize(const FieldDescriptor* field);
// Get the unqualified name that should be used for a field's field
// number constant.
std::string FieldConstantName(const FieldDescriptor* field);
@ -345,7 +351,7 @@ inline bool IsWeak(const FieldDescriptor* field, const Options& options) {
return false;
}
inline bool IsCord(const FieldDescriptor* field, const Options& options) {
inline bool IsCord(const FieldDescriptor* field) {
return field->cpp_type() == FieldDescriptor::CPPTYPE_STRING &&
internal::cpp::EffectiveStringCType(field) == FieldOptions::CORD;
}
@ -355,8 +361,7 @@ inline bool IsString(const FieldDescriptor* field, const Options& options) {
internal::cpp::EffectiveStringCType(field) == FieldOptions::STRING;
}
inline bool IsStringPiece(const FieldDescriptor* field,
const Options& options) {
inline bool IsStringPiece(const FieldDescriptor* field) {
return field->cpp_type() == FieldDescriptor::CPPTYPE_STRING &&
internal::cpp::EffectiveStringCType(field) ==
FieldOptions::STRING_PIECE;
@ -1011,6 +1016,14 @@ void GenerateUtf8CheckCodeForCord(io::Printer* p, const FieldDescriptor* field,
const Options& options, bool for_parse,
absl::string_view parameters);
inline bool ShouldGenerateExternSpecializations(const Options& options) {
// For OSS we omit the specializations to reduce codegen size.
// Some compilers can't handle that much input in a single translation unit.
// These specializations are just a link size optimization and do not affect
// correctness or performance, so it is ok to omit them.
return !options.opensource_runtime;
}
struct OneOfRangeImpl {
struct Iterator {
using iterator_category = std::forward_iterator_tag;

@ -2953,12 +2953,14 @@ void MessageGenerator::GenerateSourceInProto2Namespace(io::Printer* p) {
auto v = p->WithVars(ClassVars(descriptor_, options_));
auto t = p->WithVars(MakeTrackerCalls(descriptor_, options_));
Formatter format(p);
format(
"template<> "
"PROTOBUF_NOINLINE $classtype$*\n"
"Arena::CreateMaybeMessage< $classtype$ >(Arena* arena) {\n"
" return Arena::CreateMessageInternal< $classtype$ >(arena);\n"
"}\n");
if (ShouldGenerateExternSpecializations(options_)) {
format(
"template<> "
"PROTOBUF_NOINLINE $classtype$*\n"
"Arena::CreateMaybeMessage< $classtype$ >(Arena* arena) {\n"
" return Arena::CreateMessageInternal< $classtype$ >(arena);\n"
"}\n");
}
}
void MessageGenerator::GenerateClear(io::Printer* p) {

@ -77,7 +77,6 @@ struct Options {
bool bootstrap = false;
bool opensource_runtime = false;
bool annotate_accessor = false;
bool unverified_lazy_message_sets = false;
bool profile_driven_inline_string = true;
bool force_split = false;
bool profile_driven_split = true;

@ -119,10 +119,7 @@ class ParseFunctionGenerator::GeneratedOptionProvider final
IsStringInlined(field, gen_->options_),
IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_),
UseDirectTcParserTable(field, gen_->options_),
GetOptimizeFor(field->file(), gen_->options_) ==
FileOptions::LITE_RUNTIME,
ShouldSplit(field, gen_->options_),
/* uses_codegen */ true,
};
}
@ -145,8 +142,14 @@ ParseFunctionGenerator::ParseFunctionGenerator(
num_hasbits_(max_has_bit_index) {
if (should_generate_tctable()) {
tc_table_info_.reset(new TailCallTableInfo(
descriptor_, ordered_fields_, GeneratedOptionProvider(this),
has_bit_indices, inlined_string_indices));
descriptor_, ordered_fields_,
{
/* is_lite */ GetOptimizeFor(descriptor->file(), options_) ==
FileOptions::LITE_RUNTIME,
/* uses_codegen */ true,
},
GeneratedOptionProvider(this), has_bit_indices,
inlined_string_indices));
}
SetCommonMessageDataVariables(descriptor_, &variables_);
SetUnknownFieldsVariable(descriptor_, options_, &variables_);
@ -175,8 +178,7 @@ void ParseFunctionGenerator::GenerateMethodImpls(io::Printer* printer) {
"const char* $classname$::_InternalParse(const char* ptr,\n"
" ::_pbi::ParseContext* ctx) {\n"
"$annotate_deserialize$");
if (!options_.unverified_lazy_message_sets &&
ShouldVerify(descriptor_, options_, scc_analyzer_)) {
if (ShouldVerify(descriptor_, options_, scc_analyzer_)) {
format(
" ctx->set_lazy_eager_verify_func(&$classname$::InternalVerify);\n");
}
@ -634,20 +636,17 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) {
void ParseFunctionGenerator::GenerateFastFieldEntries(Formatter& format) {
for (const auto& info : tc_table_info_->fast_path_fields) {
if (info.field != nullptr) {
PrintFieldComment(format, info.field, options_);
}
if (info.func_name.empty()) {
format("{::_pbi::TcParser::MiniParse, {}},\n");
} else if (info.field == nullptr) {
if (auto* nonfield = info.AsNonField()) {
// Fast slot that is not associated with a field. Eg end group tags.
format("{$1$, {$2$, $3$}},\n", info.func_name, info.coded_tag,
info.nonfield_info);
} else {
ABSL_CHECK(!ShouldSplit(info.field, options_));
std::string func_name = info.func_name;
if (GetOptimizeFor(info.field->file(), options_) == FileOptions::SPEED) {
format("{$1$, {$2$, $3$}},\n", nonfield->func_name, nonfield->coded_tag,
nonfield->nonfield_info);
} else if (auto* as_field = info.AsField()) {
PrintFieldComment(format, as_field->field, options_);
ABSL_CHECK(!ShouldSplit(as_field->field, options_));
std::string func_name = as_field->func_name;
if (GetOptimizeFor(as_field->field->file(), options_) ==
FileOptions::SPEED) {
// For 1-byte tags we have a more optimized version of the varint parser
// that can hardcode the offset and has bit.
if (absl::EndsWith(func_name, "V8S1") ||
@ -659,12 +658,12 @@ void ParseFunctionGenerator::GenerateFastFieldEntries(Formatter& format) {
: "::uint64_t";
func_name = absl::StrCat(
"::_pbi::TcParser::SingularVarintNoZag1<", field_type,
", offsetof(", //
ClassName(info.field->containing_type()), //
", ", //
FieldMemberName(info.field, /*split=*/false), //
"), ", //
info.hasbit_idx, //
", offsetof(", //
ClassName(as_field->field->containing_type()), //
", ", //
FieldMemberName(as_field->field, /*split=*/false), //
"), ", //
as_field->hasbit_idx, //
">()");
}
}
@ -672,8 +671,11 @@ void ParseFunctionGenerator::GenerateFastFieldEntries(Formatter& format) {
format(
"{$1$,\n"
" {$2$, $3$, $4$, PROTOBUF_FIELD_OFFSET($classname$, $5$)}},\n",
func_name, info.coded_tag, info.hasbit_idx, info.aux_idx,
FieldMemberName(info.field, /*split=*/false));
func_name, as_field->coded_tag, as_field->hasbit_idx,
as_field->aux_idx, FieldMemberName(as_field->field, /*split=*/false));
} else {
ABSL_DCHECK(info.is_empty());
format("{::_pbi::TcParser::MiniParse, {}},\n");
}
}
}
@ -1041,12 +1043,15 @@ void ParseFunctionGenerator::GenerateLengthDelim(Formatter& format,
bool eager_verify =
IsEagerlyVerifiedLazy(field, options_, scc_analyzer_);
if (ShouldVerify(descriptor_, options_, scc_analyzer_)) {
format(
"ctx->set_lazy_eager_verify_func($1$);\n",
eager_verify
? absl::StrCat("&", ClassName(field->message_type(), true),
"::InternalVerify")
: "nullptr");
if (eager_verify) {
format("ctx->set_lazy_eager_verify_func(&$1$::InternalVerify);\n",
ClassName(field->message_type(), true));
} else {
format(
"ctx->set_lazy_eager_verify_func(nullptr);\n"
"auto old_mode = "
"ctx->set_lazy_parse_mode(::_pbi::ParseContext::kLazy);\n");
}
}
if (field->real_containing_oneof()) {
format(
@ -1072,14 +1077,15 @@ void ParseFunctionGenerator::GenerateLengthDelim(Formatter& format,
" ::$proto_ns$::internal::LazyField> parse_helper(\n"
" $1$::default_instance(),\n"
" $msg$GetArenaForAllocation(),\n"
" ::google::protobuf::internal::LazyVerifyOption::$2$,\n"
" lazy_field);\n"
"ptr = ctx->ParseMessage(&parse_helper, ptr);\n",
FieldMessageTypeName(field, options_),
eager_verify ? "kEager" : "kLazy");
if (ShouldVerify(descriptor_, options_, scc_analyzer_) &&
eager_verify) {
format("ctx->set_lazy_eager_verify_func(nullptr);\n");
FieldMessageTypeName(field, options_));
if (ShouldVerify(descriptor_, options_, scc_analyzer_)) {
if (eager_verify) {
format("ctx->set_lazy_eager_verify_func(nullptr);\n");
} else {
format("(void)ctx->set_lazy_parse_mode(old_mode);\n");
}
}
} else if (IsImplicitWeakField(field, options_, scc_analyzer_)) {
if (!field->is_repeated()) {

@ -909,8 +909,10 @@ void CodeGeneratorRequest::MergeImpl(::google::protobuf::Message& to_msg, const
(void) cached_has_bits;
_this->_internal_mutable_file_to_generate()->MergeFrom(from._internal_file_to_generate());
_this->_internal_mutable_proto_file()->MergeFrom(from._internal_proto_file());
_this->_internal_mutable_source_file_descriptors()->MergeFrom(from._internal_source_file_descriptors());
_this->_internal_mutable_proto_file()->MergeFrom(
from._internal_proto_file());
_this->_internal_mutable_source_file_descriptors()->MergeFrom(
from._internal_source_file_descriptors());
cached_has_bits = from._impl_._has_bits_[0];
if (cached_has_bits & 0x00000003u) {
if (cached_has_bits & 0x00000001u) {
@ -1536,7 +1538,8 @@ void CodeGeneratorResponse::MergeImpl(::google::protobuf::Message& to_msg, const
::uint32_t cached_has_bits = 0;
(void) cached_has_bits;
_this->_internal_mutable_file()->MergeFrom(from._internal_file());
_this->_internal_mutable_file()->MergeFrom(
from._internal_file());
cached_has_bits = from._impl_._has_bits_[0];
if (cached_has_bits & 0x00000003u) {
if (cached_has_bits & 0x00000001u) {
@ -1584,22 +1587,6 @@ void CodeGeneratorResponse::InternalSwap(CodeGeneratorResponse* other) {
} // namespace google
namespace google {
namespace protobuf {
template<> PROTOBUF_NOINLINE ::google::protobuf::compiler::Version*
Arena::CreateMaybeMessage< ::google::protobuf::compiler::Version >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::compiler::Version >(arena);
}
template<> PROTOBUF_NOINLINE ::google::protobuf::compiler::CodeGeneratorRequest*
Arena::CreateMaybeMessage< ::google::protobuf::compiler::CodeGeneratorRequest >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::compiler::CodeGeneratorRequest >(arena);
}
template<> PROTOBUF_NOINLINE ::google::protobuf::compiler::CodeGeneratorResponse_File*
Arena::CreateMaybeMessage< ::google::protobuf::compiler::CodeGeneratorResponse_File >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::compiler::CodeGeneratorResponse_File >(arena);
}
template<> PROTOBUF_NOINLINE ::google::protobuf::compiler::CodeGeneratorResponse*
Arena::CreateMaybeMessage< ::google::protobuf::compiler::CodeGeneratorResponse >(Arena* arena) {
return Arena::CreateMessageInternal< ::google::protobuf::compiler::CodeGeneratorResponse >(arena);
}
} // namespace protobuf
} // namespace google
// @@protoc_insertion_point(global_scope)

@ -77,14 +77,6 @@ class Version;
struct VersionDefaultTypeInternal;
PROTOC_EXPORT extern VersionDefaultTypeInternal _Version_default_instance_;
} // namespace compiler
template <>
PROTOC_EXPORT ::google::protobuf::compiler::CodeGeneratorRequest* Arena::CreateMaybeMessage<::google::protobuf::compiler::CodeGeneratorRequest>(Arena*);
template <>
PROTOC_EXPORT ::google::protobuf::compiler::CodeGeneratorResponse* Arena::CreateMaybeMessage<::google::protobuf::compiler::CodeGeneratorResponse>(Arena*);
template <>
PROTOC_EXPORT ::google::protobuf::compiler::CodeGeneratorResponse_File* Arena::CreateMaybeMessage<::google::protobuf::compiler::CodeGeneratorResponse_File>(Arena*);
template <>
PROTOC_EXPORT ::google::protobuf::compiler::Version* Arena::CreateMaybeMessage<::google::protobuf::compiler::Version>(Arena*);
} // namespace protobuf
} // namespace google

@ -382,6 +382,7 @@ void Generator::PrintTopBoilerplate() const {
// Prints Python imports for all modules imported by |file|.
void Generator::PrintImports() const {
bool has_importlib = false;
for (int i = 0; i < file_->dependency_count(); ++i) {
absl::string_view filename = file_->dependency(i)->name();
@ -396,7 +397,10 @@ void Generator::PrintImports() const {
// module name and import it using importlib. Otherwise the usual kind of
// import statement would result in a syntax error from the presence of
// the keyword.
printer_->Print("import importlib\n");
if (has_importlib == false) {
printer_->Print("import importlib\n");
has_importlib = true;
}
printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias",
module_alias, "name", module_name);
} else {

@ -151,41 +151,50 @@ void CheckImportModules(const Descriptor* descriptor,
}
void PyiGenerator::PrintImportForDescriptor(
const FileDescriptor& desc,
absl::flat_hash_set<std::string>* seen_aliases) const {
const FileDescriptor& desc, absl::flat_hash_set<std::string>* seen_aliases,
bool* has_importlib) const {
const std::string& filename = desc.name();
std::string module_name_owned = StrippedModuleName(filename);
absl::string_view module_name(module_name_owned);
size_t last_dot_pos = module_name.rfind('.');
std::string import_statement;
if (last_dot_pos == std::string::npos) {
import_statement = absl::StrCat("import ", module_name);
} else {
import_statement =
absl::StrCat("from ", module_name.substr(0, last_dot_pos), " import ",
module_name.substr(last_dot_pos + 1));
module_name = module_name.substr(last_dot_pos + 1);
}
std::string alias = absl::StrCat("_", module_name);
std::string alias = absl::StrCat("_", module_name.substr(last_dot_pos + 1));
// Generate a unique alias by adding _1 suffixes until we get an unused alias.
while (seen_aliases->find(alias) != seen_aliases->end()) {
absl::StrAppend(&alias, "_1");
}
printer_->Print("$statement$ as $alias$\n", "statement",
import_statement, "alias", alias);
import_map_[filename] = alias;
seen_aliases->insert(alias);
if (ContainsPythonKeyword(module_name)) {
if (*has_importlib == false) {
printer_->Print("import importlib\n");
*has_importlib = true;
}
printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias",
alias, "name", module_name);
} else {
std::string import_statement;
if (last_dot_pos == std::string::npos) {
import_statement = absl::StrCat("import ", module_name);
} else {
import_statement =
absl::StrCat("from ", module_name.substr(0, last_dot_pos), " import ",
module_name.substr(last_dot_pos + 1));
}
printer_->Print("$statement$ as $alias$\n", "statement", import_statement,
"alias", alias);
import_map_[filename] = alias;
seen_aliases->insert(alias);
}
}
void PyiGenerator::PrintImports() const {
// Prints imported dependent _pb2 files.
absl::flat_hash_set<std::string> seen_aliases;
bool has_importlib = false;
for (int i = 0; i < file_->dependency_count(); ++i) {
const FileDescriptor* dep = file_->dependency(i);
PrintImportForDescriptor(*dep, &seen_aliases);
PrintImportForDescriptor(*dep, &seen_aliases, &has_importlib);
for (int j = 0; j < dep->public_dependency_count(); ++j) {
PrintImportForDescriptor(
*dep->public_dependency(j), &seen_aliases);
PrintImportForDescriptor(*dep->public_dependency(j), &seen_aliases,
&has_importlib);
}
}

@ -77,9 +77,9 @@ class PROTOC_EXPORT PyiGenerator : public google::protobuf::compiler::CodeGenera
std::string* error) const override;
private:
void PrintImportForDescriptor(
const FileDescriptor& desc,
absl::flat_hash_set<std::string>* seen_aliases) const;
void PrintImportForDescriptor(const FileDescriptor& desc,
absl::flat_hash_set<std::string>* seen_aliases,
bool* has_importlib) const;
template <typename DescriptorT>
void Annotate(const std::string& label, const DescriptorT* descriptor) const;
void PrintImports() const;

@ -51,6 +51,7 @@ cc_library(
srcs = [
"accessors/accessors.cc",
"accessors/singular_bytes.cc",
"accessors/singular_message.cc",
"accessors/singular_scalar.cc",
],
hdrs = ["accessors/accessors.h"],

@ -65,6 +65,9 @@ std::unique_ptr<AccessorGenerator> AccessorGenerator::For(
case FieldDescriptor::TYPE_BYTES:
if (field.desc().is_repeated()) return nullptr;
return ForSingularBytes(field);
case FieldDescriptor::TYPE_MESSAGE:
if (field.desc().is_repeated()) return nullptr;
return ForSingularMessage(field);
default:
return nullptr;

@ -90,6 +90,8 @@ class AccessorGenerator {
Context<FieldDescriptor> field);
static std::unique_ptr<AccessorGenerator> ForSingularBytes(
Context<FieldDescriptor> field);
static std::unique_ptr<AccessorGenerator> ForSingularMessage(
Context<FieldDescriptor> field);
};
} // namespace rust

@ -0,0 +1,82 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google LLC. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google LLC. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "absl/strings/string_view.h"
#include "google/protobuf/compiler/rust/accessors/accessors.h"
#include "google/protobuf/compiler/rust/context.h"
#include "google/protobuf/compiler/rust/naming.h"
#include "google/protobuf/descriptor.h"
namespace google {
namespace protobuf {
namespace compiler {
namespace rust {
namespace {
class SingularMessage final : public AccessorGenerator {
public:
~SingularMessage() override = default;
void InMsgImpl(Context<FieldDescriptor> field) const override {
field.Emit(
{
{"field", field.desc().name()},
},
R"rs(
// inMsgImpl
pub fn $field$(&self) -> std::convert::Infallible {
todo!("b/285309454")
}
)rs");
}
void InExternC(Context<FieldDescriptor> field) const override {
field.Emit({},
R"rs(
// inExternC
)rs");
}
void InThunkCc(Context<FieldDescriptor> field) const override {
field.Emit({},
R"cc(
// inThunkCC
)cc");
}
};
} // namespace
std::unique_ptr<AccessorGenerator> AccessorGenerator::ForSingularMessage(
Context<FieldDescriptor> field) {
return std::make_unique<SingularMessage>();
}
} // namespace rust
} // namespace compiler
} // namespace protobuf
} // namespace google

@ -288,10 +288,61 @@ void MessageGenerator::GenerateRs(Context<Descriptor> msg) {
},
R"rs(
#[allow(non_camel_case_types)]
#[derive(Debug)]
pub struct $Msg$ {
$Msg.fields$
}
unsafe impl Sync for $Msg$ {}
unsafe impl Sync for $Msg$View<'_> {}
unsafe impl Send for $Msg$View<'_> {}
impl $pb$::Proxied for $Msg$ {
type View<'a> = $Msg$View<'a>;
type Mut<'a> = $Msg$Mut<'a>;
}
#[derive(Debug, Copy, Clone)]
pub struct $Msg$View<'a> {
msg: $NonNull$<u8>,
_phantom: std::marker::PhantomData<&'a ()>,
}
impl<'a> $pb$::ViewProxy<'a> for $Msg$View<'a> {
type Proxied = $Msg$;
fn as_view(&self) -> $pb$::View<'a, $Msg$> {
todo!("b/285309454")
}
fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> where 'a: 'shorter { todo!("b/285309454") }
}
impl<'a> $pb$::SettableValue<$Msg$> for $Msg$View<'a> {
fn set_on(self, _private: $pb$::__internal::Private, _mutator: $pb$::Mut<$Msg$>) {
todo!("b/285309454")
}
}
#[derive(Debug)]
pub struct $Msg$Mut<'a> {
_phantom: std::marker::PhantomData<&'a mut ()>,
}
impl<'a> $pb$::MutProxy<'a> for $Msg$Mut<'a> {
fn as_mut(&mut self) -> $pb$::Mut<'_, $Msg$> {
todo!("b/285309454")
}
fn into_mut<'shorter>(self) -> $pb$::Mut<'shorter, $Msg$> where 'a : 'shorter { todo!("b/285309454") }
}
impl<'a> $pb$::ViewProxy<'a> for $Msg$Mut<'a> {
type Proxied = $Msg$;
fn as_view(&self) -> $pb$::View<'_, $Msg$> {
todo!("b/285309454")
}
fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> where 'a: 'shorter { todo!("b/285309454") }
}
impl $Msg$ {
pub fn new() -> Self {
$Msg::new$

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save