diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 854f93484c..4b36b561c2 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -31,19 +31,19 @@ jobs: # Override cases with custom images - config: { name: "Bazel7", flags: --noenable_bzlmod } - version: Bazel7 + cache_key: Bazel7 image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e" targets: "//src/... //third_party/utf8_range/..." - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod --enable_workspace } - version: Bazel7bzlmod + cache_key: Bazel7bzlmod image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e" targets: "//src/... //third_party/utf8_range/..." - config: { name: "TCMalloc" } - version: TcMalloc + cache_key: TcMalloc image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:1c5133455481f4d1bb8afa477029604f41f1a3c46cebe4d9958cf1af95b5c87c" targets: "//src/... //third_party/utf8_range/..." - config: { name: "aarch64" } - version: TcMalloc + cache_key: TcMalloc targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test //third_party/utf8_range/..." image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:6.3.0-aarch64-68e662b3a56b881804dc4e9d45f949791cbc4b94" name: Linux ${{ matrix.config.name }} @@ -58,7 +58,7 @@ jobs: with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel-cache: cpp_linux/${{ matrix.version }} + bazel-cache: cpp_linux/${{ matrix.cache_key }} bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} exclude-targets: ${{ matrix.exclude-targets }} diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 64639b9d31..189e5c2907 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -18,31 +18,31 @@ jobs: matrix: include: - name: OpenJDK 8 - version: '8' + cache_key: '8' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 # TODO: b/318555165 - enable the layering check. Currently it does # not work correctly with the toolchain in this Docker image. targets: //java/... //java/internal:java_version //compatibility/... --features=-layering_check - name: OpenJDK 11 - version: '11' + cache_key: '11' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 targets: //java/... //java/internal:java_version //compatibility/... - name: OpenJDK 17 - version: '17' + cache_key: '17' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 targets: //java/... //java/internal:java_version //compatibility/... - name: Bazel7 - version: 'bazel7nobzlmod' + cache_key: 'bazel7nobzlmod' image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e targets: //java/... //java/internal:java_version //compatibility/... flags: --noenable_bzlmod - name: Bazel7 with Bzlmod - version: 'bazel7bzlmod' + cache_key: 'bazel7bzlmod' image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-cf84e92285ca133b9c8104ad7b14d70e953cbb8e targets: //java/... //java/internal:java_version //compatibility/... flags: --enable_bzlmod --enable_workspace - name: aarch64 - version: 'aarch64' + cache_key: 'aarch64' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17 targets: //java/... //compatibility/... //src/google/protobuf/compiler:protoc_aarch64_test @@ -58,7 +58,7 @@ jobs: with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - bazel-cache: java_linux/${{ matrix.version }} + bazel-cache: java_linux/${{ matrix.cache_key }} bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_JAVA_VERSION # TODO restore this test (or a better one) when gRPC has rebuilt with 26.x diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index 1db6c10c88..ea2726dd62 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -57,8 +57,7 @@ jobs: -scheme ProtocolBuffers \ -configuration ${{ matrix.xc_config }} \ -destination "${{ matrix.destination }}" \ - test \ - | xcpretty + test - name: Report ccache stats shell: bash diff --git a/MODULE.bazel b/MODULE.bazel index d9340924d3..fa8c7c5a6d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ module( # https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution # Thus the highest version in their module graph is resolved. bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl") -bazel_dep(name = "bazel_skylib", version = "1.4.1") +bazel_dep(name = "bazel_skylib", version = "1.7.0") bazel_dep(name = "jsoncpp", version = "1.9.5") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "rules_fuzzing", version = "0.5.2") @@ -27,9 +27,6 @@ bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "zlib", version = "1.3.1") bazel_dep(name = "bazel_features", version = "1.13.0", repo_name = "proto_bazel_features") -# TODO: remove after toolchain types are moved to protobuf -bazel_dep(name = "rules_proto", version = "4.0.0") - SUPPORTED_PYTHON_VERSIONS = [ "3.8", "3.9", @@ -98,5 +95,7 @@ use_repo(maven, "maven") # Development dependencies bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest", dev_dependency = True) -bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True) bazel_dep(name = "rules_buf", version = "0.3.0", dev_dependency = True) +bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True) +# rules_proto are needed for @com_google_protobuf_v25.0 used in //compatibility/... tests +bazel_dep(name = "rules_proto", version = "4.0.0", dev_dependency = True) diff --git a/WORKSPACE b/WORKSPACE index 425ba7f181..0459aff61a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -41,10 +41,10 @@ http_archive( http_archive( name = "com_google_googletest", - sha256 = "730215d76eace9dd49bf74ce044e8daa065d175f1ac891cc1d6bb184ef94e565", - strip_prefix = "googletest-f53219cdcb7b084ef57414efea92ee5b71989558", + sha256 = "7315acb6bf10e99f332c8a43f00d5fbb1ee6ca48c52f6b936991b216c586aaad", + strip_prefix = "googletest-1.15.0", urls = [ - "https://github.com/google/googletest/archive/f53219cdcb7b084ef57414efea92ee5b71989558.tar.gz" # 2023-03-16 + "https://github.com/google/googletest/releases/download/v1.15.0/googletest-1.15.0.tar.gz" # 2024-07-15 ], ) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 1d6749dd76..d0546f9b30 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -40,6 +40,7 @@ bzl_library( deps = [ "//bazel/common:proto_common_bzl", "//bazel/common:proto_info_bzl", + "//bazel/private:toolchain_helpers_bzl", "@rules_python//python:py_info_bzl", ], ) diff --git a/bazel/private/BUILD.bazel b/bazel/private/BUILD.bazel index 1ff703b53d..c90632d7b0 100644 --- a/bazel/private/BUILD.bazel +++ b/bazel/private/BUILD.bazel @@ -6,9 +6,14 @@ # https://developers.google.com/open-source/licenses/bsd load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//bazel/private:native_bool_flag.bzl", "native_bool_flag") licenses(["notice"]) +toolchain_type( + name = "proto_toolchain_type", +) + bzl_library( name = "upb_proto_library_internal_bzl", srcs = [ @@ -39,6 +44,11 @@ bzl_library( "proto_toolchain_rule.bzl", ], visibility = ["//bazel:__subpackages__"], + deps = [ + ":toolchain_helpers_bzl", + "//bazel/common:proto_common_bzl", + "//bazel/common:proto_lang_toolchain_info_bzl", + ], ) bzl_library( @@ -52,3 +62,26 @@ bzl_library( "//bazel/common:proto_lang_toolchain_info_bzl", ], ) + +native_bool_flag( + name = "experimental_proto_descriptor_sets_include_source_info", + flag = "experimental_proto_descriptor_sets_include_source_info", + match_value = "true", + visibility = ["//visibility:public"], +) + +native_bool_flag( + name = "strict_proto_deps", + flag = "strict_proto_deps", + match_value = "off", + result = False, + visibility = ["//visibility:public"], +) + +native_bool_flag( + name = "strict_public_imports", + flag = "strict_public_imports", + match_value = "off", + result = False, + visibility = ["//visibility:public"], +) diff --git a/bazel/private/native_bool_flag.bzl b/bazel/private/native_bool_flag.bzl new file mode 100644 index 0000000000..960fbed512 --- /dev/null +++ b/bazel/private/native_bool_flag.bzl @@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +A helper rule that reads a native boolean flag. +""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") + +def _impl(ctx): + return [BuildSettingInfo(value = ctx.attr.value)] + +_native_bool_flag_rule = rule( + implementation = _impl, + attrs = {"value": attr.bool()}, +) + +def native_bool_flag(*, name, flag, match_value = "true", result = True, **kwargs): + _native_bool_flag_rule( + name = name, + value = select({ + name + "_setting": result, + "//conditions:default": not result, + }), + **kwargs + ) + + native.config_setting( + name = name + "_setting", + values = {flag: match_value}, + visibility = ["//visibility:private"], + ) diff --git a/bazel/private/proto_library_rule.bzl b/bazel/private/proto_library_rule.bzl new file mode 100644 index 0000000000..b02f691f45 --- /dev/null +++ b/bazel/private/proto_library_rule.bzl @@ -0,0 +1,357 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +Implementation of proto_library rule. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/common:proto_common.bzl", "proto_common") +load("//bazel/common:proto_info.bzl", "ProtoInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") + +STRICT_DEPS_FLAG_TEMPLATE = ( + # + "--direct_dependencies_violation_msg=" + + "%%s is imported, but %s doesn't directly depend on a proto_library that 'srcs' it." +) + +def _check_srcs_package(target_package, srcs): + """Check that .proto files in sources are from the same package. + + This is done to avoid clashes with the generated sources.""" + + #TODO: this does not work with filegroups that contain files that are not in the package + for src in srcs: + if target_package != src.label.package: + fail("Proto source with label '%s' must be in same package as consuming rule." % src.label) + +def _get_import_prefix(ctx): + """Gets and verifies import_prefix attribute if it is declared.""" + + import_prefix = ctx.attr.import_prefix + + if not paths.is_normalized(import_prefix): + fail("should be normalized (without uplevel references or '.' path segments)", attr = "import_prefix") + if paths.is_absolute(import_prefix): + fail("should be a relative path", attr = "import_prefix") + + return import_prefix + +def _get_strip_import_prefix(ctx): + """Gets and verifies strip_import_prefix.""" + + strip_import_prefix = ctx.attr.strip_import_prefix + + if not paths.is_normalized(strip_import_prefix): + fail("should be normalized (without uplevel references or '.' path segments)", attr = "strip_import_prefix") + + if paths.is_absolute(strip_import_prefix): + strip_import_prefix = strip_import_prefix[1:] + else: # Relative to current package + strip_import_prefix = _join(ctx.label.package, strip_import_prefix) + + return strip_import_prefix.removesuffix("/") + +def _proto_library_impl(ctx): + # Verifies attributes. + _check_srcs_package(ctx.label.package, ctx.attr.srcs) + srcs = ctx.files.srcs + deps = [dep[ProtoInfo] for dep in ctx.attr.deps] + exports = [dep[ProtoInfo] for dep in ctx.attr.exports] + import_prefix = _get_import_prefix(ctx) + strip_import_prefix = _get_strip_import_prefix(ctx) + check_for_reexport = deps + exports if not srcs else exports + _PackageSpecificationInfo = bazel_features.globals.PackageSpecificationInfo + for proto in check_for_reexport: + if getattr(proto, "allow_exports", None): + if not _PackageSpecificationInfo: + fail("Allowlist checks not supported before Bazel 6.4.0") + if not proto.allow_exports[_PackageSpecificationInfo].contains(ctx.label): + fail("proto_library '%s' can't be reexported in package '//%s'" % (proto.direct_descriptor_set.owner, ctx.label.package)) + + proto_path, virtual_srcs = _process_srcs(ctx, srcs, import_prefix, strip_import_prefix) + descriptor_set = ctx.actions.declare_file(ctx.label.name + "-descriptor-set.proto.bin") + proto_info = ProtoInfo( + srcs = virtual_srcs, + deps = deps, + descriptor_set = descriptor_set, + proto_path = proto_path, + workspace_root = ctx.label.workspace_root, + bin_dir = ctx.bin_dir.path, + allow_exports = ctx.attr.allow_exports, + ) + + _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set) + + # We assume that the proto sources will not have conflicting artifacts + # with the same root relative path + data_runfiles = ctx.runfiles( + files = [proto_info.direct_descriptor_set], + transitive_files = depset(transitive = [proto_info.transitive_sources]), + ) + return [ + proto_info, + DefaultInfo( + files = depset([proto_info.direct_descriptor_set]), + default_runfiles = ctx.runfiles(), # empty + data_runfiles = data_runfiles, + ), + ] + +def _process_srcs(ctx, srcs, import_prefix, strip_import_prefix): + """Returns proto_path and sources, optionally symlinking them to _virtual_imports. + + Returns: + (str, [File]) A pair of proto_path and virtual_sources. + """ + if import_prefix != "" or strip_import_prefix != "": + # Use virtual source roots + return _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix) + else: + # No virtual source roots + return "", srcs + +def _join(*path): + return "/".join([p for p in path if p != ""]) + +def _symlink_to_virtual_imports(ctx, srcs, import_prefix, strip_import_prefix): + """Symlinks srcs to _virtual_imports. + + Returns: + A pair proto_path, directs_sources. + """ + virtual_imports = _join("_virtual_imports", ctx.label.name) + proto_path = _join(ctx.label.package, virtual_imports) + + if ctx.label.workspace_name == "": + full_strip_import_prefix = strip_import_prefix + else: + full_strip_import_prefix = _join("..", ctx.label.workspace_name, strip_import_prefix) + if full_strip_import_prefix: + full_strip_import_prefix += "/" + + virtual_srcs = [] + for src in srcs: + # Remove strip_import_prefix + if not src.short_path.startswith(full_strip_import_prefix): + fail(".proto file '%s' is not under the specified strip prefix '%s'" % + (src.short_path, full_strip_import_prefix)) + import_path = src.short_path[len(full_strip_import_prefix):] + + # Add import_prefix + virtual_src = ctx.actions.declare_file(_join(virtual_imports, import_prefix, import_path)) + ctx.actions.symlink( + output = virtual_src, + target_file = src, + progress_message = "Symlinking virtual .proto sources for %{label}", + ) + virtual_srcs.append(virtual_src) + return proto_path, virtual_srcs + +def _write_descriptor_set(ctx, proto_info, deps, exports, descriptor_set): + """Writes descriptor set.""" + if proto_info.direct_sources == []: + ctx.actions.write(descriptor_set, "") + return + + dependencies_descriptor_sets = depset(transitive = [dep.transitive_descriptor_sets for dep in deps]) + + args = ctx.actions.args() + + if ctx.attr._experimental_proto_descriptor_sets_include_source_info[BuildSettingInfo].value: + args.add("--include_source_info") + if hasattr(ctx.attr, "_retain_options") and ctx.attr._retain_options: + args.add("--retain_options") + + strict_deps = ctx.attr._strict_proto_deps[BuildSettingInfo].value + if strict_deps: + if proto_info.direct_sources: + strict_importable_sources = depset( + direct = proto_info._direct_proto_sources, + transitive = [dep._exported_sources for dep in deps], + ) + else: + strict_importable_sources = None + if strict_importable_sources: + args.add_joined( + "--direct_dependencies", + strict_importable_sources, + map_each = proto_common.get_import_path, + join_with = ":", + ) + # Example: `--direct_dependencies a.proto:b.proto` + + else: + # The proto compiler requires an empty list to turn on strict deps checking + args.add("--direct_dependencies=") + + # Set `-direct_dependencies_violation_msg=` + args.add(ctx.label, format = STRICT_DEPS_FLAG_TEMPLATE) + + strict_imports = ctx.attr._strict_public_imports[BuildSettingInfo].value + if strict_imports: + public_import_protos = depset(transitive = [export._exported_sources for export in exports]) + if not public_import_protos: + # This line is necessary to trigger the check. + args.add("--allowed_public_imports=") + else: + args.add_joined( + "--allowed_public_imports", + public_import_protos, + map_each = proto_common.get_import_path, + join_with = ":", + ) + if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: + toolchain = ctx.toolchains[toolchains.PROTO_TOOLCHAIN] + if not toolchain: + fail("Protocol compiler toolchain could not be resolved.") + proto_lang_toolchain_info = toolchain.proto + else: + proto_lang_toolchain_info = proto_common.ProtoLangToolchainInfo( + out_replacement_format_flag = "--descriptor_set_out=%s", + output_files = "single", + mnemonic = "GenProtoDescriptorSet", + progress_message = "Generating Descriptor Set proto_library %{label}", + proto_compiler = ctx.executable._proto_compiler, + protoc_opts = ctx.fragments.proto.experimental_protoc_opts, + plugin = None, + ) + + proto_common.compile( + ctx.actions, + proto_info, + proto_lang_toolchain_info, + generated_files = [descriptor_set], + additional_inputs = dependencies_descriptor_sets, + additional_args = args, + ) + +proto_library = rule( + _proto_library_impl, + # TODO: proto_common docs are missing + # TODO: ProtoInfo link doesn't work and docs are missing + doc = """ +

If using Bazel, please load the rule from +https://github.com/bazelbuild/rules_proto. + +

Use proto_library to define libraries of protocol buffers which +may be used from multiple languages. A proto_library may be listed +in the deps clause of supported rules, such as +java_proto_library. + +

When compiled on the command-line, a proto_library creates a file +named foo-descriptor-set.proto.bin, which is the descriptor set for +the messages the rule srcs. The file is a serialized +FileDescriptorSet, which is described in + +https://developers.google.com/protocol-buffers/docs/techniques#self-description. + +

It only contains information about the .proto files directly +mentioned by a proto_library rule; the collection of transitive +descriptor sets is available through the +[ProtoInfo].transitive_descriptor_sets Starlark provider. +See documentation in proto_info.bzl. + +

Recommended code organization: +

""", + attrs = { + "srcs": attr.label_list( + allow_files = [".proto", ".protodevel"], + flags = ["DIRECT_COMPILE_TIME_INPUT"], + # TODO: Should .protodevel be advertised or deprecated? + doc = """ +The list of .proto and .protodevel files that are +processed to create the target. This is usually a non empty list. One usecase +where srcs can be empty is an alias-library. This is a +proto_library rule having one or more other proto_library in deps. +This pattern can be used to e.g. export a public api under a persistent name.""", + ), + "deps": attr.label_list( + providers = [ProtoInfo], + doc = """ +The list of other proto_library rules that the target depends upon. +A proto_library may only depend on other proto_library +targets. It may not depend on language-specific libraries.""", + ), + "exports": attr.label_list( + providers = [ProtoInfo], + doc = """ +List of proto_library targets that can be referenced via "import public" in the +proto source. +It's an error if you use "import public" but do not list the corresponding library +in the exports attribute. +Note that you have list the library both in deps and exports since not all +lang_proto_library implementations have been changed yet.""", + ), + "strip_import_prefix": attr.string( + default = "/", + doc = """ +The prefix to strip from the paths of the .proto files in this rule. + +

When set, .proto source files in the srcs attribute of this rule are +accessible at their path with this prefix cut off. + +

If it's a relative path (not starting with a slash), it's taken as a package-relative +one. If it's an absolute one, it's understood as a repository-relative path. + +

The prefix in the import_prefix attribute is added after this prefix is +stripped.""", + ), + "import_prefix": attr.string( + doc = """ +The prefix to add to the paths of the .proto files in this rule. + +

When set, the .proto source files in the srcs attribute of this rule are +accessible at is the value of this attribute prepended to their repository-relative path. + +

The prefix in the strip_import_prefix attribute is removed before this +prefix is added.""", + ), + "allow_exports": attr.label( + cfg = "exec", + providers = [bazel_features.globals.PackageSpecificationInfo] if bazel_features.globals.PackageSpecificationInfo else [], + doc = """ +An optional allowlist that prevents proto library to be reexported or used in +lang_proto_library that is not in one of the listed packages.""", + ), + "data": attr.label_list( + allow_files = True, + flags = ["SKIP_CONSTRAINTS_OVERRIDE"], + ), + # buildifier: disable=attr-license (calling attr.license()) + "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), + "_experimental_proto_descriptor_sets_include_source_info": attr.label( + default = "//bazel/private:experimental_proto_descriptor_sets_include_source_info", + ), + "_strict_proto_deps": attr.label( + default = + "//bazel/private:strict_proto_deps", + ), + "_strict_public_imports": attr.label( + default = "//bazel/private:strict_public_imports", + ), + } | toolchains.if_legacy_toolchain({ + "_proto_compiler": attr.label( + cfg = "exec", + executable = True, + allow_files = True, + default = configuration_field("proto", "proto_compiler"), + ), + }), # buildifier: disable=attr-licenses (attribute called licenses) + fragments = ["proto"], + provides = [ProtoInfo], + toolchains = toolchains.use_toolchain(toolchains.PROTO_TOOLCHAIN), +) diff --git a/bazel/private/proto_toolchain_rule.bzl b/bazel/private/proto_toolchain_rule.bzl index 9487a06026..8a11fac9db 100644 --- a/bazel/private/proto_toolchain_rule.bzl +++ b/bazel/private/proto_toolchain_rule.bzl @@ -1,13 +1,9 @@ """A Starlark implementation of the proto_toolchain rule.""" -load("//bazel/common:proto_common.bzl", "proto_common") load("//bazel/common:proto_lang_toolchain_info.bzl", "ProtoLangToolchainInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") def _impl(ctx): - kwargs = {} - if getattr(proto_common, "INCOMPATIBLE_PASS_TOOLCHAIN_TYPE", False): - kwargs["toolchain_type"] = "@rules_proto//proto:toolchain_type" - return [ DefaultInfo( files = depset(), @@ -23,7 +19,7 @@ def _impl(ctx): protoc_opts = ctx.fragments.proto.experimental_protoc_opts, progress_message = ctx.attr.progress_message, mnemonic = ctx.attr.mnemonic, - **kwargs + toolchain_type = toolchains.PROTO_TOOLCHAIN, ), ), ] diff --git a/bazel/private/toolchain_helpers.bzl b/bazel/private/toolchain_helpers.bzl index e58c9d6a0e..aa49eb8cf4 100644 --- a/bazel/private/toolchain_helpers.bzl +++ b/bazel/private/toolchain_helpers.bzl @@ -45,5 +45,5 @@ toolchains = struct( find_toolchain = _find_toolchain, if_legacy_toolchain = _if_legacy_toolchain, INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION = _incompatible_toolchain_resolution, - PROTO_TOOLCHAIN = "@rules_proto//proto:toolchain_type", + PROTO_TOOLCHAIN = "//bazel/private:proto_toolchain_type", ) diff --git a/bazel/proto_library.bzl b/bazel/proto_library.bzl index 3006e3c0c2..f0fece1553 100644 --- a/bazel/proto_library.bzl +++ b/bazel/proto_library.bzl @@ -1,3 +1,20 @@ -"""proto_library rule""" +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +""" +Macro of proto_library rule. +""" -proto_library = native.proto_library +load("@proto_bazel_features//:features.bzl", "bazel_features") +load("//bazel/private:proto_library_rule.bzl", _proto_library = "proto_library") + +def proto_library(**kwattrs): + # This condition causes Starlark rules to be used only on Bazel >=7.0.0 + if bazel_features.proto.starlark_proto_info: + _proto_library(**kwattrs) + else: + # On older Bazel versions keep using native rules, so that mismatch in ProtoInfo doesn't happen + native.proto_library(**kwattrs) diff --git a/bazel/py_proto_library.bzl b/bazel/py_proto_library.bzl index 86af999502..6e3d041f36 100644 --- a/bazel/py_proto_library.bzl +++ b/bazel/py_proto_library.bzl @@ -3,6 +3,7 @@ load("@rules_python//python:py_info.bzl", "PyInfo") load("//bazel/common:proto_common.bzl", "proto_common") load("//bazel/common:proto_info.bzl", "ProtoInfo") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" @@ -22,9 +23,6 @@ _PyProtoInfo = provider( def _filter_provider(provider, *attrs): return [dep[provider] for attr in attrs for dep in attr if provider in dep] -def _incompatible_toolchains_enabled(): - return getattr(proto_common, "INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION", False) - def _py_proto_aspect_impl(target, ctx): """Generates and compiles Python code for a proto_library. @@ -51,7 +49,7 @@ def _py_proto_aspect_impl(target, ctx): proto.path, )) - if _incompatible_toolchains_enabled(): + if proto_common.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: toolchain = ctx.toolchains[PY_PROTO_TOOLCHAIN] if not toolchain: fail("No toolchains registered for '%s'." % PY_PROTO_TOOLCHAIN) @@ -120,15 +118,15 @@ def _py_proto_aspect_impl(target, ctx): _py_proto_aspect = aspect( implementation = _py_proto_aspect_impl, - attrs = {} if _incompatible_toolchains_enabled() else { + attrs = toolchains.if_legacy_toolchain({ "_aspect_proto_toolchain": attr.label( default = "//python:python_toolchain", ), - }, + }), attr_aspects = ["deps"], required_providers = [ProtoInfo], provides = [_PyProtoInfo], - toolchains = [PY_PROTO_TOOLCHAIN] if _incompatible_toolchains_enabled() else [], + toolchains = toolchains.use_toolchain(PY_PROTO_TOOLCHAIN), ) def _py_proto_library_rule(ctx): diff --git a/bazel/toolchains/BUILD.bazel b/bazel/toolchains/BUILD.bazel index 34a45d5bbd..2c21426b15 100644 --- a/bazel/toolchains/BUILD.bazel +++ b/bazel/toolchains/BUILD.bazel @@ -8,6 +8,7 @@ bzl_library( visibility = ["//visibility:public"], deps = [ "//bazel/private:proto_toolchain_rule_bzl", + "//bazel/private:toolchain_helpers_bzl", ], ) diff --git a/bazel/toolchains/proto_toolchain.bzl b/bazel/toolchains/proto_toolchain.bzl index 171e08cad5..1bad37b3a6 100644 --- a/bazel/toolchains/proto_toolchain.bzl +++ b/bazel/toolchains/proto_toolchain.bzl @@ -4,6 +4,7 @@ The macro additionally creates toolchain target when toolchain_type is given. """ load("//bazel/private:proto_toolchain_rule.bzl", _proto_toolchain_rule = "proto_toolchain") +load("//bazel/private:toolchain_helpers.bzl", "toolchains") def proto_toolchain(*, name, proto_compiler, exec_compatible_with = []): """Creates a proto_toolchain and toolchain target for proto_library. @@ -19,7 +20,7 @@ def proto_toolchain(*, name, proto_compiler, exec_compatible_with = []): native.toolchain( name = name + "_toolchain", - toolchain_type = "@rules_proto//proto:toolchain_type", + toolchain_type = toolchains.PROTO_TOOLCHAIN, exec_compatible_with = exec_compatible_with, target_compatible_with = [], toolchain = name, diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl index f667a40881..46b60252f6 100644 --- a/build_defs/cpp_opts.bzl +++ b/build_defs/cpp_opts.bzl @@ -15,7 +15,6 @@ COPTS = select({ "/wd4506", # no definition for inline function 'function' "/wd4800", # 'type' : forcing value to bool 'true' or 'false' (performance warning) "/wd4996", # The compiler encountered a deprecated declaration. - "/utf-8", # Set source and execution character sets to UTF-8 ], "//conditions:default": [ "-DHAVE_ZLIB", diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index 384bb56563..e38330df62 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -143,6 +143,7 @@ cc_library( ":conformance_cc_proto", "//src/google/protobuf", "//src/google/protobuf:descriptor_legacy", + "//src/google/protobuf:endian", "//src/google/protobuf:protobuf_lite", "//src/google/protobuf/util:differencer", "//src/google/protobuf/util:json_util", @@ -232,6 +233,7 @@ cc_binary( "//src/google/protobuf", "//src/google/protobuf:port", "//src/google/protobuf:protobuf_lite", + "//src/google/protobuf/json", "//src/google/protobuf/stubs", "//src/google/protobuf/util:json_util", "//src/google/protobuf/util:type_resolver", @@ -304,6 +306,11 @@ py_binary( ], ) +py_binary( + name = "update_failure_list", + srcs = ["update_failure_list.py"], +) + inline_sh_binary( name = "conformance_php", testonly = 1, diff --git a/conformance/binary_json_conformance_suite.cc b/conformance/binary_json_conformance_suite.cc index f631b9adad..892a2a4426 100644 --- a/conformance/binary_json_conformance_suite.cc +++ b/conformance/binary_json_conformance_suite.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" +#include "json/reader.h" +#include "json/value.h" #include "conformance/conformance.pb.h" #include "conformance_test.h" #include "conformance/test_protos/test_messages_edition2023.pb.h" @@ -43,6 +46,7 @@ using conformance::ConformanceRequest; using conformance::ConformanceResponse; +using conformance::TestStatus; using conformance::WireFormat; using google::protobuf::Descriptor; using google::protobuf::FieldDescriptor; @@ -302,19 +306,22 @@ bool BinaryAndJsonConformanceSuite::ParseResponse( const std::string& test_name = setting.GetTestName(); ConformanceLevel level = setting.GetLevel(); + TestStatus test; + test.set_name(test_name); switch (response.result_case()) { case ConformanceResponse::kProtobufPayload: { if (requested_output != conformance::PROTOBUF) { - ReportFailure(test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided PROTOBUF instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided PROTOBUF instead.")); + ReportFailure(test, level, request, response); return false; } if (!test_message->ParseFromString(response.protobuf_payload())) { - ReportFailure(test_name, level, request, response, - "Protobuf output we received from test was unparseable."); + test.set_failure_message( + "Protobuf output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -323,16 +330,17 @@ bool BinaryAndJsonConformanceSuite::ParseResponse( case ConformanceResponse::kJsonPayload: { if (requested_output != conformance::JSON) { - ReportFailure(test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided JSON instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided JSON instead.")); + ReportFailure(test, level, request, response); return false; } if (!ParseJsonResponse(response, test_message)) { - ReportFailure(test_name, level, request, response, - "JSON output we received from test was unparseable."); + test.set_failure_message( + "JSON output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -437,14 +445,19 @@ void BinaryAndJsonConformanceSuiteImpl:: absl::StrCat(setting.ConformanceLevelToString(level), ".", setting.GetSyntaxIdentifier(), ".ProtobufInput.", test_name); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to parse, but didn't."); + test.set_failure_message("Should have failed to parse, but didn't."); + suite_.ReportFailure(test, level, request, response); } } @@ -631,34 +644,38 @@ void BinaryAndJsonConformanceSuiteImpl< setting.ConformanceLevelToString(level), ".", setting.GetSyntaxIdentifier(), ".JsonInput.", test_name, ".Validator"); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); return; } if (response.result_case() != ConformanceResponse::kJsonPayload) { - suite_.ReportFailure(effective_test_name, level, request, response, - absl::StrCat("Expected JSON payload but got type ", - response.result_case())); + test.set_failure_message(absl::StrCat("Expected JSON payload but got type ", + response.result_case())); + suite_.ReportFailure(test, level, request, response); return; } Json::Reader reader; Json::Value value; if (!reader.parse(response.json_payload(), value)) { - suite_.ReportFailure( - effective_test_name, level, request, response, + test.set_failure_message( absl::StrCat("JSON payload cannot be parsed as valid JSON: ", reader.getFormattedErrorMessages())); + suite_.ReportFailure(test, level, request, response); return; } if (!validator(value)) { - suite_.ReportFailure(effective_test_name, level, request, response, - "JSON payload validation failed."); + test.set_failure_message("JSON payload validation failed."); + suite_.ReportFailure(test, level, request, response); return; } - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } template @@ -677,14 +694,19 @@ void BinaryAndJsonConformanceSuiteImpl::ExpectParseFailureForJson( absl::StrCat(setting.ConformanceLevelToString(level), ".", SyntaxIdentifier(), ".JsonInput.", test_name); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to parse, but didn't."); + test.set_failure_message("Should have failed to parse, but didn't."); + suite_.ReportFailure(test, level, request, response); } } @@ -707,14 +729,19 @@ void BinaryAndJsonConformanceSuiteImpl:: absl::StrCat(setting.ConformanceLevelToString(level), ".", SyntaxIdentifier(), ".", test_name, ".JsonOutput"); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kSerializeError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to serialize, but didn't."); + test.set_failure_message("Should have failed to serialize, but didn't."); + suite_.ReportFailure(test, level, request, response); } } @@ -1207,7 +1234,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( { // Tests oneof with default value. - const std::string proto = default_value; + const std::string& proto = default_value; MessageType test_message; test_message.MergeFromString(proto); std::string text; @@ -1223,7 +1250,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( { // Tests oneof with non-default value. - const std::string proto = non_default_value; + const std::string& proto = non_default_value; MessageType test_message; test_message.MergeFromString(proto); std::string text; @@ -1240,7 +1267,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( { // Tests oneof with multiple values of the same field. const std::string proto = absl::StrCat(default_value, non_default_value); - const std::string expected_proto = non_default_value; + const std::string& expected_proto = non_default_value; MessageType test_message; test_message.MergeFromString(expected_proto); std::string text; @@ -1266,7 +1293,7 @@ void BinaryAndJsonConformanceSuiteImpl::TestValidDataForOneofType( GetDefaultValue(other_type)); const std::string proto = absl::StrCat(other_value, non_default_value); - const std::string expected_proto = non_default_value; + const std::string& expected_proto = non_default_value; MessageType test_message; test_message.MergeFromString(expected_proto); std::string text; @@ -1420,12 +1447,18 @@ void BinaryAndJsonConformanceSuiteImpl::TestUnknownOrdering() { conformance::BINARY_TEST, prototype, "UnknownOrdering", serialized); const ConformanceRequest& request = setting.GetRequest(); ConformanceResponse response; - suite_.RunTest(setting.GetTestName(), request, &response); + if (!suite_.RunTest(setting.GetTestName(), request, &response)) { + return; + } + MessageType response_message; + TestStatus test; + test.set_name(setting.GetTestName()); if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(setting.GetTestName(), request, response); + suite_.ReportSkip(test, request, response); return; } + suite_.ParseResponse(response, setting, &response_message); const UnknownFieldSet& ufs = response_message.unknown_fields(); @@ -1441,10 +1474,10 @@ void BinaryAndJsonConformanceSuiteImpl::TestUnknownOrdering() { ufs.field(1).varint() != 123 || ufs.field(2).length_delimited() != "def" || ufs.field(3).varint() != 456) { - suite_.ReportFailure(setting.GetTestName(), setting.GetLevel(), request, - response, "Unknown field mismatch"); + test.set_failure_message("Unknown field mismatch"); + suite_.ReportFailure(test, setting.GetLevel(), request, response); } else { - suite_.ReportSuccess(setting.GetTestName()); + suite_.ReportSuccess(test); } } diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 2357d59ad6..a53b5e21d4 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -57,11 +57,20 @@ enum TestCategory { TEXT_FORMAT_TEST = 5; } +// Meant to encapsulate all types of tests: successes, skips, failures, etc. +// Therefore, this may or may not have a failure message. Failure messages +// may be truncated for our failure lists. +message TestStatus { + string name = 1; + string failure_message = 2; +} + // The conformance runner will request a list of failures as the first request. // This will be known by message_type == "conformance.FailureSet", a conformance // test should return a serialized FailureSet in protobuf_payload. message FailureSet { - repeated string failure = 1; + repeated TestStatus test = 2; + reserved 1; } // Represents a single test case's input. The testee should: diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 0f4196a480..45fa91aa7f 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -7,15 +7,15 @@ #include #include +#include #include #include #include +#include #include #include -#include "google/protobuf/util/json_util.h" -#include "google/protobuf/util/type_resolver_util.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" @@ -27,12 +27,15 @@ #include "editions/golden/test_messages_proto2_editions.pb.h" #include "editions/golden/test_messages_proto3_editions.pb.h" #include "google/protobuf/endian.h" +#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" #include "google/protobuf/test_messages_proto2.pb.h" #include "google/protobuf/test_messages_proto3.pb.h" #include "google/protobuf/test_messages_proto3.pb.h" #include "google/protobuf/text_format.h" +#include "google/protobuf/util/json_util.h" #include "google/protobuf/util/type_resolver.h" +#include "google/protobuf/util/type_resolver_util.h" #include "google/protobuf/stubs/status_macros.h" // Must be included last. @@ -241,8 +244,9 @@ absl::StatusOr Harness::ServeConformanceRequest() { serialized_output.size())); if (verbose_) { - ABSL_LOG(INFO) << "conformance-cpp: request=" << request.ShortDebugString() - << ", response=" << response->ShortDebugString(); + ABSL_LOG(INFO) << "conformance-cpp: request=" + << google::protobuf::ShortFormat(request) + << ", response=" << google::protobuf::ShortFormat(*response); } return false; } diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py index 60182e1e9f..77076bd885 100755 --- a/conformance/conformance_python.py +++ b/conformance/conformance_python.py @@ -98,36 +98,38 @@ def do_test(request): "Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64", ] for x in failures: - failure_set.failure.append(x) + failure_set.test.append(conformance_pb2.TestStatus(name=x)) response.protobuf_payload = failure_set.SerializeToString() return response - isJson = (request.WhichOneof('payload') == 'json_payload') + isJson = request.WhichOneof("payload") == "json_payload" test_message = _create_test_message(request.message_type) if (not isJson) and (test_message is None): raise ProtocolError("Protobuf request doesn't have specific payload type") try: - if request.WhichOneof('payload') == 'protobuf_payload': + if request.WhichOneof("payload") == "protobuf_payload": try: test_message.ParseFromString(request.protobuf_payload) except message.DecodeError as e: response.parse_error = str(e) return response - elif request.WhichOneof('payload') == 'json_payload': + elif request.WhichOneof("payload") == "json_payload": try: - ignore_unknown_fields = \ - request.test_category == \ - conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST - json_format.Parse(request.json_payload, test_message, - ignore_unknown_fields) + ignore_unknown_fields = ( + request.test_category + == conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST + ) + json_format.Parse( + request.json_payload, test_message, ignore_unknown_fields + ) except Exception as e: response.parse_error = str(e) return response - elif request.WhichOneof('payload') == 'text_payload': + elif request.WhichOneof("payload") == "text_payload": try: text_format.Parse(request.text_payload, test_message) except Exception as e: @@ -152,7 +154,8 @@ def do_test(request): elif request.requested_output_format == conformance_pb2.TEXT_FORMAT: response.text_payload = text_format.MessageToString( - test_message, print_unknown_fields=request.print_unknown_fields) + test_message, print_unknown_fields=request.print_unknown_fields + ) except Exception as e: response.runtime_error = str(e) @@ -163,7 +166,7 @@ def do_test(request): def do_test_io(): length_bytes = sys.stdin.buffer.read(4) if len(length_bytes) == 0: - return False # EOF + return False # EOF elif len(length_bytes) != 4: raise IOError("I/O error") @@ -183,17 +186,24 @@ def do_test_io(): sys.stdout.buffer.flush() if verbose: - sys.stderr.write("conformance_python: request=%s, response=%s\n" % ( - request.ShortDebugString().c_str(), - response.ShortDebugString().c_str())) + sys.stderr.write( + "conformance_python: request=%s, response=%s\n" + % ( + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str(), + ) + ) global test_count test_count += 1 return True + while True: if not do_test_io(): - sys.stderr.write("conformance_python: received EOF from test runner " + - "after %s tests, exiting\n" % (test_count)) + sys.stderr.write( + "conformance_python: received EOF from test runner " + + "after %s tests, exiting\n" % (test_count,) + ) sys.exit(0) diff --git a/conformance/conformance_rust.rs b/conformance/conformance_rust.rs index adfe8578ab..1d428dfb32 100644 --- a/conformance/conformance_rust.rs +++ b/conformance/conformance_rust.rs @@ -7,12 +7,14 @@ use conformance_rust_proto::{ConformanceRequest, ConformanceResponse, WireFormat}; #[cfg(cpp_kernel)] -use protobuf_cpp as kernel; +use protobuf_cpp as protobuf; #[cfg(upb_kernel)] -use protobuf_upb as kernel; +use protobuf_upb as protobuf; -use kernel::Optional::{Set, Unset}; +use protobuf::prelude::*; +use protobuf::Optional::{Set, Unset}; +use protobuf::ParseError; use std::io::{self, ErrorKind, Read, Write}; use test_messages_edition2023_rust_proto::TestAllTypesEdition2023; @@ -73,52 +75,39 @@ fn do_test(req: &ConformanceRequest) -> ConformanceResponse { Set(bytes) => bytes, }; + fn roundtrip(bytes: &[u8]) -> Result, ParseError> { + T::parse(bytes).map(|msg| msg.serialize().unwrap()) + } + let serialized = match message_type.as_bytes() { b"protobuf_test_messages.proto2.TestAllTypesProto2" => { - if let Ok(msg) = TestAllTypesProto2::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.proto3.TestAllTypesProto3" => { - if let Ok(msg) = TestAllTypesProto3::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.editions.TestAllTypesEdition2023" => { - if let Ok(msg) = TestAllTypesEdition2023::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.editions.proto2.TestAllTypesProto2" => { - if let Ok(msg) = EditionsTestAllTypesProto2::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } b"protobuf_test_messages.editions.proto3.TestAllTypesProto3" => { - if let Ok(msg) = EditionsTestAllTypesProto3::parse(bytes) { - msg.serialize().unwrap() - } else { - resp.set_parse_error("failed to parse bytes"); - return resp; - } + roundtrip::(bytes) } _ => panic!("unexpected msg type {message_type}"), }; - resp.set_protobuf_payload(serialized); - return resp; + match serialized { + Ok(serialized) => { + resp.set_protobuf_payload(serialized); + } + Err(_) => { + resp.set_parse_error("failed to parse bytes"); + } + } + + resp } fn main() { diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 2d7dfcd344..2b0d728ae0 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -9,14 +9,19 @@ #include +#include #include #include +#include #include #include #include +#include #include "google/protobuf/util/field_comparator.h" #include "google/protobuf/util/message_differencer.h" +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" @@ -25,11 +30,13 @@ #include "conformance/conformance.pb.h" #include "conformance/conformance.pb.h" #include "google/protobuf/descriptor_legacy.h" +#include "google/protobuf/endian.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" using conformance::ConformanceRequest; using conformance::ConformanceResponse; +using conformance::TestStatus; using conformance::WireFormat; using google::protobuf::util::DefaultFieldComparator; using google::protobuf::util::MessageDifferencer; @@ -37,6 +44,15 @@ using std::string; namespace { +static void ReplaceAll(std::string& input, std::string replace_word, + std::string replace_by) { + size_t pos = input.find(replace_word); + while (pos != std::string::npos) { + input.replace(pos, replace_word.length(), replace_by); + pos = input.find(replace_word, pos + replace_by.length()); + } +} + static std::string ToOctString(const std::string& binary_string) { std::string oct_string; for (size_t i = 0; i < binary_string.size(); i++) { @@ -52,16 +68,83 @@ static std::string ToOctString(const std::string& binary_string) { return oct_string; } -template -bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file, - absl::string_view msg, absl::string_view output_dir, - std::string* output) { +// Returns full filename path of written .txt file if successful +static std::string ProduceOctalSerialized(const std::string& request, + uint32_t len) { + char* len_split_bytes = static_cast(static_cast(&len)); + + std::string out; + + std::string hex_repr; + for (int i = 0; i < 4; i++) { + auto conversion = (unsigned int)static_cast(len_split_bytes[i]); + std::string hex = absl::StrFormat("\\x%x", conversion); + absl::StrAppend(&hex_repr, hex); + } + + absl::StrAppend(&out, hex_repr); + + absl::StrAppend(&out, ToOctString(request)); + + return out; +} + +static std::string WriteToFile(const std::string& octal_serialized, + const std::string& output_dir, + const std::string& test_name) { + std::string test_name_txt = test_name; + ReplaceAll(test_name_txt, ".", "_"); + absl::StrAppend(&test_name_txt, ".txt"); + std::string full_filename; + if (!output_dir.empty()) { + full_filename = output_dir; + if (*output_dir.rbegin() != '/') { + full_filename.push_back('/'); + } + absl::StrAppend(&full_filename, test_name_txt); + } + std::ofstream os{std::string(full_filename)}; + if (os) { + os << octal_serialized; + return full_filename; + } else { + ABSL_LOG(INFO) << "Failed to open file for debugging: " << full_filename + << "\n"; + return ""; + } +} + +// Removes all newlines. +static void Normalize(std::string& input) { + input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); +} + +// Sets up a failure message properly for our failure lists. +static TestStatus FormatFailureMessage(TestStatus& input) { + // Make copy just this once, as we need to modify it for our failure lists. + std::string formatted_failure_message = input.failure_message(); + // Remove newlines + Normalize(formatted_failure_message); + // Truncate failure message if needed + if (formatted_failure_message.length() > 128) { + formatted_failure_message = formatted_failure_message.substr(0, 128); + } + TestStatus properly_formatted; + properly_formatted.set_name(input.name()); + properly_formatted.set_failure_message(formatted_failure_message); + return properly_formatted; +} + +bool CheckSetEmpty(const absl::btree_map& set_to_check, + absl::string_view write_to_file, absl::string_view msg, + absl::string_view output_dir, std::string* output) { if (set_to_check.empty()) return true; absl::StrAppendFormat(output, "\n"); absl::StrAppendFormat(output, "%s\n\n", msg); - for (absl::string_view v : set_to_check) { - absl::StrAppendFormat(output, " %s\n", v); + for (const auto& pair : set_to_check) { + absl::StrAppendFormat(output, " %s # %s\n", pair.first, + pair.second.failure_message()); } absl::StrAppendFormat(output, "\n"); @@ -70,19 +153,18 @@ bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file, absl::string_view filename = write_to_file; if (!output_dir.empty()) { full_filename = std::string(output_dir); - if (*output_dir.rbegin() != '/') { - full_filename.push_back('/'); - } absl::StrAppend(&full_filename, write_to_file); filename = full_filename; } std::ofstream os{std::string(filename)}; if (os) { - for (absl::string_view v : set_to_check) { - os << v << "\n"; + for (const auto& pair : set_to_check) { + os << pair.first << " # " << pair.second.failure_message() << "\n"; } } else { - absl::StrAppendFormat(output, "Failed to open file: %s\n", filename); + absl::StrAppendFormat(output, + "Failed to open file: %s\n", + filename); } } @@ -265,47 +347,72 @@ ConformanceResponse ConformanceTestSuite::TruncateResponse( return debug_response; } -void ConformanceTestSuite::ReportSuccess(const std::string& test_name) { - if (expected_to_fail_.erase(test_name) != 0) { +void ConformanceTestSuite::ReportSuccess(const TestStatus& test) { + if (expected_to_fail_.erase(test.name()) != 0) { absl::StrAppendFormat( &output_, "ERROR: test %s is in the failure list, but test succeeded. " "Remove it from the failure list.\n", - test_name); - unexpected_succeeding_tests_.insert(test_name); + test.name()); + unexpected_succeeding_tests_[test.name()] = test; } successes_++; } -void ConformanceTestSuite::ReportFailure(const std::string& test_name, +void ConformanceTestSuite::ReportFailure(TestStatus& test, ConformanceLevel level, const ConformanceRequest& request, - const ConformanceResponse& response, - absl::string_view message) { - if (expected_to_fail_.erase(test_name) == 1) { - expected_failures_++; + const ConformanceResponse& response) { + if (expected_to_fail_.contains(test.name())) { + // Make copy just this once, as we need to modify them for comparison. + // Failure message from the failure list. + string expected_failure_message = + expected_to_fail_[test.name()].failure_message(); + // Actual failure message from the test run. + std::string actual_failure_message = test.failure_message(); + + Normalize(actual_failure_message); + if (actual_failure_message.rfind(expected_failure_message, 0) == 0) { + // Our failure messages match. + expected_failures_++; + } else { + // We want to add the test to the failure list with its correct failure + // message. + unexpected_failure_messages_[test.name()] = FormatFailureMessage(test); + // We want to remove the test from the failure list. That means passing + // to it the same failure message that was in the list. + TestStatus incorrect_failure_message; + incorrect_failure_message.set_name(test.name()); + incorrect_failure_message.set_failure_message( + expected_to_fail_[test.name()].failure_message()); + + expected_failure_messages_[test.name()] = incorrect_failure_message; + } + expected_to_fail_.erase(test.name()); if (!verbose_) return; } else if (level == RECOMMENDED && !enforce_recommended_) { - absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test_name); + absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test.name()); } else { - absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test_name); - unexpected_failing_tests_.insert(test_name); + absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test.name()); + + unexpected_failing_tests_[test.name()] = FormatFailureMessage(test); } - absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", message, + absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", + test.failure_message(), TruncateRequest(request).ShortDebugString(), TruncateResponse(response).ShortDebugString()); } -void ConformanceTestSuite::ReportSkip(const std::string& test_name, +void ConformanceTestSuite::ReportSkip(const TestStatus& test, const ConformanceRequest& request, const ConformanceResponse& response) { if (verbose_) { absl::StrAppendFormat( - &output_, "SKIPPED, test=%s request=%s, response=%s\n", test_name, + &output_, "SKIPPED, test=%s request=%s, response=%s\n", test.name(), request.ShortDebugString(), response.ShortDebugString()); } - skipped_.insert(test_name); + skipped_[test.name()] = test; } void ConformanceTestSuite::RunValidInputTest( @@ -326,7 +433,10 @@ void ConformanceTestSuite::RunValidBinaryInputTest( const std::string& equivalent_wire_format, bool require_same_wire_format) { const ConformanceRequest& request = setting.GetRequest(); ConformanceResponse response; - RunTest(setting.GetTestName(), request, &response); + if (!RunTest(setting.GetTestName(), request, &response)) { + return; + } + VerifyResponse(setting, equivalent_wire_format, response, true, require_same_wire_format); } @@ -345,22 +455,26 @@ void ConformanceTestSuite::VerifyResponse( ABSL_CHECK(reference_message->ParseFromString(equivalent_wire_format)) << "Failed to parse wire data for test case: " << test_name; + TestStatus test; + test.set_name(test_name); + switch (response.result_case()) { case ConformanceResponse::RESULT_NOT_SET: - ReportFailure(test_name, level, request, response, - "Response didn't have any field in the Response."); + test.set_failure_message( + "Response didn't have any field in the Response."); + ReportFailure(test, level, request, response); return; case ConformanceResponse::kParseError: case ConformanceResponse::kTimeoutError: case ConformanceResponse::kRuntimeError: case ConformanceResponse::kSerializeError: - ReportFailure(test_name, level, request, response, - "Failed to parse input or produce output."); + test.set_failure_message("Failed to parse input or produce output."); + ReportFailure(test, level, request, response); return; case ConformanceResponse::kSkipped: - ReportSkip(test_name, request, response); + ReportSkip(test, request, response); return; default: @@ -386,20 +500,18 @@ void ConformanceTestSuite::VerifyResponse( } else { check = differencer.Compare(*reference_message, *test_message); } - if (check) { if (need_report_success) { - ReportSuccess(test_name); + ReportSuccess(test); } } else { - ReportFailure( - test_name, level, request, response, - absl::StrCat("Output was not equivalent to reference message: ", - differences)); + test.set_failure_message(absl::StrCat( + "Output was not equivalent to reference message: ", differences)); + ReportFailure(test, level, request, response); } } -void ConformanceTestSuite::RunTest(const std::string& test_name, +bool ConformanceTestSuite::RunTest(const std::string& test_name, const ConformanceRequest& request, ConformanceResponse* response) { if (test_names_.insert(test_name).second == false) { @@ -410,7 +522,43 @@ void ConformanceTestSuite::RunTest(const std::string& test_name, std::string serialized_response; request.SerializeToString(&serialized_request); - runner_->RunTest(test_name, serialized_request, &serialized_response); + uint32_t len = internal::little_endian::FromHost( + static_cast(serialized_request.size())); + + if (!debug_) { // Not in debug mode. Continue. + } else if (debug_test_names_->erase(test_name) == 1) { + std::string octal = ProduceOctalSerialized(serialized_request, len); + std::string full_filename = WriteToFile(octal, output_dir_, test_name); + if (!full_filename.empty()) { + absl::StrAppendFormat( + &output_, "Produced octal serialized request file for test %s\n", + test_name); + absl::StrAppendFormat( + &output_, + " To pipe the " + "serialized request directly to " + "the " + "testee run from the root of your workspace:\n printf $(" + "<\"%s\") | " + "./bazel-bin/google/protobuf/conformance/%s\n\n", + full_filename, testee_); + absl::StrAppendFormat( + &output_, + " To inspect the wire format of the serialized request run " + "(Disclaimer: This may not work properly on non-Linux platforms):\n " + " " + "contents=$(<\"%s\"); sub=$(cut -d \\\\ -f 6- <<< " + "$contents) ; printf \"\\\\${sub}\" | protoscope \n\n\n", + full_filename); + } + } else { // Test is not ran, as it was not asked to be debugged. + expected_to_fail_.erase(test_name); + return false; + } + + response->set_protobuf_payload(serialized_request); + + runner_->RunTest(test_name, len, serialized_request, &serialized_response); if (!response->ParseFromString(serialized_response)) { response->Clear(); @@ -423,6 +571,7 @@ void ConformanceTestSuite::RunTest(const std::string& test_name, test_name, TruncateRequest(request).ShortDebugString(), TruncateResponse(*response).ShortDebugString()); } + return true; } std::string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) { @@ -443,8 +592,8 @@ std::string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) { return ""; } -void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) { - expected_to_fail_.insert(test_name); +void ConformanceTestSuite::AddExpectedFailedTest(const TestStatus& failure) { + expected_to_fail_[failure.name()] = failure; } bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, @@ -459,46 +608,89 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, unexpected_failing_tests_.clear(); unexpected_succeeding_tests_.clear(); - output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n"; + std::string mode = debug_ ? "DEBUG" : "TEST"; + absl::StrAppendFormat( + &output_, "CONFORMANCE %s BEGIN ====================================\n\n", + mode); failure_list_filename_ = filename; expected_to_fail_.clear(); - for (const std::string& failure : failure_list->failure()) { + for (const TestStatus& failure : failure_list->test()) { AddExpectedFailedTest(failure); } RunSuiteImpl(); + if (*output_dir_.rbegin() != '/') { + output_dir_.push_back('/'); + } + bool ok = true; if (!CheckSetEmpty( expected_to_fail_, "nonexistent_tests.txt", - absl::StrCat("These tests were listed in the failure list, but they " - "don't exist. Remove them from the failure list by " - "running:\n" - " ./update_failure_list.py ", - failure_list_filename_, - " --remove nonexistent_tests.txt"), + absl::StrCat( + "These tests were listed in the failure list, but they " + "don't exist. Remove them from the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --remove ", output_dir_, + "nonexistent_tests.txt"), + output_dir_, &output_)) { + ok = false; + } + if (!CheckSetEmpty( + expected_failure_messages_, "expected_failure_messages.txt", + absl::StrCat( + "These tests were listed in the failure list, but their failure " + "messages do not match. Remove them from the failure list " + "by running:\n" + " bazel run ", + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --remove ", output_dir_, + "expected_failure_messages.txt"), output_dir_, &output_)) { ok = false; } + if (!CheckSetEmpty( + unexpected_failure_messages_, "unexpected_failure_messages.txt", + absl::StrCat( + "These tests failed because their failure messages did " + "not match. If they can't be fixed right now, " + "you can add them to the failure list so the overall " + "suite can succeed. Add them to the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --add ", output_dir_, + "unexpected_failure_messages.txt"), + output_dir_, &output_)) { + ok = false; + } + if (!CheckSetEmpty( unexpected_failing_tests_, "failing_tests.txt", - absl::StrCat("These tests failed. If they can't be fixed right now, " - "you can add them to the failure list so the overall " - "suite can succeed. Add them to the failure list by " - "running:\n" - " ./update_failure_list.py ", - failure_list_filename_, " --add failing_tests.txt"), + absl::StrCat( + "These tests failed. If they can't be fixed right now, " + "you can add them to the failure list so the overall " + "suite can succeed. Add them to the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --add ", output_dir_, + "failing_tests.txt"), output_dir_, &output_)) { ok = false; } if (!CheckSetEmpty( unexpected_succeeding_tests_, "succeeding_tests.txt", - absl::StrCat("These tests succeeded, even though they were listed in " - "the failure list. Remove them from the failure list " - "by running:\n" - " ./update_failure_list.py ", - failure_list_filename_, - " --remove succeeding_tests.txt"), + absl::StrCat( + "These tests succeeded, even though they were listed in " + "the failure list. Remove them from the failure list by running " + "from the root of your workspace:\n" + " bazel run " + "//google/protobuf/conformance:update_failure_list -- ", + failure_list_filename_, " --remove ", output_dir_, + "succeeding_tests.txt"), output_dir_, &output_)) { ok = false; } diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index c78f9ea8ab..d10c9836a9 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -15,18 +15,17 @@ #define CONFORMANCE_CONFORMANCE_TEST_H #include +#include #include #include #include #include "google/protobuf/descriptor.pb.h" -#include "google/protobuf/util/type_resolver.h" -#include "absl/container/btree_set.h" +#include "absl/container/btree_map.h" #include "absl/container/flat_hash_set.h" -#include "absl/strings/string_view.h" +#include "conformance/conformance.pb.h" #include "conformance/conformance.pb.h" #include "google/protobuf/descriptor.h" -#include "google/protobuf/wire_format_lite.h" namespace conformance { class ConformanceRequest; @@ -46,17 +45,18 @@ class ConformanceTestSuite; class ConformanceTestRunner { public: - virtual ~ConformanceTestRunner() {} + virtual ~ConformanceTestRunner() = default; // Call to run a single conformance test. // + // "len" is the byte length of a serialized conformance.ConformanceRequest. // "input" is a serialized conformance.ConformanceRequest. // "output" should be set to a serialized conformance.ConformanceResponse. // // If there is any error in running the test itself, set "runtime_error" in // the response. - virtual void RunTest(const std::string& test_name, const std::string& input, - std::string* output) = 0; + virtual void RunTest(const std::string& test_name, uint32_t len, + const std::string& input, std::string* output) = 0; }; // Test runner that spawns the process being tested and communicates with it @@ -78,10 +78,10 @@ class ForkPipeRunner : public ConformanceTestRunner { explicit ForkPipeRunner(const std::string& executable) : child_pid_(-1), executable_(executable) {} - virtual ~ForkPipeRunner() {} + ~ForkPipeRunner() override = default; - void RunTest(const std::string& test_name, const std::string& request, - std::string* response); + void RunTest(const std::string& test_name, uint32_t len, + const std::string& request, std::string* response) override; private: void SpawnTestProgram(); @@ -129,12 +129,15 @@ class ForkPipeRunner : public ConformanceTestRunner { class ConformanceTestSuite { public: ConformanceTestSuite() - : verbose_(false), + : testee_(""), + verbose_(false), performance_(false), enforce_recommended_(false), maximum_edition_(Edition::EDITION_PROTO3), - failure_list_flag_name_("--failure_list") {} - virtual ~ConformanceTestSuite() {} + failure_list_flag_name_("--failure_list"), + debug_test_names_(nullptr), + debug_(false) {} + virtual ~ConformanceTestSuite() = default; void SetPerformance(bool performance) { performance_ = performance; } void SetVerbose(bool verbose) { verbose_ = verbose; } @@ -161,7 +164,18 @@ class ConformanceTestSuite { } // Sets the path of the output directory. - void SetOutputDir(const char* output_dir) { output_dir_ = output_dir; } + void SetOutputDir(const std::string& output_dir) { output_dir_ = output_dir; } + + // Sets if we are running the test in debug mode. + void SetDebug(bool debug) { debug_ = debug; } + + // Sets the testee name + void SetTestee(const std::string& testee) { testee_ = testee; } + + // Sets the debug test names + void SetDebugTestNames(absl::flat_hash_set& debug_test_names) { + debug_test_names_ = &debug_test_names; + } // Run all the conformance tests against the given test runner. // Test output will be stored in "output". @@ -170,6 +184,9 @@ class ConformanceTestSuite { // failure list. // The filename here is *only* used to create/format useful error messages for // how to update the failure list. We do NOT read this file at all. + + // "debug_test_names" holds the list of test names that the user requested to + // debug. If this is empty, we will run all the tests. bool RunSuite(ConformanceTestRunner* runner, std::string* output, const std::string& filename, conformance::FailureSet* failure_list); @@ -201,7 +218,7 @@ class ConformanceTestSuite { const Message& prototype_message, const std::string& test_name, const std::string& input); - virtual ~ConformanceRequestSetting() {} + virtual ~ConformanceRequestSetting() = default; std::unique_ptr NewTestMessage() const; @@ -259,12 +276,11 @@ class ConformanceTestSuite { conformance::ConformanceResponse TruncateResponse( const conformance::ConformanceResponse& response); - void ReportSuccess(const std::string& test_name); - void ReportFailure(const std::string& test_name, ConformanceLevel level, + void ReportSuccess(const conformance::TestStatus& test); + void ReportFailure(conformance::TestStatus& test, ConformanceLevel level, const conformance::ConformanceRequest& request, - const conformance::ConformanceResponse& response, - absl::string_view message); - void ReportSkip(const std::string& test_name, + const conformance::ConformanceResponse& response); + void ReportSkip(const conformance::TestStatus& test, const conformance::ConformanceRequest& request, const conformance::ConformanceResponse& response); @@ -274,15 +290,17 @@ class ConformanceTestSuite { const std::string& equivalent_wire_format, bool require_same_wire_format = false); - void RunTest(const std::string& test_name, + // Returns true if our runner_ ran the test and false if it did not. + bool RunTest(const std::string& test_name, const conformance::ConformanceRequest& request, conformance::ConformanceResponse* response); - void AddExpectedFailedTest(const std::string& test_name); + void AddExpectedFailedTest(const conformance::TestStatus& failure); virtual void RunSuiteImpl() = 0; ConformanceTestRunner* runner_; + std::string testee_; int successes_; int expected_failures_; bool verbose_; @@ -293,23 +311,41 @@ class ConformanceTestSuite { std::string output_dir_; std::string failure_list_flag_name_; std::string failure_list_filename_; + absl::flat_hash_set* debug_test_names_; + bool debug_; // The set of test names that are expected to fail in this run, but haven't // failed yet. - absl::btree_set expected_to_fail_; + absl::btree_map expected_to_fail_; + + // The set of tests that failed because their failure message did not match + // the actual failure message. These are failure messages that may need to be + // removed from our failure lists. + absl::btree_map + expected_failure_messages_; // The set of test names that have been run. Used to ensure that there are no // duplicate names in the suite. absl::flat_hash_set test_names_; - // The set of tests that failed, but weren't expected to. - absl::btree_set unexpected_failing_tests_; + // The set of tests that failed, but weren't expected to: They weren't + // present in our failure lists. + absl::btree_map + unexpected_failing_tests_; + + // The set of tests that succeeded, but weren't expected to: They were present + // in our failure lists, but managed to succeed. + absl::btree_map + unexpected_succeeding_tests_; - // The set of tests that succeeded, but weren't expected to. - absl::btree_set unexpected_succeeding_tests_; + // The set of tests that failed because their failure message did not match + // the actual failure message. These are failure messages that may need to be + // added to our failure lists. + absl::btree_map + unexpected_failure_messages_; // The set of tests that the testee opted out of; - absl::btree_set skipped_; + absl::btree_map skipped_; }; } // namespace protobuf diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index 35c33c63d7..3fbd9e2d44 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -32,26 +32,32 @@ #include #include +#include #include #include #include #include +#include +#include #include #include #include #include #include +#include #include +#include "absl/container/flat_hash_set.h" #include "absl/log/absl_log.h" +#include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "conformance/conformance.pb.h" +#include "conformance/conformance.pb.h" #include "conformance_test.h" #include "google/protobuf/endian.h" -using conformance::ConformanceResponse; using google::protobuf::ConformanceTestSuite; using std::string; using std::vector; @@ -76,15 +82,34 @@ void ParseFailureList(const char *filename, exit(1); } - for (string line; getline(infile, line);) { - // Remove whitespace. - line.erase(std::remove_if(line.begin(), line.end(), ::isspace), line.end()); - + for (string line; std::getline(infile, line);) { // Remove comments. - line = line.substr(0, line.find("#")); + string test_name = line.substr(0, line.find('#')); + + test_name.erase( + std::remove_if(test_name.begin(), test_name.end(), ::isspace), + test_name.end()); - if (!line.empty()) { - failure_list->add_failure(line); + if (test_name.empty()) { // Skip empty lines. + continue; + } + + // If we remove whitespace from the beginning of a line, and what we have + // left at first is a '#', then we have a comment. + if (test_name[0] != '#') { + // Find our failure message if it exists. Will be set to an empty string + // if no message is found. Empty failure messages also pass our tests. + size_t check_message = line.find('#'); + string message; + if (check_message != std::string::npos) { + message = line.substr(check_message + 1); // +1 to skip the delimiter + // If we had only whitespace after the delimiter, we will have an empty + // failure message and the test will still pass. + message = std::string(absl::StripAsciiWhitespace(message)); + } + conformance::TestStatus *test = failure_list->add_test(); + test->set_name(test_name); + test->set_failure_message(message); } } } @@ -100,7 +125,7 @@ void UsageError() { fprintf(stderr, " should contain one test name per\n"); fprintf(stderr, - " line. Use '#' for comments.\n"); + " line. Use '#' for comments.\n\n"); fprintf(stderr, " --text_format_failure_list Use to specify list \n"); fprintf(stderr, @@ -111,7 +136,7 @@ void UsageError() { fprintf(stderr, " File should contain one test name \n"); fprintf(stderr, - " per line. Use '#' for comments.\n"); + " per line. Use '#' for comments.\n\n"); fprintf(stderr, " --enforce_recommended Enforce that recommended test\n"); @@ -121,19 +146,30 @@ void UsageError() { " this flag if you want to be\n"); fprintf(stderr, " strictly conforming to protobuf\n"); - fprintf(stderr, " spec.\n"); + fprintf(stderr, " spec.\n\n"); fprintf(stderr, - " --maximum_edition Only run conformance tests up\n"); + " --maximum_edition Only run conformance tests up to\n"); fprintf(stderr, - " to and including the specified\n"); - fprintf(stderr, " edition.\n"); + " and including the specified\n"); + fprintf(stderr, " edition.\n\n"); fprintf(stderr, " --output_dir Directory to write\n" - " output files.\n"); + " output files.\n\n"); + fprintf(stderr, + " --debug ... Debug the \n"); + fprintf(stderr, " specified tests by running\n"); + fprintf(stderr, + " them in isolation and producing\n"); + fprintf(stderr, + " serialized request data for piping\n"); + fprintf(stderr, " directly to the testee.\n\n"); + fprintf(stderr, " --performance Boolean option\n"); + fprintf(stderr, " for enabling run of\n"); + fprintf(stderr, " performance tests.\n"); exit(1); } -void ForkPipeRunner::RunTest(const std::string &test_name, +void ForkPipeRunner::RunTest(const std::string &test_name, uint32_t len, const std::string &request, std::string *response) { if (child_pid_ < 0) { @@ -141,8 +177,6 @@ void ForkPipeRunner::RunTest(const std::string &test_name, } current_test_name_ = test_name; - uint32_t len = - internal::little_endian::FromHost(static_cast(request.size())); CheckedWrite(write_fd_, &len, sizeof(uint32_t)); CheckedWrite(write_fd_, request.c_str(), request.size()); @@ -188,57 +222,97 @@ int ForkPipeRunner::Run(int argc, char *argv[], fprintf(stderr, "No test suites found.\n"); return EXIT_FAILURE; } + + string program; + string testee; + std::vector program_args; + bool performance = false; + bool debug = false; + absl::flat_hash_set debug_test_names; + bool enforce_recommended = false; + Edition maximum_edition = EDITION_UNKNOWN; + std::string output_dir; + bool verbose = false; + + for (int arg = 1; arg < argc; ++arg) { + if (strcmp(argv[arg], "--performance") == 0) { + performance = true; + } else if (strcmp(argv[arg], "--verbose") == 0) { + verbose = true; + } else if (strcmp(argv[arg], "--enforce_recommended") == 0) { + enforce_recommended = true; + } else if (strcmp(argv[arg], "--maximum_edition") == 0) { + if (++arg == argc) UsageError(); + Edition edition = EDITION_UNKNOWN; + if (!Edition_Parse(absl::StrCat("EDITION_", argv[arg]), &edition)) { + fprintf(stderr, "Unknown edition: %s\n", argv[arg]); + UsageError(); + } + maximum_edition = edition; + } else if (strcmp(argv[arg], "--output_dir") == 0) { + if (++arg == argc) UsageError(); + output_dir = argv[arg]; + + } else if (strcmp(argv[arg], "--debug") == 0) { + if (++arg == argc) UsageError(); + for (int debug_arg = arg; debug_arg < argc; ++debug_arg) { + // Stop when we either find another flag or we reach the last arg + // (program arg) + if (argv[debug_arg][0] == '-' || debug_arg == argc - 1) { + arg = debug_arg - 1; + break; + } + debug_test_names.insert(argv[debug_arg]); + } + + } else if (argv[arg][0] == '-') { + bool recognized_flag = false; + for (ConformanceTestSuite *suite : suites) { + if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { + if (++arg == argc) UsageError(); + recognized_flag = true; + } + } + if (!recognized_flag) { + fprintf(stderr, "Unknown option: %s\n", argv[arg]); + UsageError(); + } + } else { + program += argv[arg++]; + while (arg < argc) { + program_args.push_back(argv[arg]); + arg++; + } + } + } + + if (!debug_test_names.empty()) { + debug = true; + } + auto last_slash = program.find_last_of('/'); + if (last_slash != string::npos) { + testee = program.substr(last_slash + 1); + } + bool all_ok = true; for (ConformanceTestSuite *suite : suites) { - string program; - std::vector program_args; string failure_list_filename; conformance::FailureSet failure_list; - - bool performance = false; for (int arg = 1; arg < argc; ++arg) { if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { if (++arg == argc) UsageError(); failure_list_filename = argv[arg]; ParseFailureList(argv[arg], &failure_list); - } else if (strcmp(argv[arg], "--performance") == 0) { - performance = true; - suite->SetPerformance(true); - } else if (strcmp(argv[arg], "--verbose") == 0) { - suite->SetVerbose(true); - } else if (strcmp(argv[arg], "--enforce_recommended") == 0) { - suite->SetEnforceRecommended(true); - } else if (strcmp(argv[arg], "--maximum_edition") == 0) { - if (++arg == argc) UsageError(); - Edition edition = EDITION_UNKNOWN; - if (!Edition_Parse(absl::StrCat("EDITION_", argv[arg]), &edition)) { - fprintf(stderr, "Unknown edition: %s\n", argv[arg]); - UsageError(); - } - suite->SetMaximumEdition(edition); - } else if (strcmp(argv[arg], "--output_dir") == 0) { - if (++arg == argc) UsageError(); - suite->SetOutputDir(argv[arg]); - } else if (argv[arg][0] == '-') { - bool recognized_flag = false; - for (ConformanceTestSuite *suite : suites) { - if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { - if (++arg == argc) UsageError(); - recognized_flag = true; - } - } - if (!recognized_flag) { - fprintf(stderr, "Unknown option: %s\n", argv[arg]); - UsageError(); - } - } else { - program += argv[arg++]; - while (arg < argc) { - program_args.push_back(argv[arg]); - arg++; - } } } + suite->SetPerformance(performance); + suite->SetVerbose(verbose); + suite->SetEnforceRecommended(enforce_recommended); + suite->SetMaximumEdition(maximum_edition); + suite->SetOutputDir(output_dir); + suite->SetDebug(debug); + suite->SetDebugTestNames(debug_test_names); + suite->SetTestee(testee); ForkPipeRunner runner(program, program_args, performance); @@ -248,6 +322,16 @@ int ForkPipeRunner::Run(int argc, char *argv[], fwrite(output.c_str(), 1, output.size(), stderr); } + + if (!debug_test_names.empty()) { + fprintf(stderr, + "These tests were requested to be debugged, but they do " + "not exist. Revise the test names:\n\n"); + for (const string &test_name : debug_test_names) { + fprintf(stderr, " %s\n", test_name.c_str()); + } + fprintf(stderr, "\n\n"); + } return all_ok ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index 0f4e10ff63..d7ffba7274 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt @@ -7,105 +7,105 @@ # TODO: insert links to corresponding bugs tracking the issue. # Should we use GitHub issues or the Google-internal bug tracker? -Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Proto3.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Proto3.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted -Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput -Recommended.Proto3.JsonInput.MapFieldValueIsNull -Recommended.Editions_Proto3.JsonInput.MapFieldValueIsNull -Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Editions_Proto3.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Editions_Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Proto3.JsonInput.RepeatedFieldTrailingComma -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingComma -Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto3.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Editions_Proto3.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Proto3.JsonInput.TrailingCommaInAnObject -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObject -Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace -Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace -Recommended.Proto2.JsonInput.FieldNameExtension.Validator -Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Editions_Proto2.JsonInput.MapFieldValueIsNull -Recommended.Editions_Proto2.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Editions_Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingComma -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto2.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObject -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto2.JsonInput.FieldNameDuplicate -Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 -Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 -Recommended.Proto2.JsonInput.FieldNameNotQuoted -Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput -Recommended.Proto2.JsonInput.MapFieldValueIsNull -Recommended.Proto2.JsonInput.RepeatedFieldMessageElementIsNull -Recommended.Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull -Recommended.Proto2.JsonInput.RepeatedFieldTrailingComma -Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines -Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace -Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace -Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto2.JsonInput.StringFieldUppercaseEscapeLetter -Recommended.Proto2.JsonInput.TrailingCommaInAnObject -Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines -Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpace -Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Editions_Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Editions_Proto2.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Editions_Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Editions_Proto3.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Proto2.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key2]: FOO +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput # Output was not equivalent to reference message: added: map_string_nested_enum[key]: FOO +Recommended.Proto3.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt index ef705ef1b8..d8f44fde1f 100644 --- a/conformance/failure_list_java.txt +++ b/conformance/failure_list_java.txt @@ -4,149 +4,149 @@ # By listing them here we can keep tabs on which ones are failing and be sure # that we don't introduce regressions in other tests. -Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Proto3.JsonInput.BoolFieldAllCapitalFalse -Recommended.Proto3.JsonInput.BoolFieldAllCapitalTrue -Recommended.Proto3.JsonInput.BoolFieldCamelCaseFalse -Recommended.Proto3.JsonInput.BoolFieldCamelCaseTrue -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto3.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted -Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Proto3.JsonInput.FieldNameDuplicate -Recommended.Proto3.JsonInput.FieldNameNotQuoted -Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted -Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Proto3.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.JsonWithComments -Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto3.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.FieldNameExtension.Validator -Required.Proto3.JsonInput.EnumFieldNotQuoted -Required.Proto3.JsonInput.Int32FieldLeadingZero -Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Proto3.JsonInput.Int32FieldPlusSign -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Proto3.JsonInput.StringFieldNotAString -Recommended.Proto2.JsonInput.BoolFieldAllCapitalFalse -Recommended.Proto2.JsonInput.BoolFieldAllCapitalTrue -Recommended.Proto2.JsonInput.BoolFieldCamelCaseFalse -Recommended.Proto2.JsonInput.BoolFieldCamelCaseTrue -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Proto2.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Proto2.JsonInput.DoubleFieldNanNotQuoted -Recommended.Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Proto2.JsonInput.FieldNameDuplicate -Recommended.Proto2.JsonInput.FieldNameNotQuoted -Recommended.Proto2.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Proto2.JsonInput.FloatFieldNanNotQuoted -Recommended.Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Proto2.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.JsonWithComments -Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Proto2.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Proto2.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Proto2.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Proto2.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Proto2.JsonInput.Uint64MapFieldKeyNotQuoted -Required.Proto2.JsonInput.EnumFieldNotQuoted -Required.Proto2.JsonInput.Int32FieldLeadingZero -Required.Proto2.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Proto2.JsonInput.Int32FieldPlusSign -Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Proto2.JsonInput.StringFieldNotAString -Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput -Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput -Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalTrue -Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseTrue -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto3.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.DoubleFieldNanNotQuoted -Recommended.Editions_Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto3.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.FloatFieldNanNotQuoted -Recommended.Editions_Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto3.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.JsonWithComments -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto3.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Editions_Proto3.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Editions_Proto3.JsonInput.Uint64MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator -Required.Editions_Proto3.JsonInput.EnumFieldNotQuoted -Required.Editions_Proto3.JsonInput.Int32FieldLeadingZero -Required.Editions_Proto3.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Editions_Proto3.JsonInput.Int32FieldPlusSign -Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Editions_Proto3.JsonInput.StringFieldNotAString -Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalTrue -Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseTrue -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse -Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue -Recommended.Editions_Proto2.JsonInput.BoolMapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.DoubleFieldInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.DoubleFieldNanNotQuoted -Recommended.Editions_Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate -Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted -Recommended.Editions_Proto2.JsonInput.FloatFieldInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.FloatFieldNanNotQuoted -Recommended.Editions_Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted -Recommended.Editions_Proto2.JsonInput.Int32MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.Int64MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.JsonWithComments -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey -Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue -Recommended.Editions_Proto2.JsonInput.StringFieldSurrogateInWrongOrder -Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedHighSurrogate -Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedLowSurrogate -Recommended.Editions_Proto2.JsonInput.Uint32MapFieldKeyNotQuoted -Recommended.Editions_Proto2.JsonInput.Uint64MapFieldKeyNotQuoted -Required.Editions_Proto2.JsonInput.EnumFieldNotQuoted -Required.Editions_Proto2.JsonInput.Int32FieldLeadingZero -Required.Editions_Proto2.JsonInput.Int32FieldNegativeWithLeadingZero -Required.Editions_Proto2.JsonInput.Int32FieldPlusSign -Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool -Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt -Required.Editions_Proto2.JsonInput.StringFieldNotAString -Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput +Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameExtension.Validator # Expected JSON payload but got type 1 +Recommended.Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.JsonInput.BoolFieldAllCapitalFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldAllCapitalTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldCamelCaseFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldCamelCaseTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolMapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FloatFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Int32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Int64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.JsonWithComments # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSurrogateInWrongOrder # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Uint32MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.Uint64MapFieldKeyNotQuoted # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Editions_Proto3.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto2.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto3.JsonInput.EnumFieldNotQuoted # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.Int32FieldLeadingZero # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.Int32FieldNegativeWithLeadingZero # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.Int32FieldPlusSign # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.StringFieldNotAString # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch \ No newline at end of file diff --git a/conformance/failure_list_java_lite.txt b/conformance/failure_list_java_lite.txt index 84606a5173..87a9758cba 100644 --- a/conformance/failure_list_java_lite.txt +++ b/conformance/failure_list_java_lite.txt @@ -4,11 +4,11 @@ # By listing them here we can keep tabs on which ones are failing and be sure # that we don't introduce regressions in other tests. -Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE -Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE \ No newline at end of file +Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Editions_Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE # Should have failed to parse, but didn't. \ No newline at end of file diff --git a/conformance/failure_list_objc.txt b/conformance/failure_list_objc.txt index 463c3606b9..0dd4279587 100644 --- a/conformance/failure_list_objc.txt +++ b/conformance/failure_list_objc.txt @@ -1,7 +1,7 @@ # JSON input or output tests are skipped (in conformance_objc.m) as mobile # platforms don't support JSON wire format to avoid code bloat. -Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput -Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput +Required.Editions_Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Editions_Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto2.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch +Required.Proto3.ProtobufInput.UnknownOrdering.ProtobufOutput # Unknown field mismatch diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index a230fa3d08..508206f5eb 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -26,6 +26,7 @@ using conformance::ConformanceRequest; using conformance::ConformanceResponse; +using conformance::TestStatus; using conformance::WireFormat; using protobuf_test_messages::editions::TestAllTypesEdition2023; using protobuf_test_messages::proto2::TestAllTypesProto2; @@ -73,19 +74,22 @@ bool TextFormatConformanceTestSuite::ParseResponse( const std::string& test_name = setting.GetTestName(); ConformanceLevel level = setting.GetLevel(); + TestStatus test; + test.set_name(test_name); switch (response.result_case()) { case ConformanceResponse::kProtobufPayload: { if (requested_output != conformance::PROTOBUF) { - ReportFailure(test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided PROTOBUF instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided PROTOBUF instead.")); + ReportFailure(test, level, request, response); return false; } if (!test_message->ParseFromString(response.protobuf_payload())) { - ReportFailure(test_name, level, request, response, - "Protobuf output we received from test was unparseable."); + test.set_failure_message( + "Protobuf output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -94,18 +98,17 @@ bool TextFormatConformanceTestSuite::ParseResponse( case ConformanceResponse::kTextPayload: { if (requested_output != conformance::TEXT_FORMAT) { - ReportFailure( - test_name, level, request, response, - absl::StrCat("Test was asked for ", - WireFormatToString(requested_output), - " output but provided TEXT_FORMAT instead.")); + test.set_failure_message(absl::StrCat( + "Test was asked for ", WireFormatToString(requested_output), + " output but provided TEXT_FORMAT instead.")); + ReportFailure(test, level, request, response); return false; } if (!ParseTextFormatResponse(response, setting, test_message)) { - ReportFailure( - test_name, level, request, response, + test.set_failure_message( "TEXT_FORMAT output we received from test was unparseable."); + ReportFailure(test, level, request, response); return false; } @@ -172,14 +175,19 @@ void TextFormatConformanceTestSuiteImpl::ExpectParseFailure( setting.ConformanceLevelToString(level), ".", setting.GetSyntaxIdentifier(), ".TextFormatInput.", test_name); - suite_.RunTest(effective_test_name, request, &response); + if (!suite_.RunTest(effective_test_name, request, &response)) { + return; + } + + TestStatus test; + test.set_name(effective_test_name); if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(effective_test_name); + suite_.ReportSuccess(test); } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(effective_test_name, request, response); + suite_.ReportSkip(test, request, response); } else { - suite_.ReportFailure(effective_test_name, level, request, response, - "Should have failed to parse, but didn't."); + test.set_failure_message("Should have failed to parse, but didn't."); + suite_.ReportFailure(test, level, request, response); } } diff --git a/conformance/text_format_failure_list_cpp.txt b/conformance/text_format_failure_list_cpp.txt index fd2d7ada1f..d1d3cd2e76 100644 --- a/conformance/text_format_failure_list_cpp.txt +++ b/conformance/text_format_failure_list_cpp.txt @@ -1,40 +1,42 @@ -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes -Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes -Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString -Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString -Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes -Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes -Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString -Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString # Should have failed to parse, but didn't. + +# End up setting the high bit as a sign instead of failing to parse. diff --git a/conformance/text_format_failure_list_java.txt b/conformance/text_format_failure_list_java.txt index a035453944..d2611ccb10 100644 --- a/conformance/text_format_failure_list_java.txt +++ b/conformance/text_format_failure_list_java.txt @@ -1,20 +1,20 @@ -Recommended.Editions_Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Editions_Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput -Recommended.Editions_Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Editions_Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput -Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.AnyField.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.AnyField.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal -Required.Proto3.TextFormatInput.AnyField.ProtobufOutput -Required.Proto3.TextFormatInput.AnyField.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex -Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal +Recommended.Editions_Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Editions_Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Editions_Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Editions_Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Required.Editions_Proto3.TextFormatInput.AnyField.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.AnyField.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.AnyField.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.AnyField.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. diff --git a/conformance/text_format_failure_list_python.txt b/conformance/text_format_failure_list_python.txt index 48abf4e55b..346852001c 100644 --- a/conformance/text_format_failure_list_python.txt +++ b/conformance/text_format_failure_list_python.txt @@ -1,99 +1,101 @@ # This is the list of text format conformance tests that are known to fail right # now. # TODO: These should be fixed. -Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput -Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput -Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput -Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Editions_Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegativeZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_f.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Proto3.TextFormatInput.NegDoubleFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_double: -0 +Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.ProtobufOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.NegFloatFieldLargeNegativeExponentParsesAsNegZero.TextFormatOutput # Output was not equivalent to reference message: deleted: optional_float: -0 +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ + +# Optional float interpreted as `inf` +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput # Output was not equivalent to reference message +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput # Output was not equivalent to reference message +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput # Output was not equivalent to reference message +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.ProtobufOutput # Output was not equivalent to reference message +Required.Proto3.TextFormatInput.FloatFieldMaxValue_f.TextFormatOutput # Output was not equivalent to reference message diff --git a/conformance/text_format_failure_list_python_cpp.txt b/conformance/text_format_failure_list_python_cpp.txt index fc233ed286..d3c827d8d5 100644 --- a/conformance/text_format_failure_list_python_cpp.txt +++ b/conformance/text_format_failure_list_python_cpp.txt @@ -1,72 +1,72 @@ -Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ diff --git a/conformance/text_format_failure_list_python_upb.txt b/conformance/text_format_failure_list_python_upb.txt index 85da8a1660..476a557e27 100644 --- a/conformance/text_format_failure_list_python_upb.txt +++ b/conformance/text_format_failure_list_python_upb.txt @@ -1,75 +1,75 @@ # This is the list of text format conformance tests that are known to fail right # now. # TODO: These should be fixed. -Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput -Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal -Required.Proto3.TextFormatInput.FloatFieldNoOctal -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput -Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput -Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Editions_Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanInt64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldLargerThanUint64_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMaxValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldMinValue_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegativeZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNegative_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZeroWithExponent_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoLeadingZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldNoOctal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooLarge_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldTooSmall_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithInt32Max_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldWithVeryPreciseNumber_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatFieldZero_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.ProtobufOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.FloatField_F.TextFormatOutput # Failed to parse input or produce output. +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput # Output was not equivalent to reference message: modified: optional_bytes: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\t +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ +Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput # Output was not equivalent to reference message: modified: optional_string: "\007\010\014\n\r\t\013?\\\'\"" -> "\007\010\014\n\r\ diff --git a/conformance/update_failure_list.py b/conformance/update_failure_list.py index 639a8c26ed..917dba312b 100755 --- a/conformance/update_failure_list.py +++ b/conformance/update_failure_list.py @@ -8,6 +8,14 @@ """Script to update a failure list file to add/remove failures. +When adding, will attempt to place them in their correct lexicographical +position relative to other test names. This requires that the failure list is +already sorted. This does not guarantee that the tests will appear neatly one +after the other, as there may be comments in between. If the failure list +is not sorted, sorted blocks may be produced, but the list as a whole will not. +Lastly, be wary of lists that have tests stripped from OSS; if you catch that +a test was added inside a stripped block, you may need to move it out. + This is sort of like comm(1), except it recognizes comments and ignores them. """ @@ -19,32 +27,58 @@ parser.add_argument('filename', type=str, help='failure list file to update') parser.add_argument('--add', dest='add_list', action='append') parser.add_argument('--remove', dest='remove_list', action='append') +DEFAULT_ALIGNMENT = 114 args = parser.parse_args() add_set = set() remove_set = set() +# Adds test + failure message for add_file in (args.add_list or []): with open(add_file) as f: for line in f: add_set.add(line) +# We only remove by name for remove_file in (args.remove_list or []): with open(remove_file) as f: for line in f: if line in add_set: raise Exception("Asked to both add and remove test: " + line) - remove_set.add(line.strip()) + remove_set.add(line.split('#')[0].strip()) add_list = sorted(add_set, reverse=True) with open(args.filename) as in_file: - existing_list = in_file.read() + existing_list = in_file.read() -with open(args.filename, "w") as f: +with open(args.filename, 'w') as f: for line in existing_list.splitlines(True): - test = line.split("#")[0].strip() - while len(add_list) > 0 and test > add_list[-1]: + test = line.split('#')[0].strip() + # As long as the tests we are adding appear lexicographically before our + # read test, put them first followed by our read test. + while add_list and test > add_list[-1]: f.write(add_list.pop()) if test not in remove_set: f.write(line) + # Any remaining tests are added at the end + while add_list: + f.write(add_list.pop()) + +# Update our read of the existing file +with open(args.filename, 'r') as f: + existing_list = f.read() + +# Actual alignment of failure messages to 'DEFAULT_ALIGNMENT' +# If test name exceeds DEFAULT_ALIGNMENT, it cannot and will not be aligned. +with open(args.filename, 'w') as f: + for line in existing_list.splitlines(True): + split = line.split('#', 1) + test = split[0].strip() + if len(split) > 1 and test: + message = split[1].lstrip() + spaces = ' ' * (DEFAULT_ALIGNMENT - len(test)) + line = test + spaces + ' # ' + message + f.write(line) + else: # ignore blank lines/lines that do not have '#'/comments + f.write(line) diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs index f8a073b938..29b19592a7 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs @@ -25,32 +25,35 @@ namespace Conformance { byte[] descriptorData = global::System.Convert.FromBase64String( string.Concat( "Ch1jb25mb3JtYW5jZS9jb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2Ui", - "HQoKRmFpbHVyZVNldBIPCgdmYWlsdXJlGAEgAygJIuMCChJDb25mb3JtYW5j", - "ZVJlcXVlc3QSGgoQcHJvdG9idWZfcGF5bG9hZBgBIAEoDEgAEhYKDGpzb25f", - "cGF5bG9hZBgCIAEoCUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRl", - "eHRfcGF5bG9hZBgIIAEoCUgAEjgKF3JlcXVlc3RlZF9vdXRwdXRfZm9ybWF0", - "GAMgASgOMhcuY29uZm9ybWFuY2UuV2lyZUZvcm1hdBIUCgxtZXNzYWdlX3R5", - "cGUYBCABKAkSMAoNdGVzdF9jYXRlZ29yeRgFIAEoDjIZLmNvbmZvcm1hbmNl", - "LlRlc3RDYXRlZ29yeRI+ChVqc3BiX2VuY29kaW5nX29wdGlvbnMYBiABKAsy", - "Hy5jb25mb3JtYW5jZS5Kc3BiRW5jb2RpbmdDb25maWcSHAoUcHJpbnRfdW5r", - "bm93bl9maWVsZHMYCSABKAhCCQoHcGF5bG9hZCL6AQoTQ29uZm9ybWFuY2VS", - "ZXNwb25zZRIVCgtwYXJzZV9lcnJvchgBIAEoCUgAEhkKD3NlcmlhbGl6ZV9l", - "cnJvchgGIAEoCUgAEhcKDXRpbWVvdXRfZXJyb3IYCSABKAlIABIXCg1ydW50", - "aW1lX2Vycm9yGAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgA", - "EhYKDGpzb25fcGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIABIW", - "Cgxqc3BiX3BheWxvYWQYByABKAlIABIWCgx0ZXh0X3BheWxvYWQYCCABKAlI", - "AEIICgZyZXN1bHQiNwoSSnNwYkVuY29kaW5nQ29uZmlnEiEKGXVzZV9qc3Bi", - "X2FycmF5X2FueV9mb3JtYXQYASABKAgqUAoKV2lyZUZvcm1hdBIPCgtVTlNQ", - "RUNJRklFRBAAEgwKCFBST1RPQlVGEAESCAoESlNPThACEggKBEpTUEIQAxIP", - "CgtURVhUX0ZPUk1BVBAEKo8BCgxUZXN0Q2F0ZWdvcnkSFAoQVU5TUEVDSUZJ", - "RURfVEVTVBAAEg8KC0JJTkFSWV9URVNUEAESDQoJSlNPTl9URVNUEAISJAog", - "SlNPTl9JR05PUkVfVU5LTk9XTl9QQVJTSU5HX1RFU1QQAxINCglKU1BCX1RF", - "U1QQBBIUChBURVhUX0ZPUk1BVF9URVNUEAVCLwofY29tLmdvb2dsZS5wcm90", - "b2J1Zi5jb25mb3JtYW5jZaICC0NvbmZvcm1hbmNlYgZwcm90bzM=")); + "MwoKVGVzdFN0YXR1cxIMCgRuYW1lGAEgASgJEhcKD2ZhaWx1cmVfbWVzc2Fn", + "ZRgCIAEoCSI5CgpGYWlsdXJlU2V0EiUKBHRlc3QYAiADKAsyFy5jb25mb3Jt", + "YW5jZS5UZXN0U3RhdHVzSgQIARACIuMCChJDb25mb3JtYW5jZVJlcXVlc3QS", + "GgoQcHJvdG9idWZfcGF5bG9hZBgBIAEoDEgAEhYKDGpzb25fcGF5bG9hZBgC", + "IAEoCUgAEhYKDGpzcGJfcGF5bG9hZBgHIAEoCUgAEhYKDHRleHRfcGF5bG9h", + "ZBgIIAEoCUgAEjgKF3JlcXVlc3RlZF9vdXRwdXRfZm9ybWF0GAMgASgOMhcu", + "Y29uZm9ybWFuY2UuV2lyZUZvcm1hdBIUCgxtZXNzYWdlX3R5cGUYBCABKAkS", + "MAoNdGVzdF9jYXRlZ29yeRgFIAEoDjIZLmNvbmZvcm1hbmNlLlRlc3RDYXRl", + "Z29yeRI+ChVqc3BiX2VuY29kaW5nX29wdGlvbnMYBiABKAsyHy5jb25mb3Jt", + "YW5jZS5Kc3BiRW5jb2RpbmdDb25maWcSHAoUcHJpbnRfdW5rbm93bl9maWVs", + "ZHMYCSABKAhCCQoHcGF5bG9hZCL6AQoTQ29uZm9ybWFuY2VSZXNwb25zZRIV", + "CgtwYXJzZV9lcnJvchgBIAEoCUgAEhkKD3NlcmlhbGl6ZV9lcnJvchgGIAEo", + "CUgAEhcKDXRpbWVvdXRfZXJyb3IYCSABKAlIABIXCg1ydW50aW1lX2Vycm9y", + "GAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgAEhYKDGpzb25f", + "cGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIABIWCgxqc3BiX3Bh", + "eWxvYWQYByABKAlIABIWCgx0ZXh0X3BheWxvYWQYCCABKAlIAEIICgZyZXN1", + "bHQiNwoSSnNwYkVuY29kaW5nQ29uZmlnEiEKGXVzZV9qc3BiX2FycmF5X2Fu", + "eV9mb3JtYXQYASABKAgqUAoKV2lyZUZvcm1hdBIPCgtVTlNQRUNJRklFRBAA", + "EgwKCFBST1RPQlVGEAESCAoESlNPThACEggKBEpTUEIQAxIPCgtURVhUX0ZP", + "Uk1BVBAEKo8BCgxUZXN0Q2F0ZWdvcnkSFAoQVU5TUEVDSUZJRURfVEVTVBAA", + "Eg8KC0JJTkFSWV9URVNUEAESDQoJSlNPTl9URVNUEAISJAogSlNPTl9JR05P", + "UkVfVU5LTk9XTl9QQVJTSU5HX1RFU1QQAxINCglKU1BCX1RFU1QQBBIUChBU", + "RVhUX0ZPUk1BVF9URVNUEAVCLwofY29tLmdvb2dsZS5wcm90b2J1Zi5jb25m", + "b3JtYW5jZaICC0NvbmZvcm1hbmNlYgZwcm90bzM=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), typeof(global::Conformance.TestCategory), }, null, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.FailureSet), global::Conformance.FailureSet.Parser, new[]{ "Failure" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.TestStatus), global::Conformance.TestStatus.Parser, new[]{ "Name", "FailureMessage" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.FailureSet), global::Conformance.FailureSet.Parser, new[]{ "Test" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "JspbPayload", "TextPayload", "RequestedOutputFormat", "MessageType", "TestCategory", "JspbEncodingOptions", "PrintUnknownFields" }, new[]{ "Payload" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceResponse), global::Conformance.ConformanceResponse.Parser, new[]{ "ParseError", "SerializeError", "TimeoutError", "RuntimeError", "ProtobufPayload", "JsonPayload", "Skipped", "JspbPayload", "TextPayload" }, new[]{ "Result" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.JspbEncodingConfig), global::Conformance.JspbEncodingConfig.Parser, new[]{ "UseJspbArrayAnyFormat" }, null, null, null, null) @@ -104,6 +107,246 @@ namespace Conformance { #endregion #region Messages + ///

+ /// Meant to encapsulate all types of tests: successes, skips, failures, etc. + /// Therefore, this may or may not have a failure message. Failure messages + /// may be truncated for our failure lists. + /// + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class TestStatus : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new TestStatus()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestStatus() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestStatus(TestStatus other) : this() { + name_ = other.name_; + failureMessage_ = other.failureMessage_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public TestStatus Clone() { + return new TestStatus(this); + } + + /// Field number for the "name" field. + public const int NameFieldNumber = 1; + private string name_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Name { + get { return name_; } + set { + name_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "failure_message" field. + public const int FailureMessageFieldNumber = 2; + private string failureMessage_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string FailureMessage { + get { return failureMessage_; } + set { + failureMessage_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as TestStatus); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(TestStatus other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (Name != other.Name) return false; + if (FailureMessage != other.FailureMessage) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (Name.Length != 0) hash ^= Name.GetHashCode(); + if (FailureMessage.Length != 0) hash ^= FailureMessage.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (FailureMessage.Length != 0) { + output.WriteRawTag(18); + output.WriteString(FailureMessage); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (Name.Length != 0) { + output.WriteRawTag(10); + output.WriteString(Name); + } + if (FailureMessage.Length != 0) { + output.WriteRawTag(18); + output.WriteString(FailureMessage); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (Name.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Name); + } + if (FailureMessage.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(FailureMessage); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(TestStatus other) { + if (other == null) { + return; + } + if (other.Name.Length != 0) { + Name = other.Name; + } + if (other.FailureMessage.Length != 0) { + FailureMessage = other.FailureMessage; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + FailureMessage = input.ReadString(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + Name = input.ReadString(); + break; + } + case 18: { + FailureMessage = input.ReadString(); + break; + } + } + } + } + #endif + + } + /// /// The conformance runner will request a list of failures as the first request. /// This will be known by message_type == "conformance.FailureSet", a conformance @@ -124,7 +367,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[0]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[1]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -144,7 +387,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public FailureSet(FailureSet other) : this() { - failure_ = other.failure_.Clone(); + test_ = other.test_.Clone(); _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -154,15 +397,15 @@ namespace Conformance { return new FailureSet(this); } - /// Field number for the "failure" field. - public const int FailureFieldNumber = 1; - private static readonly pb::FieldCodec _repeated_failure_codec - = pb::FieldCodec.ForString(10); - private readonly pbc::RepeatedField failure_ = new pbc::RepeatedField(); + /// Field number for the "test" field. + public const int TestFieldNumber = 2; + private static readonly pb::FieldCodec _repeated_test_codec + = pb::FieldCodec.ForMessage(18, global::Conformance.TestStatus.Parser); + private readonly pbc::RepeatedField test_ = new pbc::RepeatedField(); [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public pbc::RepeatedField Failure { - get { return failure_; } + public pbc::RepeatedField Test { + get { return test_; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -180,7 +423,7 @@ namespace Conformance { if (ReferenceEquals(other, this)) { return true; } - if(!failure_.Equals(other.failure_)) return false; + if(!test_.Equals(other.test_)) return false; return Equals(_unknownFields, other._unknownFields); } @@ -188,7 +431,7 @@ namespace Conformance { [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override int GetHashCode() { int hash = 1; - hash ^= failure_.GetHashCode(); + hash ^= test_.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -207,7 +450,7 @@ namespace Conformance { #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE output.WriteRawMessage(this); #else - failure_.WriteTo(output, _repeated_failure_codec); + test_.WriteTo(output, _repeated_test_codec); if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -218,7 +461,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { - failure_.WriteTo(ref output, _repeated_failure_codec); + test_.WriteTo(ref output, _repeated_test_codec); if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -229,7 +472,7 @@ namespace Conformance { [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public int CalculateSize() { int size = 0; - size += failure_.CalculateSize(_repeated_failure_codec); + size += test_.CalculateSize(_repeated_test_codec); if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -242,7 +485,7 @@ namespace Conformance { if (other == null) { return; } - failure_.Add(other.failure_); + test_.Add(other.test_); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -262,8 +505,8 @@ namespace Conformance { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; - case 10: { - failure_.AddEntriesFrom(input, _repeated_failure_codec); + case 18: { + test_.AddEntriesFrom(input, _repeated_test_codec); break; } } @@ -285,8 +528,8 @@ namespace Conformance { default: _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; - case 10: { - failure_.AddEntriesFrom(ref input, _repeated_failure_codec); + case 18: { + test_.AddEntriesFrom(ref input, _repeated_test_codec); break; } } @@ -318,7 +561,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[1]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[2]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -945,7 +1188,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[2]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[3]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -1665,7 +1908,7 @@ namespace Conformance { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public static pbr::MessageDescriptor Descriptor { - get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[3]; } + get { return global::Conformance.ConformanceReflection.Descriptor.MessageTypes[4]; } } [global::System.Diagnostics.DebuggerNonUserCodeAttribute] diff --git a/editions/generated_files_test.cc b/editions/generated_files_test.cc index 7a9b261391..22aaa18f8c 100644 --- a/editions/generated_files_test.cc +++ b/editions/generated_files_test.cc @@ -146,7 +146,11 @@ TEST(Generated, EditionDefaults2023InternalFeatures) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING } + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + } )pb")); } diff --git a/editions/golden/compare_cpp_codegen_failure.txt b/editions/golden/compare_cpp_codegen_failure.txt index ff5fbdaffc..ab0bad33f1 100644 --- a/editions/golden/compare_cpp_codegen_failure.txt +++ b/editions/golden/compare_cpp_codegen_failure.txt @@ -30,9 +30,9 @@ { - // optional int32 int32_field = 1; + // int32 int32_field = 1; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { + total_size += ::_pbi::WireFormatLite::Int32SizePlusOne( [ FAILED ] third_party/protobuf/editions/golden/simple_proto3.pb.cc [ RUN ] third_party/protobuf/editions/golden/simple_proto3.pb.h @@ @@ diff --git a/editions/golden/compare_cpp_codegen_failure.xml b/editions/golden/compare_cpp_codegen_failure.xml index 05caafb727..cb65ff53a6 100644 --- a/editions/golden/compare_cpp_codegen_failure.xml +++ b/editions/golden/compare_cpp_codegen_failure.xml @@ -2,10 +2,10 @@ - + - + diff --git a/hpb/BUILD b/hpb/BUILD index 3f39415bc0..69587c7fdd 100644 --- a/hpb/BUILD +++ b/hpb/BUILD @@ -15,7 +15,7 @@ load( ) # begin:google_only -# package(default_applicable_licenses = ["//upb:license"]) +# package(default_applicable_licenses = ["//src/google/protobuf:license"]) # end:google_only licenses(["notice"]) diff --git a/hpb/extension_lock_test.cc b/hpb/extension_lock_test.cc index ef8013bdb8..81198082e1 100644 --- a/hpb/extension_lock_test.cc +++ b/hpb/extension_lock_test.cc @@ -36,11 +36,11 @@ std::string GenerateTestData() { model.set_str1("str"); ThemeExtension extension1; extension1.set_ext_name("theme"); - ABSL_CHECK_OK(::protos::SetExtension(&model, theme, extension1)); + ABSL_CHECK_OK(::hpb::SetExtension(&model, theme, extension1)); ThemeExtension extension2; extension2.set_ext_name("theme_extension"); - ABSL_CHECK_OK(::protos::SetExtension(&model, ThemeExtension::theme_extension, - extension2)); + ABSL_CHECK_OK( + ::hpb::SetExtension(&model, ThemeExtension::theme_extension, extension2)); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); ABSL_CHECK_OK(bytes); @@ -54,7 +54,7 @@ void unlock_func(const void* msg) { m[absl::HashOf(msg) & 0x7].unlock(); } return &unlock_func; } -void TestConcurrentExtensionAccess(::protos::ExtensionRegistry registry) { +void TestConcurrentExtensionAccess(::hpb::ExtensionRegistry registry) { ::hpb::internal::upb_extension_locker_global.store(&lock_func, std::memory_order_release); const std::string payload = GenerateTestData(); @@ -62,7 +62,7 @@ void TestConcurrentExtensionAccess(::protos::ExtensionRegistry registry) { ::protos::Parse(payload, registry).value(); const auto test_main = [&] { EXPECT_EQ("str", parsed_model.str1()); }; const auto test_theme = [&] { - ASSERT_TRUE(::protos::HasExtension(&parsed_model, theme)); + ASSERT_TRUE(::hpb::HasExtension(&parsed_model, theme)); auto ext = ::protos::GetExtension(&parsed_model, theme); ASSERT_OK(ext); EXPECT_EQ((*ext)->ext_name(), "theme"); diff --git a/hpb/hpb.cc b/hpb/hpb.cc index d63c7d98d9..a170dbf6fa 100644 --- a/hpb/hpb.cc +++ b/hpb/hpb.cc @@ -27,7 +27,7 @@ #include "upb/wire/decode.h" #include "upb/wire/encode.h" -namespace protos { +namespace hpb { // begin:google_only absl::Status MessageAllocationError(SourceLocation loc) { @@ -83,10 +83,6 @@ absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc namespace internal { -using ::hpb::internal::upb_extension_locker_global; -using ::hpb::internal::UpbExtensionLocker; -using ::hpb::internal::UpbExtensionUnlocker; - upb_ExtensionRegistry* GetUpbExtensions( const ExtensionRegistry& extension_registry) { return extension_registry.registry_; @@ -186,4 +182,4 @@ absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, } // namespace internal -} // namespace protos +} // namespace hpb diff --git a/hpb/hpb.h b/hpb/hpb.h index 8de8e249e8..f6b49f5347 100644 --- a/hpb/hpb.h +++ b/hpb/hpb.h @@ -8,11 +8,13 @@ #ifndef PROTOBUF_HPB_HPB_H_ #define PROTOBUF_HPB_HPB_H_ +#include #include #include #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/string_view.h" #include "upb/base/status.hpp" #include "upb/mem/arena.hpp" #include "upb/message/copy.h" @@ -21,12 +23,8 @@ #include "upb/wire/encode.h" namespace hpb { -using Arena = ::upb::Arena; -} - -namespace protos { -using hpb::Arena; class ExtensionRegistry; +using Arena = ::upb::Arena; template using Proxy = std::conditional_t::value, @@ -85,6 +83,7 @@ class Ptr final { template Ptr(T* m) -> Ptr; +// TODO: b/354766950 - Move upb-specific chunks out of hpb header inline absl::string_view UpbStrToStringView(upb_StringView str) { return absl::string_view(str.data, str.size); } @@ -98,12 +97,6 @@ inline upb_StringView UpbStrFromStringView(absl::string_view str, return upb_StringView_FromDataAndSize(buffer, str_size); } -template -typename T::Proxy CreateMessage(::protos::Arena& arena) { - return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), - arena.ptr()); -} - // begin:github_only // // This type exists to work around an absl type that has not yet been // // released. @@ -131,6 +124,34 @@ absl::Status MessageEncodeError(upb_EncodeStatus status, SourceLocation loc = SourceLocation::current()); namespace internal { +template +struct RemovePtr; + +template +struct RemovePtr> { + using type = T; +}; + +template +struct RemovePtr { + using type = T; +}; + +template +using RemovePtrT = typename RemovePtr::type; + +template , + typename = std::enable_if_t>> +using PtrOrRaw = T; + +template +using EnableIfHpbClass = std::enable_if_t< + std::is_base_of::value && + std::is_base_of::value>; + +template +using EnableIfMutableProto = std::enable_if_t::value>; + struct PrivateAccess { template static auto* GetInternalMsg(T&& message) { @@ -148,6 +169,11 @@ struct PrivateAccess { static auto CreateMessage(upb_Arena* arena) { return typename T::Proxy(upb_Message_New(T::minitable(), arena), arena); } + + template + static constexpr uint32_t GetExtensionNumber(const ExtensionId& id) { + return id.number(); + } }; template @@ -200,6 +226,12 @@ class ExtensionIdentifier : public ExtensionMiniTableProvider { constexpr explicit ExtensionIdentifier( const upb_MiniTableExtension* mini_table_ext) : ExtensionMiniTableProvider(mini_table_ext) {} + + private: + constexpr uint32_t number() const { + return upb_MiniTableExtension_Number(mini_table_ext()); + } + friend class PrivateAccess; }; template @@ -249,82 +281,12 @@ absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, const upb_MiniTableExtension* ext, const upb_Message* extension); -template -struct RemovePtr; - -template -struct RemovePtr> { - using type = T; -}; - -template -struct RemovePtr { - using type = T; -}; - -template -using RemovePtrT = typename RemovePtr::type; - -template , - typename = std::enable_if_t>> -using PtrOrRaw = T; - -template -using EnableIfHpbClass = std::enable_if_t< - std::is_base_of::value && - std::is_base_of::value>; - -template -using EnableIfMutableProto = std::enable_if_t::value>; - } // namespace internal -template -void DeepCopy(Ptr source_message, Ptr target_message) { - static_assert(!std::is_const_v); - ::protos::internal::DeepCopy( - internal::GetInternalMsg(target_message), - internal::GetInternalMsg(source_message), T::minitable(), - static_cast(target_message->GetInternalArena())); -} - -template -typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { - return internal::PrivateAccess::Proxy( - ::protos::internal::DeepClone(internal::GetInternalMsg(message), - T::minitable(), arena), - arena); -} - -template -void DeepCopy(Ptr source_message, T* target_message) { - static_assert(!std::is_const_v); - DeepCopy(source_message, protos::Ptr(target_message)); -} - -template -void DeepCopy(const T* source_message, Ptr target_message) { - static_assert(!std::is_const_v); - DeepCopy(protos::Ptr(source_message), target_message); -} - -template -void DeepCopy(const T* source_message, T* target_message) { - static_assert(!std::is_const_v); - DeepCopy(protos::Ptr(source_message), protos::Ptr(target_message)); -} - -template -void ClearMessage(internal::PtrOrRaw message) { - auto ptr = Ptr(message); - auto minitable = internal::GetMiniTable(ptr); - upb_Message_Clear(internal::GetInternalMsg(ptr), minitable); -} - class ExtensionRegistry { public: ExtensionRegistry( - const std::vector& + const std::vector& extensions, const upb::Arena& arena) : registry_(upb_ExtensionRegistry_New(arena.ptr())) { @@ -341,151 +303,209 @@ class ExtensionRegistry { } private: - friend upb_ExtensionRegistry* ::protos::internal::GetUpbExtensions( + friend upb_ExtensionRegistry* ::hpb::internal::GetUpbExtensions( const ExtensionRegistry& extension_registry); upb_ExtensionRegistry* registry_; }; template > + typename = hpb::internal::EnableIfHpbClass> ABSL_MUST_USE_RESULT bool HasExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { - return ::protos::internal::HasExtensionOrUnknown( - ::protos::internal::GetInternalMsg(message), id.mini_table_ext()); + const ::hpb::internal::ExtensionIdentifier& id) { + return ::hpb::internal::HasExtensionOrUnknown( + ::hpb::internal::GetInternalMsg(message), id.mini_table_ext()); } template > + typename = hpb::internal::EnableIfHpbClass> ABSL_MUST_USE_RESULT bool HasExtension( const T* message, - const ::protos::internal::ExtensionIdentifier& id) { - return HasExtension(protos::Ptr(message), id); + const ::hpb::internal::ExtensionIdentifier& id) { + return HasExtension(Ptr(message), id); } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> void ClearExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { static_assert(!std::is_const_v, ""); - upb_Message_ClearExtension(internal::GetInternalMsg(message), + upb_Message_ClearExtension(hpb::internal::GetInternalMsg(message), id.mini_table_ext()); } template > + typename = hpb::internal::EnableIfHpbClass> void ClearExtension( - T* message, - const ::protos::internal::ExtensionIdentifier& id) { - ClearExtension(::protos::Ptr(message), id); + T* message, const ::hpb::internal::ExtensionIdentifier& id) { + ClearExtension(Ptr(message), id); } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::hpb::internal::ExtensionIdentifier& id, const Extension& value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); - return ::protos::internal::SetExtension(internal::GetInternalMsg(message), - message_arena, id.mini_table_ext(), - internal::GetInternalMsg(&value)); + return ::hpb::internal::SetExtension(hpb::internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + hpb::internal::GetInternalMsg(&value)); } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::hpb::internal::ExtensionIdentifier& id, Ptr value) { static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); - return ::protos::internal::SetExtension(internal::GetInternalMsg(message), - message_arena, id.mini_table_ext(), - internal::GetInternalMsg(value)); + return ::hpb::internal::SetExtension(hpb::internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + hpb::internal::GetInternalMsg(value)); } template , - typename = internal::EnableIfMutableProto> + typename = hpb::internal::EnableIfHpbClass, + typename = hpb::internal::EnableIfMutableProto> absl::Status SetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id, + const ::hpb::internal::ExtensionIdentifier& id, Extension&& value) { Extension ext = std::move(value); static_assert(!std::is_const_v); auto* message_arena = static_cast(message->GetInternalArena()); auto* extension_arena = static_cast(ext.GetInternalArena()); - return ::protos::internal::MoveExtension( - internal::GetInternalMsg(message), message_arena, id.mini_table_ext(), - internal::GetInternalMsg(&ext), extension_arena); + return ::hpb::internal::MoveExtension(hpb::internal::GetInternalMsg(message), + message_arena, id.mini_table_ext(), + hpb::internal::GetInternalMsg(&ext), + extension_arena); } template > + typename = hpb::internal::EnableIfHpbClass> absl::Status SetExtension( - T* message, const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::hpb::internal::ExtensionIdentifier& id, const Extension& value) { - return ::protos::SetExtension(::protos::Ptr(message), id, value); + return ::hpb::SetExtension(Ptr(message), id, value); } template > + typename = hpb::internal::EnableIfHpbClass> absl::Status SetExtension( - T* message, const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::hpb::internal::ExtensionIdentifier& id, Extension&& value) { - return ::protos::SetExtension(::protos::Ptr(message), id, - std::forward(value)); + return ::hpb::SetExtension(Ptr(message), id, std::forward(value)); } template > + typename = hpb::internal::EnableIfHpbClass> absl::Status SetExtension( - T* message, const ::protos::internal::ExtensionIdentifier& id, + T* message, const ::hpb::internal::ExtensionIdentifier& id, Ptr value) { - return ::protos::SetExtension(::protos::Ptr(message), id, value); + return ::hpb::SetExtension(Ptr(message), id, value); } template > + typename = hpb::internal::EnableIfHpbClass> absl::StatusOr> GetExtension( Ptr message, - const ::protos::internal::ExtensionIdentifier& id) { + const ::hpb::internal::ExtensionIdentifier& id) { // TODO: Fix const correctness issues. upb_MessageValue value; - const bool ok = ::protos::internal::GetOrPromoteExtension( - const_cast(internal::GetInternalMsg(message)), - id.mini_table_ext(), ::protos::internal::GetArena(message), &value); + const bool ok = ::hpb::internal::GetOrPromoteExtension( + const_cast(::hpb::internal::GetInternalMsg(message)), + id.mini_table_ext(), ::hpb::internal::GetArena(message), &value); if (!ok) { return ExtensionNotFoundError( upb_MiniTableExtension_Number(id.mini_table_ext())); } - return Ptr(::protos::internal::CreateMessage( - (upb_Message*)value.msg_val, ::protos::internal::GetArena(message))); + return Ptr(::hpb::internal::CreateMessage( + (upb_Message*)value.msg_val, ::hpb::internal::GetArena(message))); } template > + typename = hpb::internal::EnableIfHpbClass> absl::StatusOr> GetExtension( const T* message, - const ::protos::internal::ExtensionIdentifier& id) { - return GetExtension(protos::Ptr(message), id); + const ::hpb::internal::ExtensionIdentifier& id) { + return GetExtension(Ptr(message), id); } +template +typename T::Proxy CreateMessage(::hpb::Arena& arena) { + return typename T::Proxy(upb_Message_New(T::minitable(), arena.ptr()), + arena.ptr()); +} + +template +typename T::Proxy CloneMessage(Ptr message, upb_Arena* arena) { + return ::hpb::internal::PrivateAccess::Proxy( + ::hpb::internal::DeepClone(::hpb::internal::GetInternalMsg(message), + T::minitable(), arena), + arena); +} + +template +void DeepCopy(Ptr source_message, Ptr target_message) { + static_assert(!std::is_const_v); + ::hpb::internal::DeepCopy( + hpb::internal::GetInternalMsg(target_message), + hpb::internal::GetInternalMsg(source_message), T::minitable(), + static_cast(target_message->GetInternalArena())); +} + +template +void DeepCopy(Ptr source_message, T* target_message) { + static_assert(!std::is_const_v); + DeepCopy(source_message, Ptr(target_message)); +} + +template +void DeepCopy(const T* source_message, Ptr target_message) { + static_assert(!std::is_const_v); + DeepCopy(Ptr(source_message), target_message); +} + +template +void DeepCopy(const T* source_message, T* target_message) { + static_assert(!std::is_const_v); + DeepCopy(Ptr(source_message), Ptr(target_message)); +} + +template +void ClearMessage(hpb::internal::PtrOrRaw message) { + auto ptr = Ptr(message); + auto minitable = hpb::internal::GetMiniTable(ptr); + upb_Message_Clear(hpb::internal::GetInternalMsg(ptr), minitable); +} + +} // namespace hpb + +namespace protos { +using hpb::Arena; +using hpb::ExtensionNotFoundError; +using hpb::MessageAllocationError; +using hpb::MessageDecodeError; +using hpb::MessageEncodeError; +using hpb::Ptr; +using hpb::SourceLocation; + template ABSL_MUST_USE_RESULT bool Parse(Ptr message, absl::string_view bytes) { static_assert(!std::is_const_v); - upb_Message_Clear(internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message)); + upb_Message_Clear(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), - internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), + ::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), /* extreg= */ nullptr, /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; } @@ -493,36 +513,36 @@ ABSL_MUST_USE_RESULT bool Parse(Ptr message, absl::string_view bytes) { template ABSL_MUST_USE_RESULT bool Parse( Ptr message, absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry) { + const ::hpb::ExtensionRegistry& extension_registry) { static_assert(!std::is_const_v); - upb_Message_Clear(internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message)); + upb_Message_Clear(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), - internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), + ::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), /* extreg= */ - ::protos::internal::GetUpbExtensions(extension_registry), + ::hpb::internal::GetUpbExtensions(extension_registry), /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; } template ABSL_MUST_USE_RESULT bool Parse( T* message, absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry) { + const ::hpb::ExtensionRegistry& extension_registry) { static_assert(!std::is_const_v); - return Parse(protos::Ptr(message, bytes, extension_registry)); + return Parse(Ptr(message, bytes, extension_registry)); } template ABSL_MUST_USE_RESULT bool Parse(T* message, absl::string_view bytes) { static_assert(!std::is_const_v); - upb_Message_Clear(internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message)); + upb_Message_Clear(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message)); auto* arena = static_cast(message->GetInternalArena()); return upb_Decode(bytes.data(), bytes.size(), - internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), + ::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), /* extreg= */ nullptr, /* options= */ 0, arena) == kUpb_DecodeStatus_Ok; } @@ -533,7 +553,7 @@ absl::StatusOr Parse(absl::string_view bytes, int options = 0) { auto* arena = static_cast(message.GetInternalArena()); upb_DecodeStatus status = upb_Decode(bytes.data(), bytes.size(), message.msg(), - ::protos::internal::GetMiniTable(&message), + ::hpb::internal::GetMiniTable(&message), /* extreg= */ nullptr, /* options= */ 0, arena); if (status == kUpb_DecodeStatus_Ok) { return message; @@ -543,14 +563,14 @@ absl::StatusOr Parse(absl::string_view bytes, int options = 0) { template absl::StatusOr Parse(absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry, + const ::hpb::ExtensionRegistry& extension_registry, int options = 0) { T message; auto* arena = static_cast(message.GetInternalArena()); upb_DecodeStatus status = upb_Decode(bytes.data(), bytes.size(), message.msg(), - ::protos::internal::GetMiniTable(&message), - ::protos::internal::GetUpbExtensions(extension_registry), + ::hpb::internal::GetMiniTable(&message), + ::hpb::internal::GetUpbExtensions(extension_registry), /* options= */ 0, arena); if (status == kUpb_DecodeStatus_Ok) { return message; @@ -561,17 +581,23 @@ absl::StatusOr Parse(absl::string_view bytes, template absl::StatusOr Serialize(const T* message, upb::Arena& arena, int options = 0) { - return ::protos::internal::Serialize( - internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), arena.ptr(), options); + return ::hpb::internal::Serialize(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), + arena.ptr(), options); } template absl::StatusOr Serialize(Ptr message, upb::Arena& arena, int options = 0) { - return ::protos::internal::Serialize( - internal::GetInternalMsg(message), - ::protos::internal::GetMiniTable(message), arena.ptr(), options); + return ::hpb::internal::Serialize(::hpb::internal::GetInternalMsg(message), + ::hpb::internal::GetMiniTable(message), + arena.ptr(), options); +} + +template +constexpr uint32_t ExtensionNumber( + ::hpb::internal::ExtensionIdentifier id) { + return ::hpb::internal::PrivateAccess::GetExtensionNumber(id); } } // namespace protos diff --git a/hpb/repeated_field.h b/hpb/repeated_field.h index 3162f6a9d7..7810a043f2 100644 --- a/hpb/repeated_field.h +++ b/hpb/repeated_field.h @@ -94,13 +94,13 @@ class RepeatedFieldProxy : RepeatedFieldProxyBase(arr, arena) {} RepeatedFieldProxy(upb_Array* arr, upb_Arena* arena) : RepeatedFieldProxyMutableBase(arr, arena) {} - // Constructor used by ::protos::Ptr. + // Constructor used by ::hpb::Ptr. RepeatedFieldProxy(const RepeatedFieldProxy&) = default; // T::CProxy [] operator specialization. typename T::CProxy operator[](size_t n) const { upb_MessageValue message_value = upb_Array_Get(this->arr_, n); - return ::protos::internal::CreateMessage>( + return ::hpb::internal::CreateMessage>( (upb_Message*)message_value.msg_val, this->arena_); } @@ -109,8 +109,8 @@ class RepeatedFieldProxy template > typename T::Proxy operator[](size_t n) { - return ::protos::internal::CreateMessageProxy(this->GetMessage(n), - this->arena_); + return ::hpb::internal::CreateMessageProxy(this->GetMessage(n), + this->arena_); } // Mutable message reference specialization. @@ -119,8 +119,8 @@ class RepeatedFieldProxy void push_back(const T& t) { upb_MessageValue message_value; message_value.msg_val = upb_Message_DeepClone( - PrivateAccess::GetInternalMsg(&t), ::protos::internal::GetMiniTable(&t), - this->arena_); + ::hpb::internal::PrivateAccess::GetInternalMsg(&t), + ::hpb::internal::GetMiniTable(&t), this->arena_); upb_Array_Append(this->arr_, message_value, this->arena_); } @@ -129,8 +129,9 @@ class RepeatedFieldProxy typename = std::enable_if_t> void push_back(T&& msg) { upb_MessageValue message_value; - message_value.msg_val = PrivateAccess::GetInternalMsg(&msg); - upb_Arena_Fuse(GetArena(&msg), this->arena_); + message_value.msg_val = + ::hpb::internal::PrivateAccess::GetInternalMsg(&msg); + upb_Arena_Fuse(::hpb::internal::GetArena(&msg), this->arena_); upb_Array_Append(this->arr_, message_value, this->arena_); T moved_msg = std::move(msg); } @@ -147,7 +148,7 @@ class RepeatedFieldProxy reverse_iterator rend() const { return reverse_iterator(begin()); } private: - friend class ::protos::Ptr; + friend class ::hpb::Ptr; }; // RepeatedField proxy for repeated strings. @@ -175,7 +176,7 @@ class RepeatedFieldStringProxy // Mutable constructor. RepeatedFieldStringProxy(upb_Array* arr, upb_Arena* arena) : RepeatedFieldProxyMutableBase(arr, arena) {} - // Constructor used by ::protos::Ptr. + // Constructor used by ::hpb::Ptr. RepeatedFieldStringProxy(const RepeatedFieldStringProxy&) = default; reference operator[](size_t n) const { return begin()[n]; } @@ -222,7 +223,7 @@ class RepeatedFieldScalarProxy : RepeatedFieldProxyBase(arr, arena) {} RepeatedFieldScalarProxy(upb_Array* arr, upb_Arena* arena) : RepeatedFieldProxyMutableBase(arr, arena) {} - // Constructor used by ::protos::Ptr. + // Constructor used by ::hpb::Ptr. RepeatedFieldScalarProxy(const RepeatedFieldScalarProxy&) = default; T operator[](size_t n) const { @@ -285,10 +286,10 @@ class RepeatedField { // We would like to reference T::CProxy. Validate forwarding header design. using ValueProxy = std::conditional_t< kIsScalar, T, - std::conditional_t>>; + std::conditional_t>>; using ValueCProxy = std::conditional_t< kIsScalar, const T, - std::conditional_t>>; + std::conditional_t>>; using Access = std::conditional_t< kIsScalar, internal::RepeatedFieldScalarProxy, std::conditional_t, diff --git a/hpb/repeated_field_iterator.h b/hpb/repeated_field_iterator.h index afc05c9949..c6f9f0d46e 100644 --- a/hpb/repeated_field_iterator.h +++ b/hpb/repeated_field_iterator.h @@ -354,10 +354,10 @@ struct MessageIteratorPolicy { void AddOffset(ptrdiff_t offset) { arr += offset; } auto Get() const { if constexpr (std::is_const_v) { - return ::protos::internal::CreateMessage< - typename std::remove_const_t>(*arr, arena); + return ::hpb::internal::CreateMessage>( + *arr, arena); } else { - return ::protos::internal::CreateMessageProxy(*arr, arena); + return ::hpb::internal::CreateMessageProxy(*arr, arena); } } auto Index() const { return arr; } diff --git a/hpb_generator/BUILD b/hpb_generator/BUILD index 7ca5c590b9..fba7127106 100644 --- a/hpb_generator/BUILD +++ b/hpb_generator/BUILD @@ -11,7 +11,7 @@ load( ) # begin:google_only -# package(default_applicable_licenses = ["//upb:license"]) +# package(default_applicable_licenses = ["//src/google/protobuf:license"]) # end:google_only licenses(["notice"]) diff --git a/hpb_generator/gen_accessors.cc b/hpb_generator/gen_accessors.cc index 048efcd3df..0fd646fa2c 100644 --- a/hpb_generator/gen_accessors.cc +++ b/hpb_generator/gen_accessors.cc @@ -222,7 +222,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { output( R"cc( $1 $0::$2() const { - return ::protos::UpbStrToStringView($3_$4(msg_)); + return ::hpb::UpbStrToStringView($3_$4(msg_)); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -231,7 +231,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { output( R"cc( void $0::set_$2($1 value) { - $4_set_$3(msg_, ::protos::UpbStrFromStringView(value, $5)); + $4_set_$3(msg_, ::hpb::UpbStrFromStringView(value, $5)); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -244,7 +244,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { if (!has_$2()) { return $4::default_instance(); } - return ::protos::internal::CreateMessage<$4>( + return ::hpb::internal::CreateMessage<$4>( (upb_Message*)($3_$5(msg_)), arena_); } )cc", @@ -256,7 +256,7 @@ void WriteAccessorsInSource(const protobuf::Descriptor* desc, Output& output) { output( R"cc( $1 $0::mutable_$2() { - return ::protos::internal::CreateMessageProxy<$4>( + return ::hpb::internal::CreateMessageProxy<$4>( (upb_Message*)($3_mutable_$5(msg_, $6)), $6); } )cc", @@ -295,7 +295,7 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, R"cc( bool $0::set_$1($2 key, $3 value) { upb_Message* clone = upb_Message_DeepClone( - ::protos::internal::PrivateAccess::GetInternalMsg(value), &$9, + ::hpb::internal::PrivateAccess::GetInternalMsg(value), &$9, arena_); $6return $4_$8_set(msg_, $7, ($5*)clone, arena_); } @@ -309,7 +309,7 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, R"cc( bool $0::set_$1($2 key, $3 value) { upb_Message* clone = upb_Message_DeepClone( - ::protos::internal::PrivateAccess::GetInternalMsg(value), &$9, + ::hpb::internal::PrivateAccess::GetInternalMsg(value), &$9, arena_); $6return $4_$8_set(msg_, $7, ($5*)clone, arena_); } @@ -325,7 +325,7 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, $5* msg_value; $7bool success = $4_$9_get(msg_, $8, &msg_value); if (success) { - return ::protos::internal::CreateMessage<$6>(UPB_UPCAST(msg_value), arena_); + return ::hpb::internal::CreateMessage<$6>(UPB_UPCAST(msg_value), arena_); } return absl::NotFoundError(""); } @@ -347,9 +347,8 @@ void WriteMapAccessorDefinitions(const protobuf::Descriptor* message, output( R"cc( bool $0::set_$1($2 key, $3 value) { - $5return $4_$7_set(msg_, $6, - ::protos::UpbStrFromStringView(value, arena_), - arena_); + $5return $4_$7_set( + msg_, $6, ::hpb::UpbStrFromStringView(value, arena_), arena_); } )cc", class_name, resolved_field_name, CppConstType(key), CppConstType(val), diff --git a/hpb_generator/gen_extensions.cc b/hpb_generator/gen_extensions.cc index d0c54eb36d..00a8dcd496 100644 --- a/hpb_generator/gen_extensions.cc +++ b/hpb_generator/gen_extensions.cc @@ -38,13 +38,13 @@ void WriteExtensionIdentifierHeader(const protobuf::FieldDescriptor* ext, if (ext->extension_scope()) { output( R"cc( - static const ::protos::internal::ExtensionIdentifier<$0, $1> $2; + static const ::hpb::internal::ExtensionIdentifier<$0, $1> $2; )cc", ContainingTypeName(ext), CppTypeParameterName(ext), ext->name()); } else { output( R"cc( - extern const ::protos::internal::ExtensionIdentifier<$0, $1> $2; + extern const ::hpb::internal::ExtensionIdentifier<$0, $1> $2; )cc", ContainingTypeName(ext), CppTypeParameterName(ext), ext->name()); } diff --git a/hpb_generator/gen_messages.cc b/hpb_generator/gen_messages.cc index 75e486ed24..de7045aac3 100644 --- a/hpb_generator/gen_messages.cc +++ b/hpb_generator/gen_messages.cc @@ -113,7 +113,7 @@ void WriteModelAccessDeclaration(const protobuf::Descriptor* descriptor, friend class $2; friend class $0Proxy; friend class $0CProxy; - friend struct ::protos::internal::PrivateAccess; + friend struct ::hpb::internal::PrivateAccess; $1* msg_; upb_Arena* arena_; )cc", @@ -242,18 +242,17 @@ void WriteModelPublicDeclaration( arena_ = owned_arena_.ptr(); upb_Arena_Fuse(arena_, arena); } - ::protos::Arena owned_arena_; - friend struct ::protos::internal::PrivateAccess; + ::hpb::Arena owned_arena_; + friend struct ::hpb::internal::PrivateAccess; friend Proxy; friend CProxy; friend absl::StatusOr<$2>(::protos::Parse<$2>(absl::string_view bytes, int options)); friend absl::StatusOr<$2>(::protos::Parse<$2>( absl::string_view bytes, - const ::protos::ExtensionRegistry& extension_registry, - int options)); - friend upb_Arena* ::protos::internal::GetArena<$0>($0* message); - friend upb_Arena* ::protos::internal::GetArena<$0>(::protos::Ptr<$0> message); + const ::hpb::ExtensionRegistry& extension_registry, int options)); + friend upb_Arena* ::hpb::internal::GetArena<$0>($0* message); + friend upb_Arena* ::hpb::internal::GetArena<$0>(::hpb::Ptr<$0> message); friend $0(::hpb::internal::MoveMessage<$0>(upb_Message* msg, upb_Arena* arena)); )cc", ClassName(descriptor), MessageName(descriptor), @@ -298,22 +297,22 @@ void WriteModelProxyDeclaration(const protobuf::Descriptor* descriptor, $0Proxy(upb_Message* msg, upb_Arena* arena) : internal::$0Access(($1*)msg, arena) {} - friend $0::Proxy(::protos::CreateMessage<$0>(::protos::Arena& arena)); - friend $0::Proxy(::protos::internal::CreateMessageProxy<$0>( - upb_Message*, upb_Arena*)); - friend struct ::protos::internal::PrivateAccess; + friend $0::Proxy(::hpb::CreateMessage<$0>(::hpb::Arena& arena)); + friend $0::Proxy(::hpb::internal::CreateMessageProxy<$0>(upb_Message*, + upb_Arena*)); + friend struct ::hpb::internal::PrivateAccess; friend class RepeatedFieldProxy; friend class $0CProxy; friend class $0Access; - friend class ::protos::Ptr<$0>; - friend class ::protos::Ptr; + friend class ::hpb::Ptr<$0>; + friend class ::hpb::Ptr; static const upb_MiniTable* minitable() { return $0::minitable(); } - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0Proxy>( + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0Proxy>( const $0Proxy* message); - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0Proxy>( - ::protos::Ptr<$0Proxy> message); - friend upb_Arena* ::protos::internal::GetArena<$2>($2* message); - friend upb_Arena* ::protos::internal::GetArena<$2>(::protos::Ptr<$2> message); + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0Proxy>( + ::hpb::Ptr<$0Proxy> message); + friend upb_Arena* ::hpb::internal::GetArena<$2>($2* message); + friend upb_Arena* ::hpb::internal::GetArena<$2>(::hpb::Ptr<$2> message); static void Rebind($0Proxy& lhs, const $0Proxy& rhs) { lhs.msg_ = rhs.msg_; lhs.arena_ = rhs.arena_; @@ -334,7 +333,7 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, public: $0CProxy() = delete; $0CProxy(const $0* m) - : internal::$0Access(m->msg_, ::protos::internal::GetArena(m)) {} + : internal::$0Access(m->msg_, ::hpb::internal::GetArena(m)) {} $0CProxy($0Proxy m); using $0Access::GetInternalArena; )cc", @@ -352,15 +351,15 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, $0CProxy(const upb_Message* msg, upb_Arena* arena) : internal::$0Access(($1*)msg, arena){}; - friend struct ::protos::internal::PrivateAccess; + friend struct ::hpb::internal::PrivateAccess; friend class RepeatedFieldProxy; - friend class ::protos::Ptr<$0>; - friend class ::protos::Ptr; + friend class ::hpb::Ptr<$0>; + friend class ::hpb::Ptr; static const upb_MiniTable* minitable() { return $0::minitable(); } - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0CProxy>( + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0CProxy>( const $0CProxy* message); - friend const upb_MiniTable* ::protos::internal::GetMiniTable<$0CProxy>( - ::protos::Ptr<$0CProxy> message); + friend const upb_MiniTable* ::hpb::internal::GetMiniTable<$0CProxy>( + ::hpb::Ptr<$0CProxy> message); static void Rebind($0CProxy& lhs, const $0CProxy& rhs) { lhs.msg_ = rhs.msg_; @@ -374,7 +373,7 @@ void WriteModelCProxyDeclaration(const protobuf::Descriptor* descriptor, void WriteDefaultInstanceHeader(const protobuf::Descriptor* message, Output& output) { - output(" static ::protos::Ptr default_instance();\n", + output(" static ::hpb::Ptr default_instance();\n", ClassName(message)); } @@ -393,27 +392,27 @@ void WriteMessageImplementation( } $0::$0(const $0& from) : $0Access() { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); + msg_ = ($1*)::hpb::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); } $0::$0(const CProxy& from) : $0Access() { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone( - ::protos::internal::GetInternalMsg(&from), &$2, arena_); + msg_ = ($1*)::hpb::internal::DeepClone( + ::hpb::internal::GetInternalMsg(&from), &$2, arena_); } $0::$0(const Proxy& from) : $0(static_cast(from)) {} internal::$0CProxy::$0CProxy($0Proxy m) : $0Access() { arena_ = m.arena_; - msg_ = ($1*)::protos::internal::GetInternalMsg(&m); + msg_ = ($1*)::hpb::internal::GetInternalMsg(&m); } $0& $0::operator=(const $3& from) { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); + msg_ = ($1*)::hpb::internal::DeepClone(UPB_UPCAST(from.msg_), &$2, arena_); return *this; } $0& $0::operator=(const CProxy& from) { arena_ = owned_arena_.ptr(); - msg_ = ($1*)::protos::internal::DeepClone( - ::protos::internal::GetInternalMsg(&from), &$2, arena_); + msg_ = ($1*)::hpb::internal::DeepClone( + ::hpb::internal::GetInternalMsg(&from), &$2, arena_); return *this; } )cc", @@ -450,8 +449,8 @@ void WriteMessageImplementation( output( R"cc( - ::protos::Ptr $0::default_instance() { - return ::protos::internal::CreateMessage<$0>( + ::hpb::Ptr $0::default_instance() { + return ::hpb::internal::CreateMessage<$0>( (upb_Message *)_$0_default_instance_.msg, _$0_default_instance_.arena); } diff --git a/hpb_generator/gen_repeated_fields.cc b/hpb_generator/gen_repeated_fields.cc index f010294d43..1b4eef3c5a 100644 --- a/hpb_generator/gen_repeated_fields.cc +++ b/hpb_generator/gen_repeated_fields.cc @@ -87,7 +87,7 @@ void WriteRepeatedFieldsInMessageHeader(const protobuf::Descriptor* desc, R"cc( $1 $2(size_t index) const; const ::protos::RepeatedField::CProxy $2() const; - ::protos::Ptr<::protos::RepeatedField<$4>> mutable_$2(); + ::hpb::Ptr<::protos::RepeatedField<$4>> mutable_$2(); absl::StatusOr<$0> add_$2(); $0 mutable_$2(size_t index) const; )cc", @@ -102,7 +102,7 @@ void WriteRepeatedFieldsInMessageHeader(const protobuf::Descriptor* desc, R"cc( $0 $1(size_t index) const; const ::protos::RepeatedField<$0>::CProxy $1() const; - ::protos::Ptr<::protos::RepeatedField<$0>> mutable_$1(); + ::hpb::Ptr<::protos::RepeatedField<$0>> mutable_$1(); bool add_$1($0 val); void set_$1(size_t index, $0 val); bool resize_$1(size_t len); @@ -113,7 +113,7 @@ void WriteRepeatedFieldsInMessageHeader(const protobuf::Descriptor* desc, R"cc( $0 $1(size_t index) const; const ::protos::RepeatedField<$0>::CProxy $1() const; - ::protos::Ptr<::protos::RepeatedField<$0>> mutable_$1(); + ::hpb::Ptr<::protos::RepeatedField<$0>> mutable_$1(); bool add_$1($0 val); void set_$1(size_t index, $0 val); bool resize_$1(size_t len); @@ -135,7 +135,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_$5(msg_, &len); assert(index < len); - return ::protos::internal::CreateMessage<$4>( + return ::hpb::internal::CreateMessage<$4>( (upb_Message*)*(ptr + index), arena_); } )cc", @@ -149,7 +149,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, if (!new_msg) { return ::protos::MessageAllocationError(); } - return ::protos::internal::CreateMessageProxy<$4>((upb_Message*)new_msg, $5); + return ::hpb::internal::CreateMessageProxy<$4>((upb_Message*)new_msg, $5); } )cc", class_name, MessagePtrConstType(field, /* const */ false), @@ -162,7 +162,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_$6(msg_, &len); assert(index < len); - return ::protos::internal::CreateMessageProxy<$4>( + return ::hpb::internal::CreateMessageProxy<$4>( (upb_Message*)*(ptr + index), $5); } )cc", @@ -177,7 +177,7 @@ void WriteRepeatedMessageAccessor(const protobuf::Descriptor* message, const upb_Array* arr = _$3_$4_$5(msg_, &size); return ::protos::RepeatedField::CProxy(arr, arena_); }; - ::protos::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { + ::hpb::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { size_t size; upb_Array* arr = _$3_$4_$6(msg_, &size, arena_); return ::protos::RepeatedField<$1>::Proxy(arr, arena_); @@ -205,7 +205,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_mutable_$4(msg_, &len); assert(index < len); - return ::protos::UpbStrToStringView(*(ptr + index)); + return ::hpb::UpbStrToStringView(*(ptr + index)); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -220,7 +220,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, output( R"cc( bool $0::add_$2($1 val) { - return $3_add_$4(msg_, ::protos::UpbStrFromStringView(val, arena_), arena_); + return $3_add_$4(msg_, ::hpb::UpbStrFromStringView(val, arena_), arena_); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -231,7 +231,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, size_t len; auto* ptr = $3_mutable_$4(msg_, &len); assert(index < len); - *(ptr + index) = ::protos::UpbStrFromStringView(val, arena_); + *(ptr + index) = ::hpb::UpbStrFromStringView(val, arena_); } )cc", class_name, CppConstType(field), resolved_field_name, @@ -243,7 +243,7 @@ void WriteRepeatedStringAccessor(const protobuf::Descriptor* message, const upb_Array* arr = _$3_$4_$5(msg_, &size); return ::protos::RepeatedField<$1>::CProxy(arr, arena_); }; - ::protos::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { + ::hpb::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { size_t size; upb_Array* arr = _$3_$4_$6(msg_, &size, arena_); return ::protos::RepeatedField<$1>::Proxy(arr, arena_); @@ -307,7 +307,7 @@ void WriteRepeatedScalarAccessor(const protobuf::Descriptor* message, const upb_Array* arr = _$3_$4_$5(msg_, &size); return ::protos::RepeatedField<$1>::CProxy(arr, arena_); }; - ::protos::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { + ::hpb::Ptr<::protos::RepeatedField<$1>> $0::mutable_$2() { size_t size; upb_Array* arr = _$3_$4_$6(msg_, &size, arena_); return ::protos::RepeatedField<$1>::Proxy(arr, arena_); diff --git a/hpb_generator/names.cc b/hpb_generator/names.cc index 5ea3da9d6a..864eca5aaf 100644 --- a/hpb_generator/names.cc +++ b/hpb_generator/names.cc @@ -17,6 +17,7 @@ namespace protobuf = ::proto2; namespace { +// TODO: b/346865271 append ::hpb instead of ::protos after namespace swap std::string NamespaceFromPackageName(absl::string_view package_name) { return absl::StrCat(absl::StrReplaceAll(package_name, {{".", "::"}}), "::protos"); @@ -82,7 +83,7 @@ std::string ClassName(const protobuf::Descriptor* descriptor) { const protobuf::Descriptor* parent = descriptor->containing_type(); std::string res; // Classes in global namespace without package names are prefixed - // by protos_ to avoid collision with C compiler structs defined in + // by hpb_ to avoid collision with C compiler structs defined in // proto.upb.h. if ((parent && parent->file()->package().empty()) || descriptor->file()->package().empty()) { @@ -155,7 +156,7 @@ std::string MessagePtrConstType(const protobuf::FieldDescriptor* field, bool is_const) { ABSL_DCHECK(field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE); std::string maybe_const = is_const ? "const " : ""; - return "::protos::Ptr<" + maybe_const + + return "::hpb::Ptr<" + maybe_const + QualifiedClassName(field->message_type()) + ">"; } diff --git a/hpb_generator/tests/BUILD b/hpb_generator/tests/BUILD index 0109c6dd06..111944bebd 100644 --- a/hpb_generator/tests/BUILD +++ b/hpb_generator/tests/BUILD @@ -156,8 +156,9 @@ # "@com_google_absl//absl/strings", # "//hpb:requires", # "//hpb", -# "//upb:mem", # "//hpb:repeated_field", +# "//protos", +# "//upb:mem", # ], # ) # end:google_only diff --git a/hpb_generator/tests/test_generated.cc b/hpb_generator/tests/test_generated.cc index 12068b0234..4a063e58f2 100644 --- a/hpb_generator/tests/test_generated.cc +++ b/hpb_generator/tests/test_generated.cc @@ -20,8 +20,10 @@ #include "absl/strings/string_view.h" #include "google/protobuf/compiler/hpb/tests/child_model.upb.proto.h" #include "google/protobuf/compiler/hpb/tests/no_package.upb.proto.h" +#include "google/protobuf/compiler/hpb/tests/test_extension.upb.proto.h" #include "google/protobuf/compiler/hpb/tests/test_model.upb.proto.h" #include "google/protobuf/hpb/hpb.h" +#include "google/protobuf/hpb/repeated_field.h" #include "google/protobuf/hpb/requires.h" #include "upb/mem/arena.h" #include "upb/mem/arena.hpp" @@ -66,13 +68,13 @@ TEST(CppGeneratedCode, MessageEnumValue) { TEST(CppGeneratedCode, ArenaConstructor) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); EXPECT_EQ(false, testModel.has_b1()); } TEST(CppGeneratedCode, Booleans) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); EXPECT_FALSE(testModel.b1()); testModel.set_b1(true); EXPECT_TRUE(testModel.b1()); @@ -86,7 +88,7 @@ TEST(CppGeneratedCode, Booleans) { TEST(CppGeneratedCode, ScalarInt32) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test int32 defaults. EXPECT_EQ(testModel.value(), 0); EXPECT_FALSE(testModel.has_value()); @@ -129,7 +131,7 @@ TEST(CppGeneratedCode, Strings) { TEST(CppGeneratedCode, ScalarUInt32) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_uint32(), 0); EXPECT_FALSE(testModel.has_optional_uint32()); @@ -149,16 +151,16 @@ TEST(CppGeneratedCode, ScalarUInt32) { TEST(CppGeneratedCode, ScalarInt64) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_int64(), 0); EXPECT_FALSE(testModel.has_optional_int64()); // Set value. - testModel.set_optional_int64(0xFF00CCDDA0001000); + testModel.set_optional_int64(static_cast(0xFF00CCDDA0001000)); EXPECT_TRUE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0xFF00CCDDA0001000); // Change value. - testModel.set_optional_int64(0xFF00CCDD70002000); + testModel.set_optional_int64(static_cast(0xFF00CCDD70002000)); EXPECT_TRUE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0xFF00CCDD70002000); // Clear value. @@ -166,14 +168,14 @@ TEST(CppGeneratedCode, ScalarInt64) { EXPECT_FALSE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0); // Set after clear. - testModel.set_optional_int64(0xFF00CCDDA0001000); + testModel.set_optional_int64(static_cast(0xFF00CCDDA0001000)); EXPECT_TRUE(testModel.has_optional_int64()); EXPECT_EQ(testModel.optional_int64(), 0xFF00CCDDA0001000); } TEST(CppGeneratedCode, ScalarFloat) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_float(), 0.0f); EXPECT_FALSE(testModel.has_optional_float()); @@ -201,7 +203,7 @@ TEST(CppGeneratedCode, ScalarFloat) { TEST(CppGeneratedCode, ScalarDouble) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Test defaults. EXPECT_EQ(testModel.optional_double(), 0.0); EXPECT_FALSE(testModel.has_optional_double()); @@ -225,7 +227,7 @@ TEST(CppGeneratedCode, ScalarDouble) { TEST(CppGeneratedCode, Enums) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); // Check enum default value. EXPECT_EQ(TestModel_Category_IMAGES, 5); @@ -253,7 +255,7 @@ TEST(CppGeneratedCode, Enums) { TEST(CppGeneratedCode, FieldWithDefaultValue) { ::hpb::Arena arena; - auto testModel = ::protos::CreateMessage(arena); + auto testModel = ::hpb::CreateMessage(arena); EXPECT_FALSE(testModel.has_int_value_with_default()); EXPECT_EQ(testModel.int_value_with_default(), 65); @@ -268,7 +270,7 @@ TEST(CppGeneratedCode, FieldWithDefaultValue) { TEST(CppGeneratedCode, OneOfFields) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_FALSE(test_model.has_oneof_member1()); EXPECT_FALSE(test_model.has_oneof_member2()); @@ -296,7 +298,7 @@ TEST(CppGeneratedCode, OneOfFields) { TEST(CppGeneratedCode, Messages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(false, test_model.has_child_model_1()); auto child_model = test_model.child_model_1(); EXPECT_EQ(false, child_model->has_child_b1()); @@ -320,7 +322,7 @@ TEST(CppGeneratedCode, Messages) { TEST(CppGeneratedCode, NestedMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); auto nested_child = test_model.nested_child_1(); EXPECT_EQ(0, nested_child->nested_child_name().size()); auto mutable_nested_child = test_model.mutable_nested_child_1(); @@ -331,7 +333,7 @@ TEST(CppGeneratedCode, NestedMessages) { TEST(CppGeneratedCode, RepeatedMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models_size()); // Should be able to clear repeated field when empty. test_model.mutable_child_models()->clear(); @@ -360,7 +362,7 @@ TEST(CppGeneratedCode, RepeatedMessages) { TEST(CppGeneratedCode, RepeatedScalar) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.value_array_size()); // Should be able to clear repeated field when empty. test_model.mutable_value_array()->clear(); @@ -381,7 +383,7 @@ TEST(CppGeneratedCode, RepeatedScalar) { TEST(CppGeneratedCode, RepeatedFieldClear) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); test_model.mutable_value_array()->push_back(5); test_model.mutable_value_array()->push_back(16); test_model.mutable_value_array()->push_back(27); @@ -392,7 +394,7 @@ TEST(CppGeneratedCode, RepeatedFieldClear) { TEST(CppGeneratedCode, RepeatedFieldProxyForScalars) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.value_array().size()); EXPECT_EQ(0, test_model.mutable_value_array()->size()); @@ -425,7 +427,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForScalars) { TEST(CppGeneratedCode, RepeatedScalarIterator) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); test_model.mutable_value_array()->push_back(5); test_model.mutable_value_array()->push_back(16); test_model.mutable_value_array()->push_back(27); @@ -492,7 +494,7 @@ TEST(CppGeneratedCode, RepeatedScalarIterator) { TEST(CppGeneratedCode, RepeatedFieldProxyForStrings) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.repeated_string().size()); EXPECT_EQ(0, test_model.mutable_repeated_string()->size()); @@ -527,7 +529,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForStrings) { TEST(CppGeneratedCode, RepeatedFieldProxyForMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models().size()); ChildModel1 child1; child1.set_child_str1(kTestStr1); @@ -572,7 +574,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForMessages) { TEST(CppGeneratedCode, EmptyRepeatedFieldProxyForMessages) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models().size()); ChildModel1 child1; child1.set_child_str1(kTestStr1); @@ -585,7 +587,7 @@ TEST(CppGeneratedCode, EmptyRepeatedFieldProxyForMessages) { TEST(CppGeneratedCode, RepeatedFieldProxyForMessagesIndexOperator) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_models().size()); ChildModel1 child1; child1.set_child_str1(kTestStr1); @@ -603,7 +605,7 @@ TEST(CppGeneratedCode, RepeatedFieldProxyForMessagesIndexOperator) { TEST(CppGeneratedCode, RepeatedStrings) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.repeated_string_size()); // Should be able to clear repeated field when empty. test_model.mutable_repeated_string()->clear(); @@ -626,11 +628,11 @@ TEST(CppGeneratedCode, MessageMapInt32KeyMessageValue) { const int key_test_value = 3; ::hpb::Arena arena; ::hpb::Arena child_arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.child_map_size()); test_model.clear_child_map(); EXPECT_EQ(0, test_model.child_map_size()); - auto child_model1 = ::protos::CreateMessage(child_arena); + auto child_model1 = ::hpb::CreateMessage(child_arena); child_model1.set_child_str1("abc"); test_model.set_child_map(key_test_value, child_model1); auto map_result = test_model.get_child_map(key_test_value); @@ -647,7 +649,7 @@ TEST(CppGeneratedCode, MessageMapInt32KeyMessageValue) { TEST(CppGeneratedCode, MessageMapStringKeyAndStringValue) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.str_to_str_map_size()); test_model.clear_str_to_str_map(); EXPECT_EQ(0, test_model.str_to_str_map_size()); @@ -663,7 +665,7 @@ TEST(CppGeneratedCode, MessageMapStringKeyAndStringValue) { TEST(CppGeneratedCode, MessageMapStringKeyAndInt32Value) { ::hpb::Arena arena; - auto test_model = ::protos::CreateMessage(arena); + auto test_model = ::hpb::CreateMessage(arena); EXPECT_EQ(0, test_model.str_to_int_map_size()); test_model.clear_str_to_int_map(); EXPECT_EQ(0, test_model.str_to_int_map_size()); @@ -681,26 +683,26 @@ TEST(CppGeneratedCode, MessageMapStringKeyAndInt32Value) { TEST(CppGeneratedCode, HasExtension) { TestModel model; - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); } TEST(CppGeneratedCode, HasExtensionPtr) { TestModel model; - EXPECT_EQ(false, ::protos::HasExtension(model.recursive_child(), theme)); + EXPECT_EQ(false, ::hpb::HasExtension(model.recursive_child(), theme)); } TEST(CppGeneratedCode, ClearExtensionWithEmptyExtension) { TestModel model; - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - ::protos::ClearExtension(&model, theme); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + ::hpb::ClearExtension(&model, theme); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); } TEST(CppGeneratedCode, ClearExtensionWithEmptyExtensionPtr) { TestModel model; - ::protos::Ptr recursive_child = model.mutable_recursive_child(); - ::protos::ClearExtension(recursive_child, theme); - EXPECT_EQ(false, ::protos::HasExtension(recursive_child, theme)); + ::hpb::Ptr recursive_child = model.mutable_recursive_child(); + ::hpb::ClearExtension(recursive_child, theme); + EXPECT_EQ(false, ::hpb::HasExtension(recursive_child, theme)); } TEST(CppGeneratedCode, SetExtension) { @@ -710,44 +712,42 @@ TEST(CppGeneratedCode, SetExtension) { // Use a nested scope to make sure the arenas are fused correctly. ThemeExtension extension1; extension1.set_ext_name("Hello World"); - prior_message = ::protos::internal::GetInternalMsg(&extension1); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ( - true, - ::protos::SetExtension(&model, theme, std::move(extension1)).ok()); + prior_message = ::hpb::internal::GetInternalMsg(&extension1); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, + ::hpb::SetExtension(&model, theme, std::move(extension1)).ok()); } - EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); - EXPECT_EQ(::protos::internal::GetInternalMsg(*ext), prior_message); + EXPECT_EQ(::hpb::internal::GetInternalMsg(*ext), prior_message); } TEST(CppGeneratedCode, SetExtensionWithPtr) { ::hpb::Arena arena_model; - ::protos::Ptr model = - ::protos::CreateMessage(arena_model); + ::hpb::Ptr model = ::hpb::CreateMessage(arena_model); void* prior_message; { // Use a nested scope to make sure the arenas are fused correctly. ::hpb::Arena arena; - ::protos::Ptr extension1 = - ::protos::CreateMessage(arena); + ::hpb::Ptr extension1 = + ::hpb::CreateMessage(arena); extension1->set_ext_name("Hello World"); - prior_message = ::protos::internal::GetInternalMsg(extension1); - EXPECT_EQ(false, ::protos::HasExtension(model, theme)); - auto res = ::protos::SetExtension(model, theme, extension1); + prior_message = ::hpb::internal::GetInternalMsg(extension1); + EXPECT_EQ(false, ::hpb::HasExtension(model, theme)); + auto res = ::hpb::SetExtension(model, theme, extension1); EXPECT_EQ(true, res.ok()); } - EXPECT_EQ(true, ::protos::HasExtension(model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(model, theme)); auto ext = ::protos::GetExtension(model, theme); EXPECT_TRUE(ext.ok()); - EXPECT_NE(::protos::internal::GetInternalMsg(*ext), prior_message); + EXPECT_NE(::hpb::internal::GetInternalMsg(*ext), prior_message); } #ifndef _MSC_VER TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { ::hpb::Arena arena; - ::protos::Ptr model = ::protos::CreateMessage(arena); + ::hpb::Ptr model = ::hpb::CreateMessage(arena); ThemeExtension extension1; ContainerExtension extension2; @@ -755,52 +755,52 @@ TEST(CppGeneratedCode, SetExtensionShouldNotCompileForWrongType) { return Requires(l); }; EXPECT_TRUE(canSetExtension( - [](auto p) -> decltype(::protos::SetExtension(p, theme, extension1)) {})); + [](auto p) -> decltype(::hpb::SetExtension(p, theme, extension1)) {})); // Wrong extension value type should fail to compile. EXPECT_TRUE(!canSetExtension( - [](auto p) -> decltype(::protos::SetExtension(p, theme, extension2)) {})); + [](auto p) -> decltype(::hpb::SetExtension(p, theme, extension2)) {})); // Wrong extension id with correct extension type should fail to compile. EXPECT_TRUE( - !canSetExtension([](auto p) -> decltype(::protos::SetExtension( + !canSetExtension([](auto p) -> decltype(::hpb::SetExtension( p, container_ext, extension1)) {})); } #endif TEST(CppGeneratedCode, SetExtensionWithPtrSameArena) { ::hpb::Arena arena; - ::protos::Ptr model = ::protos::CreateMessage(arena); + ::hpb::Ptr model = ::hpb::CreateMessage(arena); void* prior_message; { // Use a nested scope to make sure the arenas are fused correctly. - ::protos::Ptr extension1 = - ::protos::CreateMessage(arena); + ::hpb::Ptr extension1 = + ::hpb::CreateMessage(arena); extension1->set_ext_name("Hello World"); - prior_message = ::protos::internal::GetInternalMsg(extension1); - EXPECT_EQ(false, ::protos::HasExtension(model, theme)); - auto res = ::protos::SetExtension(model, theme, extension1); + prior_message = ::hpb::internal::GetInternalMsg(extension1); + EXPECT_EQ(false, ::hpb::HasExtension(model, theme)); + auto res = ::hpb::SetExtension(model, theme, extension1); EXPECT_EQ(true, res.ok()); } - EXPECT_EQ(true, ::protos::HasExtension(model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(model, theme)); auto ext = ::protos::GetExtension(model, theme); EXPECT_TRUE(ext.ok()); - EXPECT_NE(::protos::internal::GetInternalMsg(*ext), prior_message); + EXPECT_NE(::hpb::internal::GetInternalMsg(*ext), prior_message); } TEST(CppGeneratedCode, SetExtensionFusingFailureShouldCopy) { // Use an initial block to disallow fusing. char initial_block[1000]; - protos::Arena arena(initial_block, sizeof(initial_block)); + hpb::Arena arena(initial_block, sizeof(initial_block)); - protos::Ptr model = protos::CreateMessage(arena); + hpb::Ptr model = ::hpb::CreateMessage(arena); ThemeExtension extension1; extension1.set_ext_name("Hello World"); ASSERT_FALSE( - upb_Arena_Fuse(arena.ptr(), ::protos::internal::GetArena(&extension1))); - EXPECT_FALSE(::protos::HasExtension(model, theme)); - auto status = ::protos::SetExtension(model, theme, std::move(extension1)); + upb_Arena_Fuse(arena.ptr(), ::hpb::internal::GetArena(&extension1))); + EXPECT_FALSE(::hpb::HasExtension(model, theme)); + auto status = ::hpb::SetExtension(model, theme, std::move(extension1)); EXPECT_TRUE(status.ok()); - EXPECT_TRUE(::protos::HasExtension(model, theme)); + EXPECT_TRUE(::hpb::HasExtension(model, theme)); EXPECT_TRUE(::protos::GetExtension(model, theme).ok()); } @@ -808,10 +808,10 @@ TEST(CppGeneratedCode, SetExtensionShouldClone) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); extension1.set_ext_name("Goodbye"); - EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); EXPECT_EQ((*ext)->ext_name(), "Hello World"); @@ -821,12 +821,11 @@ TEST(CppGeneratedCode, SetExtensionShouldCloneConst) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ( - true, - ::protos::SetExtension(&model, theme, std::as_const(extension1)).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, + ::hpb::SetExtension(&model, theme, std::as_const(extension1)).ok()); extension1.set_ext_name("Goodbye"); - EXPECT_EQ(true, ::protos::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::HasExtension(&model, theme)); auto ext = ::protos::GetExtension(&model, theme); EXPECT_TRUE(ext.ok()); EXPECT_EQ((*ext)->ext_name(), "Hello World"); @@ -836,21 +835,19 @@ TEST(CppGeneratedCode, SetExtensionOnMutableChild) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, - ::protos::HasExtension(model.mutable_recursive_child(), theme)); - EXPECT_EQ(true, ::protos::SetExtension(model.mutable_recursive_child(), theme, - extension1) + EXPECT_EQ(false, ::hpb::HasExtension(model.mutable_recursive_child(), theme)); + EXPECT_EQ(true, ::hpb::SetExtension(model.mutable_recursive_child(), theme, + extension1) .ok()); - EXPECT_EQ(true, - ::protos::HasExtension(model.mutable_recursive_child(), theme)); + EXPECT_EQ(true, ::hpb::HasExtension(model.mutable_recursive_child(), theme)); } TEST(CppGeneratedCode, GetExtension) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(false, ::protos::HasExtension(&model, theme)); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(false, ::hpb::HasExtension(&model, theme)); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); EXPECT_EQ("Hello World", ::protos::GetExtension(&model, theme).value()->ext_name()); } @@ -859,12 +856,12 @@ TEST(CppGeneratedCode, GetExtensionOnMutableChild) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - ::protos::Ptr mutable_recursive_child = + ::hpb::Ptr mutable_recursive_child = model.mutable_recursive_child(); - EXPECT_EQ(false, ::protos::HasExtension(mutable_recursive_child, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(mutable_recursive_child, theme)); EXPECT_EQ( true, - ::protos::SetExtension(mutable_recursive_child, theme, extension1).ok()); + ::hpb::SetExtension(mutable_recursive_child, theme, extension1).ok()); EXPECT_EQ("Hello World", ::protos::GetExtension(mutable_recursive_child, theme) .value() @@ -875,13 +872,13 @@ TEST(CppGeneratedCode, GetExtensionOnImmutableChild) { TestModel model; ThemeExtension extension1; extension1.set_ext_name("Hello World"); - ::protos::Ptr mutable_recursive_child = + ::hpb::Ptr mutable_recursive_child = model.mutable_recursive_child(); - EXPECT_EQ(false, ::protos::HasExtension(mutable_recursive_child, theme)); + EXPECT_EQ(false, ::hpb::HasExtension(mutable_recursive_child, theme)); EXPECT_EQ( true, - ::protos::SetExtension(mutable_recursive_child, theme, extension1).ok()); - ::protos::Ptr recursive_child = model.recursive_child(); + ::hpb::SetExtension(mutable_recursive_child, theme, extension1).ok()); + ::hpb::Ptr recursive_child = model.recursive_child(); EXPECT_EQ("Hello World", ::protos::GetExtension(recursive_child, theme).value()->ext_name()); } @@ -898,8 +895,7 @@ TEST(CppGeneratedCode, SerializeUsingArena) { TEST(CppGeneratedCode, SerializeProxyUsingArena) { ::upb::Arena message_arena; - TestModel::Proxy model_proxy = - ::protos::CreateMessage(message_arena); + TestModel::Proxy model_proxy = ::hpb::CreateMessage(message_arena); model_proxy.set_str1("Hello World"); ::upb::Arena arena; absl::StatusOr bytes = @@ -913,7 +909,7 @@ TEST(CppGeneratedCode, SerializeNestedMessageUsingArena) { TestModel model; model.mutable_recursive_child()->set_str1("Hello World"); ::upb::Arena arena; - ::protos::Ptr child = model.recursive_child(); + ::hpb::Ptr child = model.recursive_child(); absl::StatusOr bytes = ::protos::Serialize(child, arena); EXPECT_EQ(true, bytes.ok()); TestModel parsed_model = ::protos::Parse(bytes.value()).value(); @@ -925,7 +921,7 @@ TEST(CppGeneratedCode, Parse) { model.set_str1("Test123"); ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); @@ -939,12 +935,11 @@ TEST(CppGeneratedCode, ParseIntoPtrToModel) { model.set_str1("Test123"); ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); - ::protos::Ptr parsed_model = - ::protos::CreateMessage(arena); + ::hpb::Ptr parsed_model = ::hpb::CreateMessage(arena); EXPECT_TRUE(::protos::Parse(parsed_model, bytes.value())); EXPECT_EQ("Test123", parsed_model->str1()); // Should return an extension even if we don't pass ExtensionRegistry @@ -957,14 +952,14 @@ TEST(CppGeneratedCode, ParseWithExtensionRegistry) { model.set_str1("Test123"); ThemeExtension extension1; extension1.set_ext_name("Hello World"); - EXPECT_EQ(true, ::protos::SetExtension(&model, theme, extension1).ok()); - EXPECT_EQ(true, ::protos::SetExtension( - &model, ThemeExtension::theme_extension, extension1) + EXPECT_EQ(true, ::hpb::SetExtension(&model, theme, extension1).ok()); + EXPECT_EQ(true, ::hpb::SetExtension(&model, ThemeExtension::theme_extension, + extension1) .ok()); ::upb::Arena arena; auto bytes = ::protos::Serialize(&model, arena); EXPECT_EQ(true, bytes.ok()); - ::protos::ExtensionRegistry extensions( + ::hpb::ExtensionRegistry extensions( {&theme, &other_ext, &ThemeExtension::theme_extension}, arena); TestModel parsed_model = ::protos::Parse(bytes.value(), extensions).value(); @@ -990,15 +985,15 @@ TEST(CppGeneratedCode, NameCollisions) { TEST(CppGeneratedCode, SharedPointer) { std::shared_ptr model = std::make_shared(); ::upb::Arena arena; - auto bytes = protos::Serialize(model.get(), arena); - EXPECT_TRUE(protos::Parse(model.get(), bytes.value())); + auto bytes = ::protos::Serialize(model.get(), arena); + EXPECT_TRUE(::protos::Parse(model.get(), bytes.value())); } TEST(CppGeneratedCode, UniquePointer) { auto model = std::make_unique(); ::upb::Arena arena; - auto bytes = protos::Serialize(model.get(), arena); - EXPECT_TRUE(protos::Parse(model.get(), bytes.value())); + auto bytes = ::protos::Serialize(model.get(), arena); + EXPECT_TRUE(::protos::Parse(model.get(), bytes.value())); } TEST(CppGeneratedCode, Assignment) { @@ -1037,21 +1032,21 @@ TEST(CppGeneratedCode, PtrConstructor) { TEST(CppGeneratedCode, MutableToProxy) { TestModel model; - ::protos::Ptr child = model.mutable_child_model_1(); + ::hpb::Ptr child = model.mutable_child_model_1(); (void)child; } TEST(CppGeneratedCode, ProxyToCProxy) { TestModel model; - ::protos::Ptr child = model.mutable_child_model_1(); - ::protos::Ptr child2 = child; + ::hpb::Ptr child = model.mutable_child_model_1(); + ::hpb::Ptr child2 = child; (void)child2; } TEST(CppGeneratedCode, MutableAccessorsAreHiddenInCProxy) { TestModel model; - ::protos::Ptr proxy = &model; - ::protos::Ptr cproxy = proxy; + ::hpb::Ptr proxy = &model; + ::hpb::Ptr cproxy = proxy; const auto test_const_accessors = [](auto p) { // We don't want to run it, just check it compiles. @@ -1103,7 +1098,7 @@ TEST(CppGeneratedCode, MutableAccessorsAreHiddenInCProxy) { test_mutable_accessors(cproxy, false); } -bool ProxyToCProxyMethod(::protos::Ptr child) { +bool ProxyToCProxyMethod(::hpb::Ptr child) { return child->child_str1() == "text in child"; } @@ -1116,7 +1111,7 @@ TEST(CppGeneratedCode, PassProxyToCProxy) { TEST(CppGeneratedCode, PtrImplicitConversion) { TestModel model; model.set_int64(5); - ::protos::Ptr model_ptr = &model; + ::hpb::Ptr model_ptr = &model; EXPECT_EQ(model_ptr->int64(), 5); } @@ -1128,10 +1123,10 @@ TEST(CppGeneratedCode, ClearSubMessage) { new_child->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - EXPECT_TRUE(::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_TRUE(::hpb::SetExtension(&model, theme, extension1).ok()); EXPECT_TRUE(model.mutable_child_model_1()->has_child_str1()); // Clear using Ptr - ::protos::ClearMessage(model.mutable_child_model_1()); + ::hpb::ClearMessage(model.mutable_child_model_1()); EXPECT_FALSE(model.mutable_child_model_1()->has_child_str1()); } @@ -1145,14 +1140,14 @@ TEST(CppGeneratedCode, ClearMessage) { new_child.value()->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - EXPECT_TRUE(::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_TRUE(::hpb::SetExtension(&model, theme, extension1).ok()); // Clear using T* - ::protos::ClearMessage(&model); + ::hpb::ClearMessage(&model); // Verify that scalars, repeated fields and extensions are cleared. EXPECT_FALSE(model.has_int64()); EXPECT_FALSE(model.has_str2()); EXPECT_TRUE(model.child_models().empty()); - EXPECT_FALSE(::protos::HasExtension(&model, theme)); + EXPECT_FALSE(::hpb::HasExtension(&model, theme)); } TEST(CppGeneratedCode, CanInvokeClearMessageWithPtr) { @@ -1161,8 +1156,8 @@ TEST(CppGeneratedCode, CanInvokeClearMessageWithPtr) { model.set_int64(5); auto new_child = model.add_child_models(); // Clear using Ptr - auto ptr = ::protos::Ptr(&model); - ::protos::ClearMessage(ptr); + auto ptr = ::hpb::Ptr(&model); + ::hpb::ClearMessage(ptr); // Successful clear EXPECT_FALSE(model.has_int64()); } @@ -1173,19 +1168,19 @@ TEST(CppGeneratedCode, CanInvokeClearMessageWithRawPtr) { model.set_int64(5); auto new_child = model.add_child_models(); // Clear using T* - ::protos::ClearMessage(&model); + ::hpb::ClearMessage(&model); // Successful clear EXPECT_FALSE(model.has_int64()); } template bool CanCallClearMessage() { - return Requires([](auto x) -> decltype(::protos::ClearMessage(x)) {}); + return Requires([](auto x) -> decltype(::hpb::ClearMessage(x)) {}); } TEST(CppGeneratedCode, CannotInvokeClearMessageWithConstPtr) { - EXPECT_TRUE(CanCallClearMessage<::protos::Ptr>()); - EXPECT_FALSE(CanCallClearMessage<::protos::Ptr>()); + EXPECT_TRUE(CanCallClearMessage<::hpb::Ptr>()); + EXPECT_FALSE(CanCallClearMessage<::hpb::Ptr>()); } TEST(CppGeneratedCode, CannotInvokeClearMessageWithConstRawPtr) { @@ -1203,13 +1198,13 @@ TEST(CppGeneratedCode, DeepCopy) { new_child.value()->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - EXPECT_TRUE(::protos::SetExtension(&model, theme, extension1).ok()); + EXPECT_TRUE(::hpb::SetExtension(&model, theme, extension1).ok()); TestModel target; target.set_b1(true); - ::protos::DeepCopy(&model, &target); + ::hpb::DeepCopy(&model, &target); EXPECT_FALSE(target.b1()) << "Target was not cleared before copying content "; EXPECT_EQ(target.str2(), "Hello"); - EXPECT_TRUE(::protos::HasExtension(&target, theme)); + EXPECT_TRUE(::hpb::HasExtension(&target, theme)); } TEST(CppGeneratedCode, HasExtensionAndRegistry) { @@ -1222,16 +1217,16 @@ TEST(CppGeneratedCode, HasExtensionAndRegistry) { new_child.value()->set_child_str1("text in child"); ThemeExtension extension1; extension1.set_ext_name("name in extension"); - ASSERT_TRUE(::protos::SetExtension(&source, theme, extension1).ok()); + ASSERT_TRUE(::hpb::SetExtension(&source, theme, extension1).ok()); // Now that we have a source model with extension data, serialize. ::hpb::Arena arena; std::string data = std::string(::protos::Serialize(&source, arena).value()); // Test with ExtensionRegistry - ::protos::ExtensionRegistry extensions({&theme}, arena); + ::hpb::ExtensionRegistry extensions({&theme}, arena); TestModel parsed_model = ::protos::Parse(data, extensions).value(); - EXPECT_TRUE(::protos::HasExtension(&parsed_model, theme)); + EXPECT_TRUE(::hpb::HasExtension(&parsed_model, theme)); } TEST(CppGeneratedCode, FieldNumberConstants) { @@ -1239,6 +1234,10 @@ TEST(CppGeneratedCode, FieldNumberConstants) { EXPECT_EQ(225, TestModel::kChildMapFieldNumber); } +TEST(CppGeneratedCode, ExtensionFieldNumberConstant) { + EXPECT_EQ(12003, ::protos::ExtensionNumber(ThemeExtension::theme_extension)); +} + TEST(CppGeneratedCode, ClearConstMessageShouldFailForConstChild) { TestModel model; EXPECT_FALSE(CanCallClearMessage()); diff --git a/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java b/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java index f3241de509..9bf1439626 100644 --- a/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java +++ b/java/core/src/main/java/com/google/protobuf/ArrayDecoders.java @@ -24,8 +24,7 @@ import java.io.IOException; @CheckReturnValue final class ArrayDecoders { - private ArrayDecoders() { - } + private ArrayDecoders() {} /** * A helper used to return multiple values in a Java function. Java doesn't natively support diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 7b2455f12d..801347e0fd 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -287,13 +287,12 @@ public abstract class ByteString implements Iterable, Serializable { while (formerBytes.hasNext() && latterBytes.hasNext()) { int result = - Integer.valueOf(toInt(formerBytes.nextByte())) - .compareTo(toInt(latterBytes.nextByte())); + Integer.compare(toInt(formerBytes.nextByte()), toInt(latterBytes.nextByte())); if (result != 0) { return result; } } - return Integer.valueOf(former.size()).compareTo(Integer.valueOf(latter.size())); + return Integer.compare(former.size(), latter.size()); } }; diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 569fa26e0e..bade681e1d 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -2343,7 +2343,7 @@ public final class Descriptors { new Comparator() { @Override public int compare(EnumValueDescriptor o1, EnumValueDescriptor o2) { - return Integer.valueOf(o1.getNumber()).compareTo(o2.getNumber()); + return Integer.compare(o1.getNumber(), o2.getNumber()); } }; diff --git a/java/core/src/main/java/com/google/protobuf/FieldSet.java b/java/core/src/main/java/com/google/protobuf/FieldSet.java index f536be2640..debad4e971 100644 --- a/java/core/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/FieldSet.java @@ -264,7 +264,8 @@ final class FieldSet> { /** * Useful for implementing {@link Message.Builder#setField(Descriptors.FieldDescriptor,Object)}. */ - @SuppressWarnings({"unchecked", "rawtypes"}) + // Avoid iterator allocation. + @SuppressWarnings({"ForeachList", "ForeachListWithUserVar"}) public void setField(final T descriptor, Object value) { if (descriptor.isRepeated()) { if (!(value instanceof List)) { @@ -274,10 +275,14 @@ final class FieldSet> { // Wrap the contents in a new list so that the caller cannot change // the list's contents after setting it. - final List newList = new ArrayList<>(); - newList.addAll((List) value); - for (final Object element : newList) { + List list = (List) value; + int listSize = list.size(); + // Avoid extra allocations: no iterator, no intermediate array copy. + final List newList = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + Object element = list.get(i); verifyType(descriptor, element); + newList.add(element); } value = newList; } else { @@ -512,7 +517,8 @@ final class FieldSet> { } } - @SuppressWarnings({"unchecked", "rawtypes"}) + // Avoid iterator allocation. + @SuppressWarnings({"unchecked", "ForeachList", "ForeachListWithUserVar"}) private void mergeFromField(final Map.Entry entry) { final T descriptor = entry.getKey(); Object otherValue = entry.getValue(); @@ -523,11 +529,16 @@ final class FieldSet> { throw new IllegalStateException("Lazy fields can not be repeated"); } Object value = getField(descriptor); + List otherList = (List) otherValue; + int otherListSize = otherList.size(); if (value == null) { - value = new ArrayList<>(); + value = new ArrayList<>(otherListSize); } - for (Object element : (List) otherValue) { - ((List) value).add(cloneIfMutable(element)); + List list = (List) value; + // Avoid iterator allocation. + for (int i = 0; i < otherListSize; i++) { + Object element = otherList.get(i); + list.add(cloneIfMutable(element)); } fields.put(descriptor, value); } else if (descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE) { diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java index e8a7cc8f97..d22b49820a 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -843,7 +843,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial * the generated API only allows us to access it as a map. This method returns the underlying * map field directly and thus enables us to access the map field as a list. */ - @SuppressWarnings({"unused", "rawtypes"}) + @SuppressWarnings("unused") protected MapFieldReflectionAccessor internalGetMapFieldReflection(int fieldNumber) { return internalGetMapField(fieldNumber); } @@ -858,7 +858,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial } /** Like {@link #internalGetMapFieldReflection} but return a mutable version. */ - @SuppressWarnings({"unused", "rawtypes"}) + @SuppressWarnings("unused") protected MapFieldReflectionAccessor internalGetMutableMapFieldReflection(int fieldNumber) { return internalGetMutableMapField(fieldNumber); } @@ -2054,8 +2054,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial new RepeatedEnumFieldAccessor( field, camelCaseNames[i], messageClass, builderClass); } else { - fields[i] = - new RepeatedFieldAccessor(field, camelCaseNames[i], messageClass, builderClass); + fields[i] = new RepeatedFieldAccessor(camelCaseNames[i], messageClass, builderClass); } } else { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { @@ -2611,7 +2610,6 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial protected final MethodInvoker invoker; RepeatedFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, final Class messageClass, final Class> builderClass) { @@ -2916,7 +2914,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial final String camelCaseName, final Class messageClass, final Class> builderClass) { - super(descriptor, camelCaseName, messageClass, builderClass); + super(camelCaseName, messageClass, builderClass); enumDescriptor = descriptor.getEnumType(); @@ -3112,7 +3110,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial final String camelCaseName, final Class messageClass, final Class> builderClass) { - super(descriptor, camelCaseName, messageClass, builderClass); + super(camelCaseName, messageClass, builderClass); newBuilderMethod = getMethodOrDie(type, "newBuilder"); getBuilderMethodBuilder = diff --git a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java index 5d10e48884..dbcb9e899d 100644 --- a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java +++ b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java @@ -132,7 +132,7 @@ public class InvalidProtocolBufferException extends IOException { static InvalidProtocolBufferException recursionLimitExceeded() { return new InvalidProtocolBufferException( "Protocol message had too many levels of nesting. May be malicious. " - + "Use CodedInputStream.setRecursionLimit() to increase the depth limit."); + + "Use setRecursionLimit() to increase the recursion depth limit."); } static InvalidProtocolBufferException sizeLimitExceeded() { diff --git a/java/core/src/main/java/com/google/protobuf/MessageSchema.java b/java/core/src/main/java/com/google/protobuf/MessageSchema.java index de3890f702..f8f79fcdf8 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageSchema.java +++ b/java/core/src/main/java/com/google/protobuf/MessageSchema.java @@ -3006,6 +3006,7 @@ final class MessageSchema implements Schema { unknownFields = unknownFieldSchema.getBuilderFromMessage(message); } // Unknown field. + if (unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader)) { continue; } @@ -3381,6 +3382,7 @@ final class MessageSchema implements Schema { if (unknownFields == null) { unknownFields = unknownFieldSchema.getBuilderFromMessage(message); } + if (!unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader)) { return; } @@ -3397,6 +3399,7 @@ final class MessageSchema implements Schema { if (unknownFields == null) { unknownFields = unknownFieldSchema.getBuilderFromMessage(message); } + if (!unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader)) { return; } diff --git a/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java b/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java index eec3acd35c..a17037e8ef 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java +++ b/java/core/src/main/java/com/google/protobuf/MessageSetSchema.java @@ -278,6 +278,7 @@ final class MessageSetSchema implements Schema { reader, extension, extensionRegistry, extensions); return true; } else { + return unknownFieldSchema.mergeOneFieldFrom(unknownFields, reader); } } else { diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index ddd51d0675..8771403b1f 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -449,11 +449,11 @@ public final class TextFormat { } switch (fieldType) { case BOOLEAN: - return Boolean.valueOf((boolean) getKey()).compareTo((boolean) b.getKey()); + return ((Boolean) getKey()).compareTo((Boolean) b.getKey()); case LONG: - return Long.valueOf((long) getKey()).compareTo((long) b.getKey()); + return ((Long) getKey()).compareTo((Long) b.getKey()); case INT: - return Integer.valueOf((int) getKey()).compareTo((int) b.getKey()); + return ((Integer) getKey()).compareTo((Integer) b.getKey()); case STRING: String aString = (String) getKey(); String bString = (String) b.getKey(); diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java index c4ec645bf7..a43bc2a947 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSchema.java @@ -55,7 +55,6 @@ abstract class UnknownFieldSchema { /** Marks unknown fields as immutable. */ abstract void makeImmutable(Object message); - /** Merges one field into the unknown fields. */ final boolean mergeOneFieldFrom(B unknownFields, Reader reader) throws IOException { int tag = reader.getTag(); int fieldNumber = WireFormat.getTagFieldNumber(tag); @@ -88,7 +87,7 @@ abstract class UnknownFieldSchema { } } - final void mergeFrom(B unknownFields, Reader reader) throws IOException { + private final void mergeFrom(B unknownFields, Reader reader) throws IOException { while (true) { if (reader.getFieldNumber() == Reader.READ_DONE || !mergeOneFieldFrom(unknownFields, reader)) { diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java index f34a448c51..58ac736d4a 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java @@ -80,7 +80,7 @@ public class UnknownFieldSetTest { // ================================================================= @Test - public void testFieldBuildersAreReusable() { + public void testFixed32FieldBuildersAreReusable() { UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); fieldBuilder.addFixed32(10); UnknownFieldSet.Field first = fieldBuilder.build(); @@ -92,6 +92,64 @@ public class UnknownFieldSetTest { assertThat(first).isNotEqualTo(third); } + @Test + public void testFixed64FieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addFixed64(10); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addFixed64(11); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + + @Test + public void testVarintFieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addVarint(10); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addVarint(11); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + + @Test + public void testLengthDelimitedFieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addLengthDelimited(ByteString.copyFromUtf8("foo")); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addLengthDelimited(ByteString.copyFromUtf8("bar")); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + + @Test + public void testGroupFieldBuildersAreReusable() { + UnknownFieldSet.Field.Builder fieldBuilder = UnknownFieldSet.Field.newBuilder(); + fieldBuilder.addGroup( + UnknownFieldSet.newBuilder() + .addField(10, UnknownFieldSet.Field.newBuilder().addVarint(10).build()) + .build()); + UnknownFieldSet.Field first = fieldBuilder.build(); + UnknownFieldSet.Field second = fieldBuilder.build(); + fieldBuilder.addGroup( + UnknownFieldSet.newBuilder() + .addField(11, UnknownFieldSet.Field.newBuilder().addVarint(11).build()) + .build()); + UnknownFieldSet.Field third = fieldBuilder.build(); + + assertThat(first).isEqualTo(second); + assertThat(first).isNotEqualTo(third); + } + @Test public void testClone() { UnknownFieldSet.Builder unknownSetBuilder = UnknownFieldSet.newBuilder(); diff --git a/java/kotlin/BUILD.bazel b/java/kotlin/BUILD.bazel index 3af561b74d..7faced6715 100644 --- a/java/kotlin/BUILD.bazel +++ b/java/kotlin/BUILD.bazel @@ -276,6 +276,7 @@ kt_jvm_library( "//java/core:test_util", "@maven//:com_google_truth_truth", "@maven//:junit_junit", + "@rules_kotlin//kotlin/compiler:kotlin-reflect", ], ) diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt index 7c319bfaac..754a3294cc 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt @@ -26,12 +26,14 @@ import protobuf_unittest.UnittestProto.TestAllTypes.NestedEnum import protobuf_unittest.UnittestProto.TestEmptyMessage import protobuf_unittest.UnittestProto.TestEmptyMessageWithExtensions import protobuf_unittest.copy +import protobuf_unittest.deprecatedMessageOrNull import protobuf_unittest.foreignMessage import protobuf_unittest.optionalGroupExtension import protobuf_unittest.optionalNestedMessageOrNull import protobuf_unittest.repeatedGroupExtension import protobuf_unittest.testAllExtensions import protobuf_unittest.testAllTypes +import protobuf_unittest.testDeprecatedFields import protobuf_unittest.testEmptyMessage import protobuf_unittest.testEmptyMessageWithExtensions import protobuf_unittest.testEnumMap @@ -959,4 +961,15 @@ class Proto2Test { assertThat(someNestedMessage.optionalNestedMessageOrNull) .isEqualTo(TestAllTypesKt.nestedMessage { bb = 118 }) } + + @Test + fun testDeprecated() { + val testInstance = + protobuf_unittest.UnittestProto.TestDeprecatedFields.getDefaultInstance() + assertThat(testInstance::deprecatedMessageOrNull.annotations.any { it is Deprecated }).isTrue() + + val unused = testDeprecatedFields { + assertThat(::deprecatedMessage.annotations.any { it is Deprecated }).isTrue() + } + } } diff --git a/java/lite/src/test/java/com/google/protobuf/LiteTest.java b/java/lite/src/test/java/com/google/protobuf/LiteTest.java index 0c7b8b535b..411bc63f08 100644 --- a/java/lite/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/lite/src/test/java/com/google/protobuf/LiteTest.java @@ -10,12 +10,14 @@ package com.google.protobuf; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static java.util.Collections.singletonList; +import static org.junit.Assert.assertThrows; import com.google.protobuf.FieldPresenceTestProto.TestAllTypes; import com.google.protobuf.UnittestImportLite.ImportEnumLite; import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; import com.google.protobuf.UnittestLite.ForeignEnumLite; import com.google.protobuf.UnittestLite.ForeignMessageLite; +import com.google.protobuf.UnittestLite.RecursiveGroup; import com.google.protobuf.UnittestLite.RecursiveMessage; import com.google.protobuf.UnittestLite.TestAllExtensionsLite; import com.google.protobuf.UnittestLite.TestAllTypesLite; @@ -50,6 +52,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 5092588132..97bdbb0b70 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -1790,7 +1790,7 @@ public class JsonFormat { try { BigDecimal decimalValue = new BigDecimal(json.getAsString()); BigInteger value = decimalValue.toBigIntegerExact(); - if (value.signum() < 0 || value.compareTo(new BigInteger("FFFFFFFF", 16)) > 0) { + if (value.signum() < 0 || value.compareTo(MAX_UINT32) > 0) { throw new InvalidProtocolBufferException("Out of range uint32 value: " + json); } return value.intValue(); @@ -1802,6 +1802,7 @@ public class JsonFormat { } } + private static final BigInteger MAX_UINT32 = new BigInteger("FFFFFFFF", 16); private static final BigInteger MAX_UINT64 = new BigInteger("FFFFFFFFFFFFFFFF", 16); private long parseUint64(JsonElement json) throws InvalidProtocolBufferException { diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m index c39648b588..4ee03ef580 100644 --- a/objectivec/GPBCodedInputStream.m +++ b/objectivec/GPBCodedInputStream.m @@ -13,6 +13,10 @@ #import "GPBUtilities_PackagePrivate.h" #import "GPBWireFormat.h" +// TODO: Consider using on other functions to reduce bloat when +// some compiler optimizations are enabled. +#define GPB_NOINLINE __attribute__((noinline)) + NSString *const GPBCodedInputStreamException = GPBNSStringifySymbol(GPBCodedInputStreamException); NSString *const GPBCodedInputStreamUnderlyingErrorKey = @@ -28,7 +32,8 @@ NSString *const GPBCodedInputStreamErrorDomain = // int CodedInputStream::default_recursion_limit_ = 100; static const NSUInteger kDefaultRecursionLimit = 100; -static void RaiseException(NSInteger code, NSString *reason) { +GPB_NOINLINE +void GPBRaiseStreamError(NSInteger code, NSString *reason) { NSDictionary *errorInfo = nil; if ([reason length]) { errorInfo = @{GPBErrorReasonKey : reason}; @@ -44,7 +49,7 @@ static void RaiseException(NSInteger code, NSString *reason) { GPB_INLINE void CheckRecursionLimit(GPBCodedInputStreamState *state) { if (state->recursionDepth >= kDefaultRecursionLimit) { - RaiseException(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorRecursionDepthExceeded, nil); } } @@ -56,19 +61,19 @@ GPB_INLINE void CheckFieldSize(uint64_t size) { if (size > 0x7fffffff) { // TODO: Maybe a different error code for this, but adding one is a breaking // change so reuse an existing one. - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidSize, nil); } } static void CheckSize(GPBCodedInputStreamState *state, size_t size) { size_t newSize = state->bufferPos + size; if (newSize > state->bufferSize) { - RaiseException(GPBCodedInputStreamErrorInvalidSize, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidSize, nil); } if (newSize > state->currentLimit) { // Fast forward to end of currentLimit; state->bufferPos = state->currentLimit; - RaiseException(GPBCodedInputStreamErrorSubsectionLimitReached, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorSubsectionLimitReached, nil); } } @@ -110,7 +115,7 @@ static int64_t ReadRawVarint64(GPBCodedInputStreamState *state) { } shift += 7; } - RaiseException(GPBCodedInputStreamErrorInvalidVarInt, @"Invalid VarInt64"); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidVarInt, @"Invalid VarInt64"); return 0; } @@ -201,12 +206,12 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) { state->lastTag = ReadRawVarint32(state); // Tags have to include a valid wireformat. if (!GPBWireFormatIsValidTag(state->lastTag)) { - RaiseException(GPBCodedInputStreamErrorInvalidTag, @"Invalid wireformat in tag."); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Invalid wireformat in tag."); } // Zero is not a valid field number. if (GPBWireFormatGetTagFieldNumber(state->lastTag) == 0) { - RaiseException(GPBCodedInputStreamErrorInvalidTag, - @"A zero field number on the wire is invalid."); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, + @"A zero field number on the wire is invalid."); } return state->lastTag; } @@ -231,7 +236,7 @@ NSString *GPBCodedInputStreamReadRetainedString(GPBCodedInputStreamState *state) NSLog(@"UTF-8 failure, is some field type 'string' when it should be " @"'bytes'?"); #endif - RaiseException(GPBCodedInputStreamErrorInvalidUTF8, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidUTF8, nil); } } return result; @@ -262,11 +267,65 @@ NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *sta return result; } +static void SkipToEndGroupInternal(GPBCodedInputStreamState *state, uint32_t endGroupTag) { + CheckRecursionLimit(state); + ++state->recursionDepth; + while (YES) { + uint32_t tag = GPBCodedInputStreamReadTag(state); + if (tag == endGroupTag || tag == 0) { + GPBCodedInputStreamCheckLastTagWas(state, endGroupTag); // Will fail for end of input. + --state->recursionDepth; + return; + } + switch (GPBWireFormatGetTagWireType(tag)) { + case GPBWireFormatVarint: + (void)ReadRawVarint64(state); + break; + case GPBWireFormatFixed64: + SkipRawData(state, sizeof(uint64_t)); + break; + case GPBWireFormatLengthDelimited: { + uint64_t size = ReadRawVarint64(state); + CheckFieldSize(size); + size_t size2 = (size_t)size; // Cast safe on 32bit because of CheckFieldSize() above. + SkipRawData(state, size2); + break; + } + case GPBWireFormatStartGroup: + SkipToEndGroupInternal(state, GPBWireFormatMakeTag(GPBWireFormatGetTagFieldNumber(tag), + GPBWireFormatEndGroup)); + break; + case GPBWireFormatEndGroup: + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unmatched end group"); + break; + case GPBWireFormatFixed32: + SkipRawData(state, sizeof(uint32_t)); + break; + } + } +} + +// This doesn't include the start group, but it collects all bytes until the end group including +// the end group tag. +NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state, + int32_t fieldNumber) { + // Better have just read the start of the group. + GPBCodedInputStreamCheckLastTagWas(state, + GPBWireFormatMakeTag(fieldNumber, GPBWireFormatStartGroup)); + const uint8_t *start = state->bytes + state->bufferPos; + SkipToEndGroupInternal(state, GPBWireFormatMakeTag(fieldNumber, GPBWireFormatEndGroup)); + // This will be after the end group tag. + const uint8_t *end = state->bytes + state->bufferPos; + return [[NSData alloc] initWithBytesNoCopy:(void *)start + length:(NSUInteger)(end - start) + freeWhenDone:NO]; +} + size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit) { byteLimit += state->bufferPos; size_t oldLimit = state->currentLimit; if (byteLimit > oldLimit) { - RaiseException(GPBCodedInputStreamErrorInvalidSubsectionLimit, nil); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidSubsectionLimit, nil); } state->currentLimit = byteLimit; return oldLimit; @@ -286,7 +345,7 @@ BOOL GPBCodedInputStreamIsAtEnd(GPBCodedInputStreamState *state) { void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state, int32_t value) { if (state->lastTag != value) { - RaiseException(GPBCodedInputStreamErrorInvalidTag, @"Unexpected tag read"); + GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unexpected tag read"); } } diff --git a/objectivec/GPBCodedInputStream_PackagePrivate.h b/objectivec/GPBCodedInputStream_PackagePrivate.h index fbeecf130c..720bd60ee8 100644 --- a/objectivec/GPBCodedInputStream_PackagePrivate.h +++ b/objectivec/GPBCodedInputStream_PackagePrivate.h @@ -53,6 +53,7 @@ typedef struct GPBCodedInputStreamState { CF_EXTERN_C_BEGIN +void GPBRaiseStreamError(NSInteger code, NSString *reason); int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state); double GPBCodedInputStreamReadDouble(GPBCodedInputStreamState *state); @@ -75,6 +76,9 @@ NSData *GPBCodedInputStreamReadRetainedBytes(GPBCodedInputStreamState *state) __attribute((ns_returns_retained)); NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *state) __attribute((ns_returns_retained)); +NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state, + int32_t fieldNumber) + __attribute((ns_returns_retained)); size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit); void GPBCodedInputStreamPopLimit(GPBCodedInputStreamState *state, size_t oldLimit); diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 9ef625b01f..b0367699dc 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -2085,6 +2085,7 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { - (BOOL)mergeFromData:(NSData *)data extensionRegistry:(nullable id)extensionRegistry error:(NSError **)errorPtr { + GPBBecomeVisibleToAutocreator(self); GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; @try { [self mergeFromCodedInputStream:input extensionRegistry:extensionRegistry endingTag:0]; @@ -3683,4 +3684,23 @@ id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { return expected; } +NSData *GPBMessageUnknownFieldsData(GPBMessage *self) { + NSData *result = nil; + GPBUnknownFieldSet *unknownFields = self->unknownFields_; + if (unknownFields) { + if (self.descriptor.isWireFormat) { + NSMutableData *mutableData = + [NSMutableData dataWithLength:unknownFields.serializedSizeAsMessageSet]; + GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; + [unknownFields writeAsMessageSetTo:output]; + [output flush]; + [output release]; + result = mutableData; + } else { + result = [unknownFields data]; + } + } + return result; +} + #pragma clang diagnostic pop diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index 65a1a7c5ba..58649c44f7 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -81,4 +81,7 @@ void GPBAutocreatedDictionaryModified(GPBMessage *self, id dictionary); // autocreated reference to this message. void GPBClearMessageAutocreator(GPBMessage *self); +// The data (or null) from the unknown fields of a message; +NSData *GPBMessageUnknownFieldsData(GPBMessage *self); + CF_EXTERN_C_END diff --git a/objectivec/GPBUnknownFieldSet.m b/objectivec/GPBUnknownFieldSet.m index 5fe8ce03c4..10fb80c0c7 100644 --- a/objectivec/GPBUnknownFieldSet.m +++ b/objectivec/GPBUnknownFieldSet.m @@ -154,7 +154,10 @@ static void CopyWorker(__unused const void *key, const void *value, void *contex - (NSString *)description { NSMutableString *description = [NSMutableString stringWithFormat:@"<%@ %p>: TextFormat: {\n", [self class], self]; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" NSString *textFormat = GPBTextFormatForUnknownFieldSet(self, @" "); +#pragma clang diagnostic pop [description appendString:textFormat]; [description appendString:@"}"]; return description; diff --git a/objectivec/GPBUnknownFields.m b/objectivec/GPBUnknownFields.m index f3073ad16d..836e249f67 100644 --- a/objectivec/GPBUnknownFields.m +++ b/objectivec/GPBUnknownFields.m @@ -14,6 +14,7 @@ #import "GPBCodedOutputStream_PackagePrivate.h" #import "GPBDescriptor.h" #import "GPBMessage.h" +#import "GPBMessage_PackagePrivate.h" #import "GPBUnknownField.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" @@ -196,22 +197,8 @@ static BOOL MergeFromInputStream(GPBUnknownFields *self, GPBCodedInputStream *in self = [super init]; if (self) { fields_ = [[NSMutableArray alloc] init]; - // TODO: b/349146447 - Move off the legacy class and directly to the data once Message is - // updated. - GPBUnknownFieldSet *legacyUnknownFields = [message unknownFields]; - if (legacyUnknownFields) { - NSData *data; - if (message.descriptor.isWireFormat) { - NSMutableData *mutableData = - [NSMutableData dataWithLength:legacyUnknownFields.serializedSizeAsMessageSet]; - GPBCodedOutputStream *output = [[GPBCodedOutputStream alloc] initWithData:mutableData]; - [legacyUnknownFields writeAsMessageSetTo:output]; - [output flush]; - [output release]; - data = mutableData; - } else { - data = [legacyUnknownFields data]; - } + NSData *data = GPBMessageUnknownFieldsData(message); + if (data) { GPBCodedInputStream *input = [[GPBCodedInputStream alloc] initWithData:data]; // Parse until the end of the data (tag will be zero). if (!MergeFromInputStream(self, input, 0)) { diff --git a/objectivec/GPBUtilities.h b/objectivec/GPBUtilities.h index 9673aa4261..578545df05 100644 --- a/objectivec/GPBUtilities.h +++ b/objectivec/GPBUtilities.h @@ -40,7 +40,9 @@ NSString *GPBTextFormatForMessage(GPBMessage *message, NSString *__nullable line * @return An NSString with the TextFormat of the unknown field set. **/ NSString *GPBTextFormatForUnknownFieldSet(GPBUnknownFieldSet *__nullable unknownSet, - NSString *__nullable lineIndent); + NSString *__nullable lineIndent) + __attribute__((deprecated("Use GPBTextFormatForMessage will include the unknown fields, and " + "GPBUnknownFieldSet it going away."))); /** * Checks if the given field number is set on a message. diff --git a/objectivec/GPBUtilities.m b/objectivec/GPBUtilities.m index ff309860cd..0764c96910 100644 --- a/objectivec/GPBUtilities.m +++ b/objectivec/GPBUtilities.m @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#import "GPBUtilities_PackagePrivate.h" +#import "GPBUtilities.h" #import @@ -15,6 +15,9 @@ #import "GPBMessage_PackagePrivate.h" #import "GPBUnknownField.h" #import "GPBUnknownFieldSet.h" +#import "GPBUnknownField_PackagePrivate.h" +#import "GPBUnknownFields.h" +#import "GPBUtilities_PackagePrivate.h" // Direct access is use for speed, to avoid even internally declaring things // read/write, etc. The warning is enabled in the project to ensure code calling @@ -1929,6 +1932,78 @@ static void AppendTextFormatForMessageExtensionRange(GPBMessage *message, NSArra } // for..in(activeExtensions) } +static void AppendTextFormatForUnknownFields(GPBUnknownFields *ufs, NSMutableString *toStr, + NSString *lineIndent) { +#if defined(DEBUG) && DEBUG + NSCAssert(!ufs.empty, @"Internal Error: No unknown fields to format."); +#endif + // Extract the fields and sort them by field number. Use a stable sort and sort just by the field + // numbers, that way the output will still show the order the different types were added as well + // as maintaining the order within a give number/type pair (i.e. - repeated fields say in order). + NSMutableArray *sortedFields = [[NSMutableArray alloc] initWithCapacity:ufs.count]; + for (GPBUnknownField *field in ufs) { + [sortedFields addObject:field]; + } + [sortedFields + sortWithOptions:NSSortStable + usingComparator:^NSComparisonResult(GPBUnknownField *field1, GPBUnknownField *field2) { + int32_t fieldNumber1 = field1->number_; + int32_t fieldNumber2 = field2->number_; + if (fieldNumber1 < fieldNumber2) { + return NSOrderedAscending; + } else if (fieldNumber1 > fieldNumber2) { + return NSOrderedDescending; + } else { + return NSOrderedSame; + } + }]; + + NSString *subIndent = nil; + + for (GPBUnknownField *field in sortedFields) { + int32_t fieldNumber = field->number_; + switch (field->type_) { + case GPBUnknownFieldTypeVarint: + [toStr appendFormat:@"%@%d: %llu\n", lineIndent, fieldNumber, field->storage_.intValue]; + break; + case GPBUnknownFieldTypeFixed32: + [toStr appendFormat:@"%@%d: 0x%X\n", lineIndent, fieldNumber, + (uint32_t)field->storage_.intValue]; + break; + case GPBUnknownFieldTypeFixed64: + [toStr appendFormat:@"%@%d: 0x%llX\n", lineIndent, fieldNumber, field->storage_.intValue]; + break; + case GPBUnknownFieldTypeLengthDelimited: + [toStr appendFormat:@"%@%d: ", lineIndent, fieldNumber]; + AppendBufferAsString(field->storage_.lengthDelimited, toStr); + [toStr appendString:@"\n"]; + break; + case GPBUnknownFieldTypeGroup: { + GPBUnknownFields *group = field->storage_.group; + if (group.empty) { + [toStr appendFormat:@"%@%d: {}\n", lineIndent, fieldNumber]; + } else { + [toStr appendFormat:@"%@%d: {\n", lineIndent, fieldNumber]; + if (subIndent == nil) { + subIndent = [lineIndent stringByAppendingString:@" "]; + } + AppendTextFormatForUnknownFields(group, toStr, subIndent); + [toStr appendFormat:@"%@}\n", lineIndent]; + } + } break; + case GPBUnknownFieldTypeLegacy: +#if defined(DEBUG) && DEBUG + NSCAssert( + NO, + @"Internal error: Shouldn't have gotten a legacy field type in the unknown fields."); +#endif + break; + } + } + [subIndent release]; + [sortedFields release]; +} + static void AppendTextFormatForMessage(GPBMessage *message, NSMutableString *toStr, NSString *lineIndent) { GPBDescriptor *descriptor = [message descriptor]; @@ -1951,11 +2026,12 @@ static void AppendTextFormatForMessage(GPBMessage *message, NSMutableString *toS } } - NSString *unknownFieldsStr = GPBTextFormatForUnknownFieldSet(message.unknownFields, lineIndent); - if ([unknownFieldsStr length] > 0) { + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:message]; + if (ufs.count > 0) { [toStr appendFormat:@"%@# --- Unknown fields ---\n", lineIndent]; - [toStr appendString:unknownFieldsStr]; + AppendTextFormatForUnknownFields(ufs, toStr, lineIndent); } + [ufs release]; } NSString *GPBTextFormatForMessage(GPBMessage *message, NSString *lineIndent) { diff --git a/objectivec/Tests/GPBCodedInputStreamTests.m b/objectivec/Tests/GPBCodedInputStreamTests.m index 3dae80d2ac..29802eb15a 100644 --- a/objectivec/Tests/GPBCodedInputStreamTests.m +++ b/objectivec/Tests/GPBCodedInputStreamTests.m @@ -6,12 +6,16 @@ // https://developers.google.com/open-source/licenses/bsd #import -#import "GPBTestUtilities.h" #import "GPBCodedInputStream.h" +#import "GPBCodedInputStream_PackagePrivate.h" #import "GPBCodedOutputStream.h" +#import "GPBTestUtilities.h" +#import "GPBUnknownField.h" #import "GPBUnknownFieldSet_PackagePrivate.h" +#import "GPBUnknownFields.h" #import "GPBUtilities_PackagePrivate.h" +#import "GPBWireFormat.h" #import "objectivec/Tests/Unittest.pbobjc.h" @interface CodedInputStreamTests : GPBTestCase @@ -266,20 +270,55 @@ TestAllTypes* message = [self allSetRepeatedCount:kGPBDefaultRepeatCount]; NSData* rawBytes = message.data; - // Create two parallel inputs. Parse one as unknown fields while using - // skipField() to skip each field on the other. Expect the same tags. + TestEmptyMessage* empty = [TestEmptyMessage parseFromData:rawBytes error:NULL]; + XCTAssertNotNil(empty); + GPBUnknownFields* ufs = [[[GPBUnknownFields alloc] initFromMessage:empty] autorelease]; + NSMutableArray* fieldNumbers = [NSMutableArray arrayWithCapacity:ufs.count]; + for (GPBUnknownField* field in ufs) { + GPBWireFormat wireFormat; + switch (field.type) { + case GPBUnknownFieldTypeFixed32: + wireFormat = GPBWireFormatFixed32; + break; + case GPBUnknownFieldTypeFixed64: + wireFormat = GPBWireFormatFixed64; + break; + case GPBUnknownFieldTypeVarint: + wireFormat = GPBWireFormatVarint; + break; + case GPBUnknownFieldTypeLengthDelimited: + wireFormat = GPBWireFormatLengthDelimited; + break; + case GPBUnknownFieldTypeGroup: + wireFormat = GPBWireFormatStartGroup; + break; + case GPBUnknownFieldTypeLegacy: + XCTFail(@"Legacy field type not expected"); + wireFormat = GPBWireFormatVarint; + break; + } + uint32_t tag = GPBWireFormatMakeTag(field.number, wireFormat); + [fieldNumbers addObject:@(tag)]; + } + + // Check the tags compared to what's in the UnknownFields to confirm the stream is + // skipping as expected (this covers the tags within a group also). GPBCodedInputStream* input1 = [GPBCodedInputStream streamWithData:rawBytes]; GPBCodedInputStream* input2 = [GPBCodedInputStream streamWithData:rawBytes]; GPBUnknownFieldSet* unknownFields = [[[GPBUnknownFieldSet alloc] init] autorelease]; + NSUInteger idx = 0; while (YES) { int32_t tag = [input1 readTag]; XCTAssertEqual(tag, [input2 readTag]); if (tag == 0) { + XCTAssertEqual(idx, fieldNumbers.count); break; } + XCTAssertEqual(tag, [fieldNumbers[idx] intValue]); [unknownFields mergeFieldFrom:tag input:input1]; [input2 skipField:tag]; + ++idx; } } @@ -444,4 +483,102 @@ } } +- (void)assertReadByteToEndGroupFails:(NSData*)data { + GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data]; + uint32_t tag = [input readTag]; + XCTAssertThrows(GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy( + &input->state_, GPBWireFormatGetTagFieldNumber(tag))); +} + +- (void)assertReadByteToEndGroup:(NSData*)data value:(NSData*)value { + GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data]; + uint32_t tag = [input readTag]; + NSData* readValue = GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy( + &input->state_, GPBWireFormatGetTagFieldNumber(tag)); + XCTAssertNotNil(readValue); + XCTAssertEqualObjects(readValue, value); + [readValue release]; +} + +static NSData* DataForGroupsOfDepth(NSUInteger depth) { + NSMutableData* data = [NSMutableData dataWithCapacity:0]; + + uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + + byte = 8; // 8 = 0b1000, -> field 1/varint + [data appendBytes:&byte length:1]; + byte = 1; // 1 -> varint value of 1 + [data appendBytes:&byte length:1]; + + byte = 36; // 36 = 0b100100 -> field 4/end group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + return data; +} + +- (void)testBytesToEndGroup { + // 35 = 0b100011 -> field 4/start group + // 36 = 0b100100 -> field 4/end group + // 43 = 0b101011 -> field 5/end group + // 44 = 0b101100 -> field 5/end group + // 8 = 0b1000, 1 -> field 1/varint, value of 1 + // 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678 + // 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64, + // value of 0x123456789abcdef0LL + // 50 = 0b110010, 0x0 -> field 6/length delimited, length 0 + // 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42 + // 0 -> field 0 which is invalid/varint + // 15 = 0b1111 -> field 1, wire type 7 which is invalid + + [self assertReadByteToEndGroup:bytes(35, 36) value:bytes(36)]; // empty group + [self assertReadByteToEndGroup:bytes(35, 8, 1, 36) value:bytes(8, 1, 36)]; // varint + [self assertReadByteToEndGroup:bytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) // fixed32 + value:bytes(21, 0x78, 0x56, 0x34, 0x12, 36)]; + [self assertReadByteToEndGroup:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, + 36) // fixed64 + value:bytes(25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, 36)]; + [self assertReadByteToEndGroup:bytes(35, 50, 0, 36) + value:bytes(50, 0, 36)]; // length delimited, length 0 + [self assertReadByteToEndGroup:bytes(35, 50, 1, 42, 36) + value:bytes(50, 1, 42, 36)]; // length delimited, length 1, byte 42 + [self assertReadByteToEndGroup:bytes(35, 43, 44, 36) value:bytes(43, 44, 36)]; // Sub group + [self assertReadByteToEndGroup:bytes(35, 8, 1, 43, 8, 1, 44, + 36) // varint and sub group with varint + value:bytes(8, 1, 43, 8, 1, 44, 36)]; + + [self assertReadByteToEndGroupFails:bytes(35, 0, 36)]; // Invalid field number + [self assertReadByteToEndGroupFails:bytes(35, 15, 36)]; // Invalid wire type + [self assertReadByteToEndGroupFails:bytes(35, 21, 0x78, 0x56, 0x34)]; // truncated fixed32 + [self assertReadByteToEndGroupFails:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, + 0x34)]; // truncated fixed64 + + // Mising end group + [self assertReadByteToEndGroupFails:bytes(35)]; + [self assertReadByteToEndGroupFails:bytes(35, 8, 1)]; + [self assertReadByteToEndGroupFails:bytes(35, 43)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1)]; + + // Wrong end group + [self assertReadByteToEndGroupFails:bytes(35, 44)]; + [self assertReadByteToEndGroupFails:bytes(35, 8, 1, 44)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 36)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 36)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 44, 44)]; + [self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 44, 44)]; + + // This is the same limit as within GPBCodedInputStream. + const NSUInteger kDefaultRecursionLimit = 100; + // That depth parses. + NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit); + [self assertReadByteToEndGroup:testData + value:[testData subdataWithRange:NSMakeRange(1, testData.length - 1)]]; + // One more level deep fails. + testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1); + [self assertReadByteToEndGroupFails:testData]; +} + @end diff --git a/objectivec/Tests/GPBMessageTests+Serialization.m b/objectivec/Tests/GPBMessageTests+Serialization.m index f3e8abce15..b42bfdd564 100644 --- a/objectivec/Tests/GPBMessageTests+Serialization.m +++ b/objectivec/Tests/GPBMessageTests+Serialization.m @@ -419,6 +419,17 @@ // All the values should be in unknown fields. + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:msg]; + XCTAssertEqual(ufs.count, 3U); + uint64_t varint; + XCTAssertTrue([ufs getFirst:Message2_FieldNumber_OptionalEnum varint:&varint]); + XCTAssertEqual(varint, (uint64_t)Message3_Enum_Extra3); + XCTAssertTrue([ufs getFirst:Message2_FieldNumber_RepeatedEnumArray varint:&varint]); + XCTAssertEqual(varint, (uint64_t)Message3_Enum_Extra3); + XCTAssertTrue([ufs getFirst:Message2_FieldNumber_OneofEnum varint:&varint]); + XCTAssertEqual(varint, (uint64_t)Message3_Enum_Extra3); + [ufs release]; + GPBUnknownFieldSet *unknownFields = msg.unknownFields; XCTAssertEqual([unknownFields countOfFields], 3U); @@ -1398,6 +1409,9 @@ int32_t val = -1; XCTAssertTrue([msg1.knownMapField getEnum:&val forKey:0]); XCTAssertEqual(val, Proto2MapEnum_Proto2MapEnumFoo); + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:msg1]; + XCTAssertEqual(ufs.count, 1U); + [ufs release]; XCTAssertEqual(msg1.unknownFields.countOfFields, 1U); data = [msg1 data]; @@ -1410,6 +1424,9 @@ XCTAssertEqual(msg2.unknownMapField.count, 1U); XCTAssertTrue([msg2.unknownMapField getEnum:&val forKey:0]); XCTAssertEqual(val, Proto2MapEnumPlusExtra_EProto2MapEnumExtra); + ufs = [[GPBUnknownFields alloc] initFromMessage:msg2]; + XCTAssertTrue(ufs.empty); + [ufs release]; XCTAssertEqual(msg2.unknownFields.countOfFields, 0U); XCTAssertEqualObjects(orig, msg2); diff --git a/objectivec/Tests/GPBMessageTests.m b/objectivec/Tests/GPBMessageTests.m index d598a48b60..f3272dbaf2 100644 --- a/objectivec/Tests/GPBMessageTests.m +++ b/objectivec/Tests/GPBMessageTests.m @@ -5,16 +5,18 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#import "GPBTestUtilities.h" - #import +#import "GPBArray.h" #import "GPBArray_PackagePrivate.h" #import "GPBDescriptor.h" #import "GPBDictionary_PackagePrivate.h" #import "GPBMessage_PackagePrivate.h" +#import "GPBTestUtilities.h" +#import "GPBUnknownField.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" +#import "GPBUnknownFields.h" #import "objectivec/Tests/Unittest.pbobjc.h" #import "objectivec/Tests/UnittestImport.pbobjc.h" #import "objectivec/Tests/UnittestObjc.pbobjc.h" @@ -501,6 +503,11 @@ [message setUnknownFields:unknownFields]; + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:1234 fixed32:1234]; + [ufs addFieldNumber:2345 varint:54321]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; + NSString *description = [message description]; XCTAssertGreaterThan([description length], 0U); @@ -985,6 +992,19 @@ XCTAssertFalse([message hasOptionalNestedMessage]); [message.optionalNestedMessage setUnknownFields:unknownFields]; XCTAssertTrue([message hasOptionalNestedMessage]); + + message.optionalNestedMessage = nil; + XCTAssertFalse([message hasOptionalNestedMessage]); + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:1 varint:1]; + [message.optionalNestedMessage mergeUnknownFields:ufs extensionRegistry:nil]; + XCTAssertTrue([message hasOptionalNestedMessage]); + + message.optionalNestedMessage = nil; + XCTAssertFalse([message hasOptionalNestedMessage]); + [ufs clear]; // Also make sure merging zero length forces it to become visible. + [message.optionalNestedMessage mergeUnknownFields:ufs extensionRegistry:nil]; + XCTAssertTrue([message hasOptionalNestedMessage]); } - (void)testSetAutocreatedSubmessageToSelf { @@ -1481,6 +1501,19 @@ XCTAssertFalse([msg hasExtension:[UnittestRoot repeatedNestedEnumExtension]]); XCTAssertFalse([msg hasExtension:[UnittestRoot repeatedForeignEnumExtension]]); + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] initFromMessage:msg] autorelease]; + XCTAssertEqual(ufs.count, 3); + uint64_t varint; + XCTAssertTrue([ufs getFirst:[UnittestRoot optionalNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 10); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 11); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedForeignEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 12); + GPBUnknownFieldSet *unknownFields = msg.unknownFields; GPBUnknownField *field = [unknownFields getField:[UnittestRoot optionalNestedEnumExtension].fieldNumber]; @@ -1523,6 +1556,18 @@ expected = @[ @4, @6 ]; XCTAssertEqualObjects([msg getExtension:[UnittestRoot repeatedForeignEnumExtension]], expected); + ufs = [[[GPBUnknownFields alloc] initFromMessage:msg] autorelease]; + XCTAssertEqual(ufs.count, 3); + XCTAssertTrue([ufs getFirst:[UnittestRoot optionalNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 10); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedNestedEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 11); + XCTAssertTrue([ufs getFirst:[UnittestRoot repeatedForeignEnumExtension].fieldNumber + varint:&varint]); + XCTAssertEqual(varint, 12); + unknownFields = msg.unknownFields; field = [unknownFields getField:[UnittestRoot optionalNestedEnumExtension].fieldNumber]; XCTAssertNotNil(field); @@ -1840,6 +1885,9 @@ [unknowns mergeVarintField:123 value:456]; GPBMessage *message = [GPBMessage message]; [message setUnknownFields:unknowns]; + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:1234 varint:5678]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; NSData *data = [message data]; GPBMessage *message2 = [GPBMessage parseFromData:data extensionRegistry:nil error:NULL]; XCTAssertEqualObjects(message, message2); @@ -1850,12 +1898,19 @@ [unknowns1 mergeVarintField:123 value:456]; GPBMessage *message1 = [GPBMessage message]; [message1 setUnknownFields:unknowns1]; + GPBUnknownFields *ufs1 = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs1 addFieldNumber:1234 varint:5678]; + [message1 mergeUnknownFields:ufs1 extensionRegistry:nil]; GPBUnknownFieldSet *unknowns2 = [[[GPBUnknownFieldSet alloc] init] autorelease]; [unknowns2 mergeVarintField:789 value:987]; [unknowns2 mergeVarintField:654 value:321]; GPBMessage *message2 = [GPBMessage message]; [message2 setUnknownFields:unknowns2]; + GPBUnknownFields *ufs2 = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs2 addFieldNumber:2345 fixed32:6789]; + [ufs2 addFieldNumber:3456 fixed32:7890]; + [message2 mergeUnknownFields:ufs2 extensionRegistry:nil]; NSMutableData *delimitedData = [NSMutableData data]; [delimitedData appendData:[message1 delimitedData]]; @@ -2022,6 +2077,54 @@ XCTAssertEqual([msgPrime.mumbleArray valueAtIndex:4], EnumTestMsg_MyEnum_NegTwo); } +- (void)testCloseEnumsValuesOutOfRange { + // The unknown values should all make it into the unknown fields. + EnumTestMsg *msg1 = [EnumTestMsg message]; + msg1.bar = EnumTestMsg_MyEnum_NegTwo; + msg1.baz = EnumTestMsg_MyEnum_Two; + [msg1.mumbleArray addValue:EnumTestMsg_MyEnum_Two]; + [msg1.mumbleArray addValue:EnumTestMsg_MyEnum_NegTwo]; + + NSData *data = [msg1 data]; + XCTAssertNotNil(data); + + EnumTestMsgPrime *msg2 = [EnumTestMsgPrime parseFromData:data error:NULL]; + XCTAssertNotNil(msg2); + XCTAssertEqualObjects(data, [msg2 data]); + XCTAssertFalse(msg2.hasBar); + XCTAssertFalse(msg2.hasBaz); + XCTAssertEqual(msg2.mumbleArray_Count, 0U); + + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] initFromMessage:msg2] autorelease]; + XCTAssertEqual(ufs.count, 4U); + uint64_t varint; + XCTAssertTrue([ufs getFirst:EnumTestMsg_FieldNumber_Bar varint:&varint]); + XCTAssertEqual(varint, (uint64_t)EnumTestMsg_MyEnum_NegTwo); + XCTAssertTrue([ufs getFirst:EnumTestMsg_FieldNumber_Baz varint:&varint]); + XCTAssertEqual(varint, (uint64_t)EnumTestMsg_MyEnum_Two); + NSArray *fields = [ufs fields:EnumTestMsg_FieldNumber_MumbleArray]; + XCTAssertEqual(fields.count, 2U); + XCTAssertEqual(fields[0].varint, (uint64_t)EnumTestMsg_MyEnum_Two); + XCTAssertEqual(fields[1].varint, (uint64_t)EnumTestMsg_MyEnum_NegTwo); + + GPBUnknownFieldSet *unknownFields = msg2.unknownFields; + XCTAssertNotNil(unknownFields); + XCTAssertEqual(unknownFields.countOfFields, 3U); + XCTAssertTrue([unknownFields hasField:EnumTestMsg_FieldNumber_Bar]); + XCTAssertTrue([unknownFields hasField:EnumTestMsg_FieldNumber_Baz]); + XCTAssertTrue([unknownFields hasField:EnumTestMsg_FieldNumber_MumbleArray]); + GPBUnknownField *field = [unknownFields getField:EnumTestMsg_FieldNumber_Bar]; + XCTAssertEqual(field.varintList.count, 1U); + XCTAssertEqual([field.varintList valueAtIndex:0], (uint64_t)EnumTestMsg_MyEnum_NegTwo); + field = [unknownFields getField:EnumTestMsg_FieldNumber_Baz]; + XCTAssertEqual(field.varintList.count, 1U); + XCTAssertEqual([field.varintList valueAtIndex:0], (uint64_t)EnumTestMsg_MyEnum_Two); + field = [unknownFields getField:EnumTestMsg_FieldNumber_MumbleArray]; + XCTAssertEqual(field.varintList.count, 2U); + XCTAssertEqual([field.varintList valueAtIndex:0], (uint64_t)EnumTestMsg_MyEnum_Two); + XCTAssertEqual([field.varintList valueAtIndex:1], (uint64_t)EnumTestMsg_MyEnum_NegTwo); +} + - (void)testReservedWordNaming { // names.cc has some special handing to make sure that some "reserved" objc // names get renamed in a way so they don't conflict. diff --git a/objectivec/Tests/GPBUnknownFieldSetTest.m b/objectivec/Tests/GPBUnknownFieldSetTest.m index 10478ac772..890ce8f4ad 100644 --- a/objectivec/Tests/GPBUnknownFieldSetTest.m +++ b/objectivec/Tests/GPBUnknownFieldSetTest.m @@ -6,6 +6,7 @@ // https://developers.google.com/open-source/licenses/bsd #import "GPBTestUtilities.h" +#import "GPBUnknownFieldSet.h" #import "GPBUnknownFieldSet_PackagePrivate.h" #import "GPBUnknownField_PackagePrivate.h" @@ -281,6 +282,10 @@ TestEmptyMessage* destination2 = [TestEmptyMessage message]; [destination2 mergeFrom:source3]; + XCTAssertEqualObjects(destination1.unknownFields, destination2.unknownFields); + XCTAssertEqualObjects(destination1.unknownFields, source3.unknownFields); + XCTAssertEqualObjects(destination2.unknownFields, source3.unknownFields); + XCTAssertEqualObjects(destination1.data, destination2.data); XCTAssertEqualObjects(destination1.data, source3.data); XCTAssertEqualObjects(destination2.data, source3.data); @@ -376,6 +381,180 @@ XCTAssertEqual(0x7FFFFFFFFFFFFFFFULL, [field2.varintList valueAtIndex:0]); } +static NSData* DataForGroupsOfDepth(NSUInteger depth) { + NSMutableData* data = [NSMutableData dataWithCapacity:0]; + + uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + + byte = 8; // 8 = 0b1000, -> field 1/varint + [data appendBytes:&byte length:1]; + byte = 1; // 1 -> varint value of 1 + [data appendBytes:&byte length:1]; + + byte = 36; // 36 = 0b100100 -> field 4/end group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + return data; +} + +- (void)testParsingNestingGroupData { + // 35 = 0b100011 -> field 4/start group + // 36 = 0b100100 -> field 4/end group + // 43 = 0b101011 -> field 5/end group + // 44 = 0b101100 -> field 5/end group + // 8 = 0b1000, 1 -> field 1/varint, value of 1 + // 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678 + // 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64, + // value of 0x123456789abcdef0LL + // 50 = 0b110010, 0x0 -> field 6/length delimited, length 0 + // 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42 + // 0 -> field 0 which is invalid/varint + // 15 = 0b1111 -> field 1, wire type 7 which is invalid + + TestEmptyMessage* m = [TestEmptyMessage parseFromData:DataFromBytes(35, 36) + error:NULL]; // empty group + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + GPBUnknownField* field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + GPBUnknownFieldSet* group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)0); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 36) error:NULL]; // varint + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:1]; + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], 1); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) + error:NULL]; // fixed32 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:2]; + XCTAssertEqual(field.fixed32List.count, (NSUInteger)1); + XCTAssertEqual([field.fixed32List valueAtIndex:0], 0x12345678); + + m = [TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, + 36) + error:NULL]; // fixed64 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:3]; + XCTAssertEqual(field.fixed64List.count, (NSUInteger)1); + XCTAssertEqual([field.fixed64List valueAtIndex:0], 0x123456789abcdef0LL); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36) + error:NULL]; // length delimited, length 0 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:6]; + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1); + XCTAssertEqualObjects(field.lengthDelimitedList[0], [NSData data]); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 1, 42, 36) + error:NULL]; // length delimited, length 1, byte 42 + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:6]; + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1); + XCTAssertEqualObjects(field.lengthDelimitedList[0], DataFromBytes(42)); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 36) error:NULL]; // Sub group + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)1); + field = [group getField:5]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)0); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 43, 8, 2, 44, 36) + error:NULL]; // varint and sub group with varint + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)2); + field = [group getField:1]; + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], 1); + field = [group getField:5]; + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + field = [group getField:1]; + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], 2); + + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 0, 36) + error:NULL]); // Invalid field number + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 15, 36) + error:NULL]); // Invalid wire type + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34) + error:NULL]); // truncated fixed32 + XCTAssertNil([TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, + 0x34) + error:NULL]); // truncated fixed64 + + // Mising end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1) error:NULL]); + + // Wrong end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 44, 44) error:NULL]); + + // This is the same limit as within GPBCodedInputStream. + const NSUInteger kDefaultRecursionLimit = 100; + // That depth parses. + NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit); + m = [TestEmptyMessage parseFromData:testData error:NULL]; + XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1); + field = [m.unknownFields getField:4]; + for (NSUInteger i = 0; i < kDefaultRecursionLimit; ++i) { + XCTAssertEqual(field.varintList.count, (NSUInteger)0); + XCTAssertEqual(field.fixed32List.count, (NSUInteger)0); + XCTAssertEqual(field.fixed64List.count, (NSUInteger)0); + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0); + XCTAssertEqual(field.groupList.count, (NSUInteger)1); + group = field.groupList[0]; + XCTAssertEqual(group.countOfFields, (NSUInteger)1); + field = [group getField:(i < (kDefaultRecursionLimit - 1) ? 4 : 1)]; + } + // field is of the inner most group + XCTAssertEqual(field.varintList.count, (NSUInteger)1); + XCTAssertEqual([field.varintList valueAtIndex:0], (NSUInteger)1); + XCTAssertEqual(field.fixed32List.count, (NSUInteger)0); + XCTAssertEqual(field.fixed64List.count, (NSUInteger)0); + XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0); + XCTAssertEqual(field.groupList.count, (NSUInteger)0); + // One more level deep fails. + testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1); + XCTAssertNil([TestEmptyMessage parseFromData:testData error:NULL]); +} + #pragma mark - Field tests // Some tests directly on fields since the dictionary in FieldSet can gate // testing some of these. diff --git a/objectivec/Tests/GPBUnknownFieldsTest.m b/objectivec/Tests/GPBUnknownFieldsTest.m index 9726d8361f..8c2adcf32c 100644 --- a/objectivec/Tests/GPBUnknownFieldsTest.m +++ b/objectivec/Tests/GPBUnknownFieldsTest.m @@ -847,4 +847,163 @@ XCTAssertEqual(varint, 0x7FFFFFFFFFFFFFFFL); } +static NSData* DataForGroupsOfDepth(NSUInteger depth) { + NSMutableData* data = [NSMutableData dataWithCapacity:0]; + + uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + + byte = 8; // 8 = 0b1000, -> field 1/varint + [data appendBytes:&byte length:1]; + byte = 1; // 1 -> varint value of 1 + [data appendBytes:&byte length:1]; + + byte = 36; // 36 = 0b100100 -> field 4/end group + for (NSUInteger i = 0; i < depth; ++i) { + [data appendBytes:&byte length:1]; + } + return data; +} + +- (void)testParsingNestingGroupData { + // 35 = 0b100011 -> field 4/start group + // 36 = 0b100100 -> field 4/end group + // 43 = 0b101011 -> field 5/end group + // 44 = 0b101100 -> field 5/end group + // 8 = 0b1000, 1 -> field 1/varint, value of 1 + // 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678 + // 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64, + // value of 0x123456789abcdef0LL + // 50 = 0b110010, 0x0 -> field 6/length delimited, length 0 + // 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42 + // 0 -> field 0 which is invalid/varint + // 15 = 0b1111 -> field 1, wire type 7 which is invalid + + TestEmptyMessage* m = [TestEmptyMessage parseFromData:DataFromBytes(35, 36) + error:NULL]; // empty group + GPBUnknownFields* ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + GPBUnknownFields* group = [ufs firstGroup:4]; + XCTAssertTrue(group.empty); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 36) error:NULL]; // varint + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + uint64_t varint = 0; + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 1); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) + error:NULL]; // fixed32 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + uint32_t fixed32 = 0; + XCTAssertTrue([group getFirst:2 fixed32:&fixed32]); + XCTAssertEqual(fixed32, 0x12345678); + + m = [TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, + 36) + error:NULL]; // fixed64 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + uint64_t fixed64 = 0; + XCTAssertTrue([group getFirst:3 fixed64:&fixed64]); + XCTAssertEqual(fixed64, 0x123456789abcdef0LL); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36) + error:NULL]; // length delimited, length 0 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + NSData* lengthDelimited = [group firstLengthDelimited:6]; + XCTAssertEqualObjects(lengthDelimited, [NSData data]); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 1, 42, 36) + error:NULL]; // length delimited, length 1, byte 42 + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + lengthDelimited = [group firstLengthDelimited:6]; + XCTAssertEqualObjects(lengthDelimited, DataFromBytes(42)); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 36) error:NULL]; // Sub group + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)1); + group = [group firstGroup:5]; + XCTAssertTrue(group.empty); + + m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 43, 8, 2, 44, 36) + error:NULL]; // varint and sub group with varint + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + XCTAssertEqual(group.count, (NSUInteger)2); + varint = 0; + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 1); + group = [group firstGroup:5]; + XCTAssertEqual(group.count, (NSUInteger)1); + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 2); + + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 0, 36) + error:NULL]); // Invalid field number + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 15, 36) + error:NULL]); // Invalid wire type + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34) + error:NULL]); // truncated fixed32 + XCTAssertNil([TestEmptyMessage + parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, + 0x34) + error:NULL]); // truncated fixed64 + + // Mising end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1) error:NULL]); + + // Wrong end group + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 36) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 44) error:NULL]); + XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 44, 44) error:NULL]); + + // This is the same limit as within GPBCodedInputStream. + const NSUInteger kDefaultRecursionLimit = 100; + // That depth parses. + NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit); + m = [TestEmptyMessage parseFromData:testData error:NULL]; + ufs = [[[GPBUnknownFields alloc] initFromMessage:m] autorelease]; + XCTAssertEqual(ufs.count, (NSUInteger)1); + group = [ufs firstGroup:4]; + for (NSUInteger i = 1; i < kDefaultRecursionLimit; ++i) { + XCTAssertEqual(group.count, (NSUInteger)1); + group = [group firstGroup:4]; + } + // group is now the inner most group. + XCTAssertEqual(group.count, (NSUInteger)1); + varint = 0; + XCTAssertTrue([group getFirst:1 varint:&varint]); + XCTAssertEqual(varint, 1); + // One more level deep fails. + testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1); + XCTAssertNil([TestEmptyMessage parseFromData:testData error:NULL]); +} + @end diff --git a/objectivec/Tests/GPBUtilitiesTests.m b/objectivec/Tests/GPBUtilitiesTests.m index e32fccda46..fb0175ecca 100644 --- a/objectivec/Tests/GPBUtilitiesTests.m +++ b/objectivec/Tests/GPBUtilitiesTests.m @@ -6,17 +6,16 @@ // https://developers.google.com/open-source/licenses/bsd #import - -#import "GPBUtilities_PackagePrivate.h" - #import -#import "GPBTestUtilities.h" - #import "GPBDescriptor.h" #import "GPBDescriptor_PackagePrivate.h" #import "GPBMessage.h" +#import "GPBTestUtilities.h" +#import "GPBUnknownField.h" #import "GPBUnknownField_PackagePrivate.h" +#import "GPBUtilities.h" +#import "GPBUtilities_PackagePrivate.h" #import "objectivec/Tests/MapUnittest.pbobjc.h" #import "objectivec/Tests/Unittest.pbobjc.h" @@ -168,6 +167,54 @@ [expected release]; } +- (void)testTextFormatUnknownFields { + GPBUnknownFields *ufs = [[[GPBUnknownFields alloc] init] autorelease]; + [ufs addFieldNumber:100 varint:5]; + [ufs addFieldNumber:100 varint:4]; + [ufs addFieldNumber:10 varint:1]; + [ufs addFieldNumber:300 fixed32:0x50]; + [ufs addFieldNumber:300 fixed32:0x40]; + [ufs addFieldNumber:10 fixed32:0x10]; + [ufs addFieldNumber:200 fixed64:0x5000]; + [ufs addFieldNumber:200 fixed64:0x4000]; + [ufs addFieldNumber:10 fixed64:0x1000]; + [ufs addFieldNumber:10 lengthDelimited:DataFromCStr("foo")]; + [ufs addFieldNumber:10 lengthDelimited:DataFromCStr("bar")]; + GPBUnknownFields *group = [ufs addGroupWithFieldNumber:150]; + [group addFieldNumber:2 varint:2]; + [group addFieldNumber:1 varint:1]; + group = [ufs addGroupWithFieldNumber:150]; + [group addFieldNumber:1 varint:1]; + [group addFieldNumber:3 fixed32:0x3]; + [group addFieldNumber:2 fixed64:0x2]; + TestEmptyMessage *message = [TestEmptyMessage message]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; + + NSString *expected = @"# --- Unknown fields ---\n" + @"10: 1\n" + @"10: 0x10\n" + @"10: 0x1000\n" + @"10: \"foo\"\n" + @"10: \"bar\"\n" + @"100: 5\n" + @"100: 4\n" + @"150: {\n" + @" 1: 1\n" + @" 2: 2\n" + @"}\n" + @"150: {\n" + @" 1: 1\n" + @" 2: 0x2\n" + @" 3: 0x3\n" + @"}\n" + @"200: 0x5000\n" + @"200: 0x4000\n" + @"300: 0x50\n" + @"300: 0x40\n"; + NSString *result = GPBTextFormatForMessage(message, nil); + XCTAssertEqualObjects(expected, result); +} + - (void)testSetRepeatedFields { TestAllTypes *message = [TestAllTypes message]; @@ -204,11 +251,27 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { return result; } +// Helper to add an unknown field data to messages. +static void AddUnknownFields(GPBMessage *message, int num) { + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] init]; + [ufs addFieldNumber:num varint:num]; + [message mergeUnknownFields:ufs extensionRegistry:nil]; + [ufs release]; +} + +static BOOL HasUnknownFields(GPBMessage *message) { + GPBUnknownFields *ufs = [[GPBUnknownFields alloc] initFromMessage:message]; + BOOL result = !ufs.empty; + [ufs release]; + return result; +} + - (void)testDropMessageUnknownFieldsRecursively { TestAllExtensions *message = [TestAllExtensions message]; // Give it unknownFields. message.unknownFields = UnknownFieldsSetHelper(777); + AddUnknownFields(message, 1777); // Given it extensions that include a message with unknown fields of its own. { @@ -219,18 +282,21 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { OptionalGroup_extension *optionalGroup = [OptionalGroup_extension message]; optionalGroup.a = 123; optionalGroup.unknownFields = UnknownFieldsSetHelper(779); + AddUnknownFields(optionalGroup, 1779); [message setExtension:[UnittestRoot optionalGroupExtension] value:optionalGroup]; // Message TestAllTypes_NestedMessage *nestedMessage = [TestAllTypes_NestedMessage message]; nestedMessage.bb = 456; nestedMessage.unknownFields = UnknownFieldsSetHelper(778); + AddUnknownFields(nestedMessage, 1778); [message setExtension:[UnittestRoot optionalNestedMessageExtension] value:nestedMessage]; // Repeated Group RepeatedGroup_extension *repeatedGroup = [[RepeatedGroup_extension alloc] init]; repeatedGroup.a = 567; repeatedGroup.unknownFields = UnknownFieldsSetHelper(780); + AddUnknownFields(repeatedGroup, 1780); [message addExtension:[UnittestRoot repeatedGroupExtension] value:repeatedGroup]; [repeatedGroup release]; @@ -238,6 +304,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { nestedMessage = [[TestAllTypes_NestedMessage alloc] init]; nestedMessage.bb = 678; nestedMessage.unknownFields = UnknownFieldsSetHelper(781); + AddUnknownFields(nestedMessage, 1781); [message addExtension:[UnittestRoot repeatedNestedMessageExtension] value:nestedMessage]; [nestedMessage release]; } @@ -245,6 +312,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { // Confirm everything is there. XCTAssertNotNil(message); + XCTAssertTrue(HasUnknownFields(message)); XCTAssertNotNil(message.unknownFields); XCTAssertTrue([message hasExtension:[UnittestRoot optionalInt32Extension]]); @@ -254,6 +322,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalGroupExtension]]; XCTAssertNotNil(optionalGroup); XCTAssertEqual(optionalGroup.a, 123); + XCTAssertTrue(HasUnknownFields(optionalGroup)); XCTAssertNotNil(optionalGroup.unknownFields); } @@ -263,6 +332,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalNestedMessageExtension]]; XCTAssertNotNil(nestedMessage); XCTAssertEqual(nestedMessage.bb, 456); + XCTAssertTrue(HasUnknownFields(nestedMessage)); XCTAssertNotNil(nestedMessage.unknownFields); } @@ -273,6 +343,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { RepeatedGroup_extension *repeatedGroup = repeatedGroups.firstObject; XCTAssertNotNil(repeatedGroup); XCTAssertEqual(repeatedGroup.a, 567); + XCTAssertTrue(HasUnknownFields(repeatedGroup)); XCTAssertNotNil(repeatedGroup.unknownFields); } @@ -284,6 +355,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { TestAllTypes_NestedMessage *repeatedNestedMessage = repeatedNestedMessages.firstObject; XCTAssertNotNil(repeatedNestedMessage); XCTAssertEqual(repeatedNestedMessage.bb, 678); + XCTAssertTrue(HasUnknownFields(repeatedNestedMessage)); XCTAssertNotNil(repeatedNestedMessage.unknownFields); } @@ -293,6 +365,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { // Confirm unknowns are gone from within the messages. XCTAssertNotNil(message); + XCTAssertFalse(HasUnknownFields(message)); XCTAssertNil(message.unknownFields); XCTAssertTrue([message hasExtension:[UnittestRoot optionalInt32Extension]]); @@ -302,6 +375,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalGroupExtension]]; XCTAssertNotNil(optionalGroup); XCTAssertEqual(optionalGroup.a, 123); + XCTAssertFalse(HasUnknownFields(optionalGroup)); XCTAssertNil(optionalGroup.unknownFields); } @@ -311,6 +385,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { [message getExtension:[UnittestRoot optionalNestedMessageExtension]]; XCTAssertNotNil(nestedMessage); XCTAssertEqual(nestedMessage.bb, 456); + XCTAssertFalse(HasUnknownFields(nestedMessage)); XCTAssertNil(nestedMessage.unknownFields); } @@ -321,6 +396,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { RepeatedGroup_extension *repeatedGroup = repeatedGroups.firstObject; XCTAssertNotNil(repeatedGroup); XCTAssertEqual(repeatedGroup.a, 567); + XCTAssertFalse(HasUnknownFields(repeatedGroup)); XCTAssertNil(repeatedGroup.unknownFields); } @@ -332,6 +408,7 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { TestAllTypes_NestedMessage *repeatedNestedMessage = repeatedNestedMessages.firstObject; XCTAssertNotNil(repeatedNestedMessage); XCTAssertEqual(repeatedNestedMessage.bb, 678); + XCTAssertFalse(HasUnknownFields(repeatedNestedMessage)); XCTAssertNil(repeatedNestedMessage.unknownFields); } } @@ -342,10 +419,12 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { { ForeignMessage *foreignMessage = [ForeignMessage message]; foreignMessage.unknownFields = UnknownFieldsSetHelper(100); + AddUnknownFields(foreignMessage, 1000); [message.mapInt32ForeignMessage setObject:foreignMessage forKey:100]; foreignMessage = [ForeignMessage message]; foreignMessage.unknownFields = UnknownFieldsSetHelper(101); + AddUnknownFields(foreignMessage, 1001); [message.mapStringForeignMessage setObject:foreignMessage forKey:@"101"]; } @@ -356,12 +435,14 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { { ForeignMessage *foreignMessage = [message.mapInt32ForeignMessage objectForKey:100]; XCTAssertNotNil(foreignMessage); + XCTAssertTrue(HasUnknownFields(foreignMessage)); XCTAssertNotNil(foreignMessage.unknownFields); } { ForeignMessage *foreignMessage = [message.mapStringForeignMessage objectForKey:@"101"]; XCTAssertNotNil(foreignMessage); + XCTAssertTrue(HasUnknownFields(foreignMessage)); XCTAssertNotNil(foreignMessage.unknownFields); } @@ -374,12 +455,14 @@ static GPBUnknownFieldSet *UnknownFieldsSetHelper(int num) { { ForeignMessage *foreignMessage = [message.mapInt32ForeignMessage objectForKey:100]; XCTAssertNotNil(foreignMessage); + XCTAssertFalse(HasUnknownFields(foreignMessage)); XCTAssertNil(foreignMessage.unknownFields); } { ForeignMessage *foreignMessage = [message.mapStringForeignMessage objectForKey:@"101"]; XCTAssertNotNil(foreignMessage); + XCTAssertFalse(HasUnknownFields(foreignMessage)); XCTAssertNil(foreignMessage.unknownFields); } } diff --git a/objectivec/Tests/unittest_objc.proto b/objectivec/Tests/unittest_objc.proto index 4f0e348569..b7b67fc372 100644 --- a/objectivec/Tests/unittest_objc.proto +++ b/objectivec/Tests/unittest_objc.proto @@ -7,11 +7,11 @@ syntax = "proto2"; +package objc.protobuf.tests; + import "google/protobuf/any.proto"; import "objectivec/Tests/unittest.proto"; -package objc.protobuf.tests; - // Explicit empty prefix, tests some validations code paths also. option objc_class_prefix = ""; @@ -78,143 +78,142 @@ message self { } enum autorelease { - retain = 1; - release = 2; + retain = 1; + release = 2; retainCount = 3; } // Singular // Objective C Keywords - optional bool id = 1; - optional bool _cmd = 2; + optional bool id = 1; + optional bool _cmd = 2; // super is used as submessage above - optional bool in = 4; - optional bool out = 5; - optional bool inout = 6; - optional bool bycopy = 7; - optional bool byref = 8; - optional bool oneway = 9; - optional bool self = 10; - optional bool instancetype = 11; - optional bool nullable = 12; - optional bool nonnull = 13; - optional bool nil = 14; + optional bool in = 4; + optional bool out = 5; + optional bool inout = 6; + optional bool bycopy = 7; + optional bool byref = 8; + optional bool oneway = 9; + optional bool self = 10; + optional bool instancetype = 11; + optional bool nullable = 12; + optional bool nonnull = 13; + optional bool nil = 14; // Nil and nil can't be in the same message - optional bool YES = 16; - optional bool NO = 17; - optional bool weak = 18; + optional bool YES = 16; + optional bool NO = 17; + optional bool weak = 18; // Some C/C++ Keywords - optional bool case = 30; - optional bool if = 31; - optional bool and_eq = 32; - optional bool public = 33; - optional bool private = 34; - optional bool typename = 35; - optional bool static_cast = 36; - optional bool typeof = 37; - optional bool restrict = 38; - optional bool NULL = 39; + optional bool case = 30; + optional bool if = 31; + optional bool and_eq = 32; + optional bool public = 33; + optional bool private = 34; + optional bool typename = 35; + optional bool static_cast = 36; + optional bool typeof = 37; + optional bool restrict = 38; + optional bool NULL = 39; // Some NSObject Methods - optional bool dealloc = 110; - optional bool isProxy = 111; - optional bool copy = 112; - optional bool description = 113; - optional bool zone = 114; - optional bool className = 115; - optional bool __retain_OA = 116; - optional bool CAMLType = 117; - optional bool isNSDictionary__ = 118; + optional bool dealloc = 110; + optional bool isProxy = 111; + optional bool copy = 112; + optional bool description = 113; + optional bool zone = 114; + optional bool className = 115; + optional bool __retain_OA = 116; + optional bool CAMLType = 117; + optional bool isNSDictionary__ = 118; optional bool accessibilityLabel = 119; // Some Objc "keywords" that we shouldn't // have to worry about because they // can only appear in specialized areas. - optional bool assign = 200; - optional bool getter = 201; - optional bool setter = 202; - optional bool atomic = 203; - optional bool nonatomic = 204; - optional bool strong = 205; - optional bool null_resettable = 206; - optional bool readonly = 207; + optional bool assign = 200; + optional bool getter = 201; + optional bool setter = 202; + optional bool atomic = 203; + optional bool nonatomic = 204; + optional bool strong = 205; + optional bool null_resettable = 206; + optional bool readonly = 207; // Some GPBMessage methods - optional bool clear = 300; - optional bool data = 301; - optional bool descriptor = 302; - optional bool delimitedData = 303; + optional bool clear = 300; + optional bool data = 301; + optional bool descriptor = 302; + optional bool delimitedData = 303; // Some MacTypes - optional bool Fixed = 400; - optional bool Point = 401; - optional bool FixedPoint = 402; - optional bool Style = 403; + optional bool Fixed = 400; + optional bool Point = 401; + optional bool FixedPoint = 402; + optional bool Style = 403; // C/C++ reserved identifiers - optional bool _Generic = 500; - optional bool __block = 501; + optional bool _Generic = 500; + optional bool __block = 501; // Try a keyword as a type - optional autorelease SubEnum = 1000; + optional autorelease SubEnum = 1000; optional group New = 2000 { - optional string copy = 1; + optional string copy = 1; } optional group MutableCopy = 2001 { optional int32 extensionRegistry = 1; } extensions 3000 to 3999; - } enum retain { - count = 4; - initialized = 5; + count = 4; + initialized = 5; serializedSize = 6; } message ObjCPropertyNaming { // Test that the properties properly get things all caps. - optional string url = 1; + optional string url = 1; optional string thumbnail_url = 2; - optional string url_foo = 3; + optional string url_foo = 3; optional string some_url_blah = 4; - optional string http = 5; - optional string https = 6; + optional string http = 5; + optional string https = 6; // This one doesn't. - repeated string urls = 7; + repeated string urls = 7; } // EnumValueShortName: The short names shouldn't get suffixes/prefixes. enum Foo { SERIALIZED_SIZE = 1; - SIZE = 2; - OTHER = 3; + SIZE = 2; + OTHER = 3; } // EnumValueShortName: The enum name gets a prefix. enum Category { - RED = 1; + RED = 1; BLUE = 2; } // EnumValueShortName: Twist case, full name gets PB, but the short names // should still end up correct. enum Time { - BASE = 1; - RECORD = 2; - SOMETHING_ELSE = 3; + BASE = 1; + RECORD = 2; + SOMETHING_ELSE = 3; } extend self { - repeated int32 debugDescription = 3000 [packed = true]; - repeated int64 finalize = 3001 [packed = true]; - repeated uint32 hash = 3002 [packed = true]; - repeated uint64 classForCoder = 3003 [packed = true]; - repeated sint32 byref = 3004 [packed = true]; + repeated int32 debugDescription = 3000 [packed = true]; + repeated int64 finalize = 3001 [packed = true]; + repeated uint32 hash = 3002 [packed = true]; + repeated uint64 classForCoder = 3003 [packed = true]; + repeated sint32 byref = 3004 [packed = true]; } // Test handing of fields that start with init*. @@ -700,13 +699,17 @@ message JustToScopeExtensions { repeated string mutableCopy_val_lower_complex_repeated = 2711; repeated string mutableCopy_Val_upper_complex_repeated = 2712; - repeated string mutableCopyvalue_lower_no_underscore_complex_repeated = 2713; - repeated string mutableCopyValue_upper_no_underscore_complex_repeated = 2714; + repeated string mutableCopyvalue_lower_no_underscore_complex_repeated = + 2713; + repeated string mutableCopyValue_upper_no_underscore_complex_repeated = + 2714; repeated int32 mutableCopy_val_lower_primitive_repeated = 2715; repeated int32 mutableCopy_Val_upper_primitive_repeated = 2716; - repeated int32 mutableCopyvalue_lower_no_underscore_primitive_repeated = 2717; - repeated int32 mutableCopyValue_upper_no_underscore_primitive_repeated = 2718; + repeated int32 mutableCopyvalue_lower_no_underscore_primitive_repeated = + 2717; + repeated int32 mutableCopyValue_upper_no_underscore_primitive_repeated = + 2718; repeated self mutableCopy_val_lower_message_repeated = 2719; repeated self mutableCopy_Val_upper_message_repeated = 2720; @@ -781,9 +784,9 @@ message ObjcWeirdDefaults { // Used to confirm negative enum values work as expected. message EnumTestMsg { enum MyEnum { - ZERO = 0; - ONE = 1; - TWO = 2; + ZERO = 0; + ONE = 1; + TWO = 2; NEG_ONE = -1; NEG_TWO = -2; } @@ -794,6 +797,20 @@ message EnumTestMsg { repeated MyEnum mumble = 4; } +message EnumTestMsgPrime { + enum MyEnumPrime { + ZERO = 0; + ONE = 1; + NEG_ONE = -1; + // Lacks 2, -2. + } + optional MyEnumPrime foo = 1; + optional MyEnumPrime bar = 2 [default = ONE]; + optional MyEnumPrime baz = 3 [default = NEG_ONE]; + + repeated MyEnumPrime mumble = 4; +} + // Test case for https://github.com/protocolbuffers/protobuf/issues/1453 // Message with no explicit defaults, but a non zero default for an enum. message MessageWithOneBasedEnum { diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9dcf0c0428..0c90a525d4 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -51,11 +51,11 @@ def protobuf_deps(): if not native.existing_rule("bazel_skylib"): http_archive( name = "bazel_skylib", + sha256 = "d00f1389ee20b60018e92644e0948e16e350a7707219e7a390fb0a99b6ec9262", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.7.0/bazel-skylib-1.7.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.7.0/bazel-skylib-1.7.0.tar.gz", ], - sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", ) if not native.existing_rule("com_google_absl"): @@ -84,8 +84,8 @@ def protobuf_deps(): _github_archive( name = "jsoncpp", repo = "https://github.com/open-source-parsers/jsoncpp", - commit = "9059f5cad030ba11d37818847443a53918c327b1", # 1.9.4 - sha256 = "c0c583c7b53a53bcd1f7385f15439dcdf0314d550362379e2db9919a918d1996", + commit = "5defb4ed1a4293b8e2bf641e16b156fb9de498cc", # 1.9.5 + sha256 = "a03d3136ff6dd092143bba8d3ded641e87b44e6c0b1f632b368f6cc8587524b5", build_file = Label("//:third_party/jsoncpp.BUILD"), ) @@ -120,17 +120,6 @@ def protobuf_deps(): sha256 = "160d1ebf33763124766fb35316329d907ca67f733238aa47624a8e3ff3cf2ef4", ) - # TODO: remove after toolchain types are moved to protobuf - if not native.existing_rule("rules_proto"): - http_archive( - name = "rules_proto", - sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", - strip_prefix = "rules_proto-5.3.0-21.7", - urls = [ - "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz", - ], - ) - if not native.existing_rule("proto_bazel_features"): proto_bazel_features(name = "proto_bazel_features") diff --git a/protos/protos.cc b/protos/protos.cc deleted file mode 100644 index fedbb6bc30..0000000000 --- a/protos/protos.cc +++ /dev/null @@ -1,185 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google LLC. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#include "protos/protos.h" - -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" -#include "protos/protos_extension_lock.h" -#include "upb/mem/arena.h" -#include "upb/message/accessors.h" -#include "upb/message/copy.h" -#include "upb/message/message.h" -#include "upb/message/promote.h" -#include "upb/message/value.h" -#include "upb/mini_table/extension.h" -#include "upb/mini_table/extension_registry.h" -#include "upb/mini_table/message.h" -#include "upb/wire/decode.h" -#include "upb/wire/encode.h" - -namespace protos { - -// begin:google_only -// absl::Status MessageAllocationError(SourceLocation loc) { -// return absl::Status(absl::StatusCode::kInternal, -// "Upb message allocation error", loc); -// } -// -// absl::Status ExtensionNotFoundError(int extension_number, SourceLocation loc) { -// return absl::Status( -// absl::StatusCode::kInternal, -// absl::StrFormat("Extension %d not found", extension_number), loc); -// } -// -// absl::Status MessageEncodeError(upb_EncodeStatus status, SourceLocation loc) { -// return absl::Status(absl::StatusCode::kInternal, -// absl::StrFormat("Upb message encoding error %d", status), -// loc -// -// ); -// } -// -// absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc -// -// ) { -// return absl::Status(absl::StatusCode::kInternal, -// absl::StrFormat("Upb message parse error %d", status), loc -// -// ); -// } -// end:google_only - -// begin:github_only -absl::Status MessageAllocationError(SourceLocation loc) { - return absl::Status(absl::StatusCode::kUnknown, - "Upb message allocation error"); -} - -absl::Status ExtensionNotFoundError(int ext_number, SourceLocation loc) { - return absl::Status(absl::StatusCode::kUnknown, - absl::StrFormat("Extension %d not found", ext_number)); -} - -absl::Status MessageEncodeError(upb_EncodeStatus s, SourceLocation loc) { - return absl::Status(absl::StatusCode::kUnknown, "Encoding error"); -} - -absl::Status MessageDecodeError(upb_DecodeStatus status, SourceLocation loc - -) { - return absl::Status(absl::StatusCode::kUnknown, "Upb message parse error"); -} -// end:github_only - -namespace internal { - -upb_ExtensionRegistry* GetUpbExtensions( - const ExtensionRegistry& extension_registry) { - return extension_registry.registry_; -} - -/** - * MessageLock(msg) acquires lock on msg when constructed and releases it when - * destroyed. - */ -class MessageLock { - public: - explicit MessageLock(const upb_Message* msg) : msg_(msg) { - UpbExtensionLocker locker = - upb_extension_locker_global.load(std::memory_order_acquire); - unlocker_ = (locker != nullptr) ? locker(msg) : nullptr; - } - MessageLock(const MessageLock&) = delete; - void operator=(const MessageLock&) = delete; - ~MessageLock() { - if (unlocker_ != nullptr) { - unlocker_(msg_); - } - } - - private: - const upb_Message* msg_; - UpbExtensionUnlocker unlocker_; -}; - -bool HasExtensionOrUnknown(const upb_Message* msg, - const upb_MiniTableExtension* eid) { - MessageLock msg_lock(msg); - if (upb_Message_HasExtension(msg, eid)) return true; - - const int number = upb_MiniTableExtension_Number(eid); - return upb_Message_FindUnknown(msg, number, 0).status == kUpb_FindUnknown_Ok; -} - -bool GetOrPromoteExtension(upb_Message* msg, const upb_MiniTableExtension* eid, - upb_Arena* arena, upb_MessageValue* value) { - MessageLock msg_lock(msg); - upb_GetExtension_Status ext_status = upb_Message_GetOrPromoteExtension( - (upb_Message*)msg, eid, 0, arena, value); - return ext_status == kUpb_GetExtension_Ok; -} - -absl::StatusOr Serialize(const upb_Message* message, - const upb_MiniTable* mini_table, - upb_Arena* arena, int options) { - MessageLock msg_lock(message); - size_t len; - char* ptr; - upb_EncodeStatus status = - upb_Encode(message, mini_table, options, arena, &ptr, &len); - if (status == kUpb_EncodeStatus_Ok) { - return absl::string_view(ptr, len); - } - return MessageEncodeError(status); -} - -void DeepCopy(upb_Message* target, const upb_Message* source, - const upb_MiniTable* mini_table, upb_Arena* arena) { - MessageLock msg_lock(source); - upb_Message_DeepCopy(target, source, mini_table, arena); -} - -upb_Message* DeepClone(const upb_Message* source, - const upb_MiniTable* mini_table, upb_Arena* arena) { - MessageLock msg_lock(source); - return upb_Message_DeepClone(source, mini_table, arena); -} - -absl::Status MoveExtension(upb_Message* message, upb_Arena* message_arena, - const upb_MiniTableExtension* ext, - upb_Message* extension, upb_Arena* extension_arena) { - if (message_arena != extension_arena && - // Try fuse, if fusing is not allowed or fails, create copy of extension. - !upb_Arena_Fuse(message_arena, extension_arena)) { - extension = DeepClone(extension, upb_MiniTableExtension_GetSubMessage(ext), - message_arena); - } - return upb_Message_SetExtension(message, ext, &extension, message_arena) - ? absl::OkStatus() - : MessageAllocationError(); -} - -absl::Status SetExtension(upb_Message* message, upb_Arena* message_arena, - const upb_MiniTableExtension* ext, - const upb_Message* extension) { - // Clone extension into target message arena. - extension = DeepClone(extension, upb_MiniTableExtension_GetSubMessage(ext), - message_arena); - return upb_Message_SetExtension(message, ext, &extension, message_arena) - ? absl::OkStatus() - : MessageAllocationError(); -} - -} // namespace internal - -} // namespace protos diff --git a/protos/protos.h b/protos/protos.h index a3e8380041..4d61918050 100644 --- a/protos/protos.h +++ b/protos/protos.h @@ -4,8 +4,35 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd - #ifndef UPB_PROTOS_PROTOS_H_ #define UPB_PROTOS_PROTOS_H_ #include "google/protobuf/hpb/hpb.h" +namespace protos { +namespace internal { +using hpb::internal::CreateMessage; +using hpb::internal::CreateMessageProxy; +using hpb::internal::DeepClone; +using hpb::internal::DeepCopy; +using hpb::internal::ExtensionIdentifier; +using hpb::internal::GetArena; +using hpb::internal::GetInternalMsg; +using hpb::internal::GetMiniTable; +using hpb::internal::GetOrPromoteExtension; +using hpb::internal::GetUpbExtensions; +using hpb::internal::HasExtensionOrUnknown; +using hpb::internal::MoveExtension; +using hpb::internal::PrivateAccess; +using hpb::internal::Serialize; +using hpb::internal::SetExtension; +} // namespace internal +using hpb::ClearMessage; +using hpb::CloneMessage; +using hpb::CreateMessage; +using hpb::DeepCopy; + +using hpb::ClearExtension; +using hpb::GetExtension; +using hpb::HasExtension; +using hpb::SetExtension; +} // namespace protos #endif diff --git a/python/google/protobuf/internal/any_test.proto b/python/google/protobuf/internal/any_test.proto deleted file mode 100644 index 6c21ef8721..0000000000 --- a/python/google/protobuf/internal/any_test.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -// Author: jieluo@google.com (Jie Luo) - -syntax = "proto2"; - -package google.protobuf.internal; - -import "google/protobuf/any.proto"; - -message TestAny { - optional google.protobuf.Any value = 1; - optional int32 int_value = 2; - map map_value = 3; - extensions 10 to max; -} - -message TestAnyExtension1 { - extend TestAny { - optional TestAnyExtension1 extension1 = 98418603; - } - optional int32 i = 15; -} diff --git a/python/google/protobuf/internal/api_implementation.cc b/python/google/protobuf/internal/api_implementation.cc index e32f4acb68..06eb05f706 100644 --- a/python/google/protobuf/internal/api_implementation.cc +++ b/python/google/protobuf/internal/api_implementation.cc @@ -51,7 +51,7 @@ static const char kModuleDocstring[] = "\n" "It complements api_implementation.py by setting defaults using compile-time\n" "constants defined in C, such that one can set defaults at compilation\n" -"(e.g. with blaze flag --copt=-DPYTHON_PROTO2_CPP_IMPL_V2)."; +"(e.g. with bazel flag --copt=-DPYTHON_PROTO2_CPP_IMPL_V2)."; #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef _module = { diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index e9ddae6d16..1eed7af25b 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -525,7 +525,7 @@ class DescriptorPoolTestBase(object): unittest_import_pb2.DESCRIPTOR.serialized_pb)) pool.Add(descriptor_pb2.FileDescriptorProto.FromString( unittest_pb2.DESCRIPTOR.serialized_pb)) - message_class = message_factory.MessageFactory(pool).GetPrototype( + message_class = message_factory.GetMessageClass( pool.FindMessageTypeByName( unittest_pb2.TestAllTypes.DESCRIPTOR.full_name)) _CheckDefaultValues(message_class()) diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index e72ecde094..09467cf72d 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -65,7 +65,7 @@ class MessageFactoryTest(unittest.TestCase): result = cls.FromString(reserialized) self.assertEqual(msg, result) - def testGetPrototype(self): + def testGetMessageClass(self): db = descriptor_database.DescriptorDatabase() pool = descriptor_pool.DescriptorPool(db) db.Add(self.factory_test1_fd) diff --git a/python/google/protobuf/internal/more_messages.proto b/python/google/protobuf/internal/more_messages.proto index fec0d72495..6ce788c927 100644 --- a/python/google/protobuf/internal/more_messages.proto +++ b/python/google/protobuf/internal/more_messages.proto @@ -352,10 +352,3 @@ message ConflictJsonName { optional int32 value = 1 [json_name = "old_value"]; optional int32 new_value = 2 [json_name = "value"]; } - -message WKTMessage { - optional Timestamp optional_timestamp = 1; - optional Duration optional_duration = 2; - optional Struct optional_struct = 3; - optional ListValue optional_list_value = 4; -} diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index fabc6aa078..28f599f759 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -72,7 +72,7 @@ class GeneratedProtocolMessageType(type): mydescriptor = Descriptor(.....) factory = symbol_database.Default() factory.pool.AddDescriptor(mydescriptor) - MyProtoClass = factory.GetPrototype(mydescriptor) + MyProtoClass = message_factory.GetMessageClass(mydescriptor) myproto_instance = MyProtoClass() myproto.foo_field = 23 ... @@ -120,7 +120,7 @@ class GeneratedProtocolMessageType(type): # to achieve similar results. # # This most commonly happens in `text_format.py` when using descriptors from - # a custom pool; it calls symbol_database.Global().getPrototype() on a + # a custom pool; it calls message_factory.GetMessageClass() on a # descriptor which already has an existing concrete class. new_class = getattr(descriptor, '_concrete_class', None) if new_class: @@ -988,7 +988,10 @@ def _InternalUnpackAny(msg): if descriptor is None: return None - message_class = factory.GetPrototype(descriptor) + # Unable to import message_factory at top becaue of circular import. + # pylint: disable=g-import-not-at-top + from google.protobuf import message_factory + message_class = message_factory.GetMessageClass(descriptor) message = message_class() message.ParseFromString(msg.value) diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index 47675fcf3d..45cc979667 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -35,11 +35,6 @@ class SymbolDatabaseTest(unittest.TestCase): db.RegisterServiceDescriptor(unittest_pb2._TESTSERVICE) return db - def testGetPrototype(self): - instance = self._Database().GetPrototype( - unittest_pb2.TestAllTypes.DESCRIPTOR) - self.assertTrue(instance is unittest_pb2.TestAllTypes) - def testGetMessages(self): messages = self._Database().GetMessages( ['google/protobuf/unittest.proto']) diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index a527e5bd67..cb2efb3a3b 100644 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -19,7 +19,7 @@ import unittest from google.protobuf import any_pb2 from google.protobuf import struct_pb2 from google.protobuf import descriptor_pb2 -from google.protobuf.internal import any_test_pb2 as test_extend_any +from google.protobuf.internal import well_known_types_test_pb2 as test_extend_any from google.protobuf.internal import api_implementation from google.protobuf.internal import message_set_extensions_pb2 from google.protobuf.internal import test_proto3_optional_pb2 diff --git a/python/google/protobuf/internal/well_known_types_test.proto b/python/google/protobuf/internal/well_known_types_test.proto new file mode 100644 index 0000000000..dad4f53c72 --- /dev/null +++ b/python/google/protobuf/internal/well_known_types_test.proto @@ -0,0 +1,29 @@ +edition = "2023"; + +package google.protobuf.internal; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +message TestAny { + google.protobuf.Any value = 1; + int32 int_value = 2; + map map_value = 3; + extensions 10 to max; +} + +message TestAnyExtension1 { + extend TestAny { + TestAnyExtension1 extension1 = 98418603; + } + int32 i = 15; +} + +message WKTMessage { + Timestamp optional_timestamp = 1; + Duration optional_duration = 2; + Struct optional_struct = 3; + ListValue optional_list_value = 4; +} diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 971ef8d788..1f05245f0d 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -14,9 +14,9 @@ import datetime import unittest from google.protobuf import text_format -from google.protobuf.internal import any_test_pb2 from google.protobuf.internal import more_messages_pb2 from google.protobuf.internal import well_known_types +from google.protobuf.internal import well_known_types_test_pb2 from google.protobuf import any_pb2 from google.protobuf import duration_pb2 @@ -361,7 +361,7 @@ class TimeUtilTest(TimeUtilTestBase): ) def testTimestampAssignment(self, date_parts, tzinfo): original_datetime = datetime.datetime(*date_parts, tzinfo=tzinfo) # pylint:disable=g-tzinfo-datetime - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_timestamp = original_datetime self.assertEqual(7200, msg.optional_timestamp.seconds) self.assertEqual(0, msg.optional_timestamp.nanos) @@ -374,11 +374,13 @@ class TimeUtilTest(TimeUtilTestBase): ) def testTimestampCreation(self, date_parts, tzinfo): original_datetime = datetime.datetime(*date_parts, tzinfo=tzinfo) # pylint:disable=g-tzinfo-datetime - msg = more_messages_pb2.WKTMessage(optional_timestamp=original_datetime) + msg = well_known_types_test_pb2.WKTMessage( + optional_timestamp=original_datetime + ) self.assertEqual(7200, msg.optional_timestamp.seconds) self.assertEqual(0, msg.optional_timestamp.nanos) - msg2 = more_messages_pb2.WKTMessage( + msg2 = well_known_types_test_pb2.WKTMessage( optional_timestamp=msg.optional_timestamp ) self.assertEqual(7200, msg2.optional_timestamp.seconds) @@ -401,24 +403,24 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testTimestampAdd(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_timestamp = old_time # Timestamp + timedelta - new_msg1 = more_messages_pb2.WKTMessage() + new_msg1 = well_known_types_test_pb2.WKTMessage() new_msg1.optional_timestamp = msg.optional_timestamp + time_delta self.assertEqual(expected_sec, new_msg1.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg1.optional_timestamp.nanos) # timedelta + Timestamp - new_msg2 = more_messages_pb2.WKTMessage() + new_msg2 = well_known_types_test_pb2.WKTMessage() new_msg2.optional_timestamp = time_delta + msg.optional_timestamp self.assertEqual(expected_sec, new_msg2.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg2.optional_timestamp.nanos) # Timestamp + Duration msg.optional_duration.FromTimedelta(time_delta) - new_msg3 = more_messages_pb2.WKTMessage() + new_msg3 = well_known_types_test_pb2.WKTMessage() new_msg3.optional_timestamp = msg.optional_timestamp + msg.optional_duration self.assertEqual(expected_sec, new_msg3.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg3.optional_timestamp.nanos) @@ -440,23 +442,23 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testTimestampSub(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_timestamp = old_time # Timestamp - timedelta - new_msg1 = more_messages_pb2.WKTMessage() + new_msg1 = well_known_types_test_pb2.WKTMessage() new_msg1.optional_timestamp = msg.optional_timestamp - time_delta self.assertEqual(expected_sec, new_msg1.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg1.optional_timestamp.nanos) # Timestamp - Duration msg.optional_duration = time_delta - new_msg2 = more_messages_pb2.WKTMessage() + new_msg2 = well_known_types_test_pb2.WKTMessage() new_msg2.optional_timestamp = msg.optional_timestamp - msg.optional_duration self.assertEqual(expected_sec, new_msg2.optional_timestamp.seconds) self.assertEqual(expected_nano, new_msg2.optional_timestamp.nanos) - result_msg = more_messages_pb2.WKTMessage() + result_msg = well_known_types_test_pb2.WKTMessage() result_msg.optional_timestamp = old_time - time_delta # Timestamp - Timestamp td = msg.optional_timestamp - result_msg.optional_timestamp @@ -532,12 +534,12 @@ class TimeUtilTest(TimeUtilTestBase): message.ToJsonString) self.assertRaisesRegex(ValueError, 'Timestamp is not valid', message.FromSeconds, -62135596801) - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() with self.assertRaises(AttributeError): msg.optional_timestamp = 1 with self.assertRaises(AttributeError): - msg2 = more_messages_pb2.WKTMessage(optional_timestamp=1) + msg2 = well_known_types_test_pb2.WKTMessage(optional_timestamp=1) with self.assertRaises(TypeError): msg.optional_timestamp + '' @@ -577,12 +579,12 @@ class TimeUtilTest(TimeUtilTestBase): self.assertRaisesRegex(ValueError, r'Duration is not valid\: Sign mismatch.', message.ToJsonString) - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() with self.assertRaises(AttributeError): msg.optional_duration = 1 with self.assertRaises(AttributeError): - msg2 = more_messages_pb2.WKTMessage(optional_duration=1) + msg2 = well_known_types_test_pb2.WKTMessage(optional_duration=1) with self.assertRaises(TypeError): msg.optional_duration + '' @@ -594,7 +596,7 @@ class TimeUtilTest(TimeUtilTestBase): ('test1', -1999999, -1, -999999000), ('test2', 1999999, 1, 999999000) ) def testDurationAssignment(self, microseconds, expected_sec, expected_nano): - message = more_messages_pb2.WKTMessage() + message = well_known_types_test_pb2.WKTMessage() expected_td = datetime.timedelta(microseconds=microseconds) message.optional_duration = expected_td self.assertEqual(expected_td, message.optional_duration.ToTimedelta()) @@ -605,7 +607,7 @@ class TimeUtilTest(TimeUtilTestBase): ('test1', -1999999, -1, -999999000), ('test2', 1999999, 1, 999999000) ) def testDurationCreation(self, microseconds, expected_sec, expected_nano): - message = more_messages_pb2.WKTMessage( + message = well_known_types_test_pb2.WKTMessage( optional_duration=datetime.timedelta(microseconds=microseconds) ) expected_td = datetime.timedelta(microseconds=microseconds) @@ -630,24 +632,24 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testDurationAdd(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_duration = time_delta msg.optional_timestamp = old_time # Duration + datetime - msg1 = more_messages_pb2.WKTMessage() + msg1 = well_known_types_test_pb2.WKTMessage() msg1.optional_timestamp = msg.optional_duration + old_time self.assertEqual(expected_sec, msg1.optional_timestamp.seconds) self.assertEqual(expected_nano, msg1.optional_timestamp.nanos) # datetime + Duration - msg2 = more_messages_pb2.WKTMessage() + msg2 = well_known_types_test_pb2.WKTMessage() msg2.optional_timestamp = old_time + msg.optional_duration self.assertEqual(expected_sec, msg2.optional_timestamp.seconds) self.assertEqual(expected_nano, msg2.optional_timestamp.nanos) # Duration + Timestamp - msg3 = more_messages_pb2.WKTMessage() + msg3 = well_known_types_test_pb2.WKTMessage() msg3.optional_timestamp = msg.optional_duration + msg.optional_timestamp self.assertEqual(expected_sec, msg3.optional_timestamp.seconds) self.assertEqual(expected_nano, msg3.optional_timestamp.nanos) @@ -669,7 +671,7 @@ class TimeUtilTest(TimeUtilTestBase): ), ) def testDurationSub(self, old_time, time_delta, expected_sec, expected_nano): - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_duration = time_delta # datetime - Duration @@ -843,7 +845,7 @@ class StructTest(unittest.TestCase): 'key2': 'abc', 'key3': {'subkey': 11.0, 'k': False}, } - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_struct = dictionary self.assertEqual(msg.optional_struct, dictionary) @@ -855,7 +857,7 @@ class StructTest(unittest.TestCase): self.assertEqual(msg.optional_struct, dictionary2) # Tests assign empty - msg2 = more_messages_pb2.WKTMessage() + msg2 = well_known_types_test_pb2.WKTMessage() self.assertNotIn('optional_struct', msg2) msg2.optional_struct = {} self.assertIn('optional_struct', msg2) @@ -863,7 +865,7 @@ class StructTest(unittest.TestCase): def testListValueAssignment(self): list_value = [6, 'seven', True, False, None, {}] - msg = more_messages_pb2.WKTMessage() + msg = well_known_types_test_pb2.WKTMessage() msg.optional_list_value = list_value self.assertEqual(msg.optional_list_value, list_value) @@ -874,7 +876,7 @@ class StructTest(unittest.TestCase): 'key3': {'subkey': 11.0, 'k': False}, } list_value = [6, 'seven', True, False, None, dictionary] - msg = more_messages_pb2.WKTMessage( + msg = well_known_types_test_pb2.WKTMessage( optional_struct=dictionary, optional_list_value=list_value ) self.assertEqual(len(msg.optional_struct), len(dictionary)) @@ -882,7 +884,7 @@ class StructTest(unittest.TestCase): self.assertEqual(len(msg.optional_list_value), len(list_value)) self.assertEqual(msg.optional_list_value, list_value) - msg2 = more_messages_pb2.WKTMessage( + msg2 = well_known_types_test_pb2.WKTMessage( optional_struct={}, optional_list_value=[] ) self.assertIn('optional_struct', msg2) @@ -892,17 +894,17 @@ class StructTest(unittest.TestCase): def testSpecialStructConstruct(self): dictionary = {'key1': 6.0} - msg = more_messages_pb2.WKTMessage(optional_struct=dictionary) + msg = well_known_types_test_pb2.WKTMessage(optional_struct=dictionary) self.assertEqual(msg.optional_struct, dictionary) dictionary2 = {'fields': 7.0} - msg2 = more_messages_pb2.WKTMessage(optional_struct=dictionary2) + msg2 = well_known_types_test_pb2.WKTMessage(optional_struct=dictionary2) self.assertEqual(msg2.optional_struct, dictionary2) # Construct Struct as normal message value_msg = struct_pb2.Value(number_value=5.0) dictionary3 = {'fields': {'key1': value_msg}} - msg3 = more_messages_pb2.WKTMessage(optional_struct=dictionary3) + msg3 = well_known_types_test_pb2.WKTMessage(optional_struct=dictionary3) self.assertEqual(msg3.optional_struct, {'key1': 5.0}) def testMergeFrom(self): @@ -955,7 +957,7 @@ class AnyTest(unittest.TestCase): def testAnyMessage(self): # Creates and sets message. - msg = any_test_pb2.TestAny() + msg = well_known_types_test_pb2.TestAny() msg_descriptor = msg.DESCRIPTOR all_types = unittest_pb2.TestAllTypes() all_descriptor = all_types.DESCRIPTOR @@ -985,7 +987,7 @@ class AnyTest(unittest.TestCase): msg_descriptor.full_name) def testUnpackWithNoSlashInTypeUrl(self): - msg = any_test_pb2.TestAny() + msg = well_known_types_test_pb2.TestAny() all_types = unittest_pb2.TestAllTypes() all_descriptor = all_types.DESCRIPTOR msg.value.Pack(all_types) @@ -997,14 +999,14 @@ class AnyTest(unittest.TestCase): def testMessageName(self): # Creates and sets message. - submessage = any_test_pb2.TestAny() + submessage = well_known_types_test_pb2.TestAny() submessage.int_value = 12345 msg = any_pb2.Any() msg.Pack(submessage) self.assertEqual(msg.TypeName(), 'google.protobuf.internal.TestAny') def testPackWithCustomTypeUrl(self): - submessage = any_test_pb2.TestAny() + submessage = well_known_types_test_pb2.TestAny() submessage.int_value = 12345 msg = any_pb2.Any() # Pack with a custom type URL prefix. @@ -1020,12 +1022,12 @@ class AnyTest(unittest.TestCase): self.assertEqual(msg.type_url, '/%s' % submessage.DESCRIPTOR.full_name) # Test unpacking the type. - unpacked_message = any_test_pb2.TestAny() + unpacked_message = well_known_types_test_pb2.TestAny() self.assertTrue(msg.Unpack(unpacked_message)) self.assertEqual(submessage, unpacked_message) def testPackDeterministic(self): - submessage = any_test_pb2.TestAny() + submessage = well_known_types_test_pb2.TestAny() for i in range(10): submessage.map_value[str(i)] = i * 2 msg = any_pb2.Any() diff --git a/python/google/protobuf/pyext/cpp_message.py b/python/google/protobuf/pyext/cpp_message.py index 623b52fbff..54cfe30646 100644 --- a/python/google/protobuf/pyext/cpp_message.py +++ b/python/google/protobuf/pyext/cpp_message.py @@ -34,7 +34,7 @@ class GeneratedProtocolMessageType(_message.MessageMeta): mydescriptor = Descriptor(.....) factory = symbol_database.Default() factory.pool.AddDescriptor(mydescriptor) - MyProtoClass = factory.GetPrototype(mydescriptor) + MyProtoClass = message_factory.GetMessageClass(mydescriptor) myproto_instance = MyProtoClass() myproto.foo_field = 23 ... diff --git a/rust/BUILD b/rust/BUILD index 2b273172d5..31c8c800d5 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -1,6 +1,12 @@ # Protobuf Rust runtime packages. load("@bazel_skylib//rules:common_settings.bzl", "string_flag") + +# begin:github_only +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") +load("@rules_pkg//pkg:zip.bzl", "pkg_zip") +# end:github_only + load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") @@ -33,6 +39,7 @@ rust_library( ":use_upb_kernel": ["--cfg=upb_kernel"], "//conditions:default": ["--cfg=cpp_kernel"], }), + visibility = ["//visibility:public"], deps = select({ ":use_upb_kernel": [":protobuf_upb"], "//conditions:default": [":protobuf_cpp"], @@ -47,17 +54,21 @@ rust_library( # # shared.rs is the root of the crate and has public items re-exported in protobuf.rs for user use. PROTOBUF_SHARED = [ + # go/keep-sorted start "codegen_traits.rs", + "cord.rs", "enum.rs", "internal.rs", - "primitive.rs", + "map.rs", "optional.rs", + "prelude.rs", + "primitive.rs", + "proto_macro.rs", "proxied.rs", "repeated.rs", "shared.rs", "string.rs", - "map.rs", - "proto_macro.rs", + # go/keep-sorted end ] # The Rust Protobuf runtime using the upb kernel. @@ -72,7 +83,10 @@ rust_library( proc_macro_deps = [ "@crate_index//:paste", ], - rustc_flags = ["--cfg=upb_kernel"], + rustc_flags = [ + "--cfg=upb_kernel", + "--cfg=bzl", + ], visibility = [ "//rust:__subpackages__", "//src/google/protobuf:__subpackages__", @@ -88,6 +102,7 @@ rust_test( crate = ":protobuf_upb", rustc_flags = [ "--cfg=upb_kernel", + "--cfg=bzl", ], deps = [ "@crate_index//:googletest", @@ -131,6 +146,7 @@ rust_test( crate = ":protobuf_cpp", rustc_flags = [ "--cfg=cpp_kernel", + "--cfg=bzl", ], deps = [ "@crate_index//:googletest", @@ -193,3 +209,57 @@ config_setting( ":rust_proto_library_kernel": "upb", }, ) + +# begin:github_only +pkg_files( + name = "rust_protobuf_src", + srcs = glob( + [ + "*", + ], + ), + strip_prefix = strip_prefix.from_root("rust"), + visibility = ["//:__pkg__"], +) + +pkg_files( + name = "crate_root_files", + srcs = glob(["cargo/**/*"]), + strip_prefix = strip_prefix.from_root("rust/cargo"), + visibility = ["//:__pkg__"], +) + +pkg_filegroup( + name = "rust_protobuf_src_dir", + srcs = [ + ":rust_protobuf_src", + "//rust/upb:rust_protobuf_upb_src", + ], + prefix = "src", +) + +pkg_files( + name = "amalgamated_upb", + srcs = ["//upb:gen_amalgamation"], + strip_prefix = strip_prefix.from_root(""), +) + +pkg_filegroup( + name = "rust_protobuf_libupb_src", + srcs = [ + ":amalgamated_upb", + "//upb/cmake:upb_cmake_dist", + ], + prefix = "libupb", +) + +pkg_zip( + name = "rust_crate", + srcs = [ + ":crate_root_files", + ":rust_protobuf_libupb_src", + ":rust_protobuf_src_dir", + "//:LICENSE", + ], +) +# end:github_only diff --git a/rust/cargo/Cargo.toml b/rust/cargo/Cargo.toml new file mode 100644 index 0000000000..7a389e764e --- /dev/null +++ b/rust/cargo/Cargo.toml @@ -0,0 +1,28 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2024 Google LLC. All rights reserved. + +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +[package] +name = "protobuf" +version = "4.27.3-beta.0" +edition = "2021" +links = "libupb" +license = "BSD-3-Clause" + +[lib] +path = "src/shared.rs" + +[dependencies] +paste = "1.0.15" + +[dev-dependencies] +googletest = {git = "https://github.com/google/googletest-rust.git" } + +[build-dependencies] +cc = "1.1.6" + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bzl)', 'cfg(cpp_kernel)', 'cfg(upb_kernel)'] } \ No newline at end of file diff --git a/rust/cargo/build.rs b/rust/cargo/build.rs new file mode 100644 index 0000000000..c69ef7c393 --- /dev/null +++ b/rust/cargo/build.rs @@ -0,0 +1,21 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. + +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +fn main() { + cc::Build::new() + .flag("-std=c99") + // TODO: Come up with a way to enable lto + // .flag("-flto=thin") + .warnings(false) + .include("libupb") + .file("libupb/upb/upb.c") + .file("libupb/third_party/utf8_range/utf8_range.c") + .define("UPB_BUILD_API", Some("1")) + .compile("libupb"); + let path = std::path::Path::new("libupb"); + println!("cargo:include={}", path.canonicalize().unwrap().display()) +} diff --git a/rust/codegen_traits.rs b/rust/codegen_traits.rs index 89c2db871d..d9a39411f8 100644 --- a/rust/codegen_traits.rs +++ b/rust/codegen_traits.rs @@ -7,11 +7,173 @@ //! Traits that are implemeted by codegen types. -use crate::Proxied; +use crate::__internal::SealedInternal; +use crate::__runtime::RawMessage; +use crate::{MutProxied, MutProxy, ViewProxy}; +use create::Parse; +use interop::{MessageMutInterop, MessageViewInterop, OwnedMessageInterop}; +use read::Serialize; use std::fmt::Debug; +use write::{Clear, ClearAndParse}; -pub trait Message: Default + Debug + Proxied + Send + Sync {} +/// A trait that all generated owned message types implement. +pub trait Message: SealedInternal + + MutProxied + // Create traits: + + Parse + Default + // Read traits: + + Debug + Serialize + // Write traits: + + Clear + ClearAndParse + // Thread safety: + + Send + Sync + // Copy/Clone: + + Clone + // C++ Interop: + + OwnedMessageInterop +{ +} -pub trait MessageView: Debug + Send + Sync {} +/// A trait that all generated message views implement. +pub trait MessageView<'msg>: SealedInternal + + ViewProxy<'msg, Proxied = Self::Message> + // Read traits: + + Debug + Serialize + // Thread safety: + + Send + Sync + // Copy/Clone: + + Copy + Clone + // C++ Interop: + + MessageViewInterop<'msg> +{ + #[doc(hidden)] + type Message: Message; +} -pub trait MessageMut: Debug + Sync {} +/// A trait that all generated message muts implement. +pub trait MessageMut<'msg>: SealedInternal + + MutProxy<'msg, MutProxied = Self::Message> + // Read traits: + + Debug + Serialize + // Write traits: + // TODO: MsgMut should impl ClearAndParse. + + Clear + // Thread safety: + + Sync + // Copy/Clone: + // (Neither) + // C++ Interop: + + MessageMutInterop<'msg> +{ + #[doc(hidden)] + type Message: Message; +} + +/// Operations related to constructing a message. Only owned messages implement +/// these traits. +pub(crate) mod create { + use super::SealedInternal; + pub trait Parse: SealedInternal + Sized { + fn parse(serialized: &[u8]) -> Result; + } +} + +/// Operations related to reading some aspect of a message (methods that would +/// have a `&self` receiver on an owned message). Owned messages, views, and +/// muts all implement these traits. +pub(crate) mod read { + use super::SealedInternal; + + pub trait Serialize: SealedInternal { + fn serialize(&self) -> Result, crate::SerializeError>; + } +} + +/// Operations related to mutating a message (methods that would have a `&mut +/// self` receiver on an owned message). Owned messages and muts implement these +/// traits. +pub(crate) mod write { + use super::SealedInternal; + + pub trait Clear: SealedInternal { + fn clear(&mut self); + } + + pub trait ClearAndParse: SealedInternal { + fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), crate::ParseError>; + } +} + +/// Traits related to interop with C or C++. +/// +/// These traits are deliberately not available on the prelude, as they should +/// be used rarely and with great care. +pub(crate) mod interop { + use super::{RawMessage, SealedInternal}; + + /// Traits related to owned message interop. Note that these trait fns + /// are only available on C++ kernel as upb interop of owned messages + /// requires more work to handle the Arena behavior. + pub trait OwnedMessageInterop: SealedInternal { + /// Drops `self` and returns the `RawMessage` that it was wrapping + /// without deleting it. + /// + /// The caller is responsible for ensuring the returned RawMessage is + /// subsequently deleted (eg by moving it into a std::unique_ptr in + /// C++), or else it will leak. + #[cfg(cpp_kernel)] + fn __unstable_leak_raw_message(self) -> RawMessage; + + /// Takes exclusive ownership of the `raw_message`. + /// + /// # Safety + /// - The underlying message must be for the same type as `Self` + /// - The pointer passed in must not be used by the caller after being + /// passed here (must not be read, written, or deleted) + #[cfg(cpp_kernel)] + unsafe fn __unstable_take_ownership_of_raw_message(raw_message: RawMessage) -> Self; + } + + /// Traits related to message view interop. + pub trait MessageViewInterop<'msg>: SealedInternal { + /// Borrows `self` as an underlying `RawMessage`. + /// + /// Note that the returned Value must be used under the same constraints + /// as though it were a borrow of `self`: it should be treated as a + /// `const Message*` in C++, and not be mutated in any way, and any + /// mutation to the parent message may invalidate it, and it + /// must not be deleted. + fn __unstable_as_raw_message(&self) -> RawMessage; + + /// Wraps the provided `RawMessage` as a MessageView. + /// + /// # Safety + /// - The underlying message must be for the same type as `Self` + /// - The underlying message must be alive for 'msg and not mutated + /// while the wrapper is live. + unsafe fn __unstable_wrap_raw_message(raw: &'msg RawMessage) -> Self; + } + + /// Traits related to message mut interop. Note that these trait fns + /// are only available on C++ kernel as upb interop of owned messages + /// requires more work to handle the Arena behavior. + pub trait MessageMutInterop<'msg>: SealedInternal { + /// Exclusive borrows `self` as a `RawMessage`. + /// + /// Note that the returned Value must be used under the same constraints + /// as though it were a mut borrow of `self`: it should be treated as a + /// non-owned `Message*` in C++. And any mutation to the parent message + /// may invalidate it, and it must not be deleted. + #[cfg(cpp_kernel)] + fn __unstable_as_raw_message_mut(&mut self) -> RawMessage; + + /// Wraps the provided `RawMessage` as a MessageMut. + /// + /// # Safety + /// - The underlying message must be for the same type as `Self` + /// - The underlying message must be alive for 'msg and not read or + /// mutated while the wrapper is live. + #[cfg(cpp_kernel)] + unsafe fn __unstable_wrap_raw_message_mut(raw: &'msg mut RawMessage) -> Self; + } +} diff --git a/rust/cord.rs b/rust/cord.rs new file mode 100644 index 0000000000..19fdb0ba5a --- /dev/null +++ b/rust/cord.rs @@ -0,0 +1,128 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +use crate::__internal::{Private, SealedInternal}; +use crate::{ + AsView, IntoProxied, IntoView, ProtoBytes, ProtoStr, ProtoString, Proxied, Proxy, View, + ViewProxy, +}; +use paste::paste; +use std::cmp::PartialEq; +use std::ops::Deref; + +macro_rules! impl_cord_types { + ($($t:ty, $vt:ty);*) => { + paste! { $( + #[derive(Debug)] + pub struct [< $t Cord>](Private); + + #[derive(Debug)] + pub enum [< $t Cow>]<'a> { + Borrowed(View<'a, $t>), + Owned($t), + } + + impl SealedInternal for [< $t Cord>] {} + + impl<'msg> SealedInternal for [< $t Cow>]<'msg> {} + + impl Proxied for [< $t Cord>] { + type View<'msg> = [< $t Cow>]<'msg>; + } + + impl AsView for [< $t Cord>] { + type Proxied = Self; + fn as_view(&self) -> [< $t Cow>]<'_> { + unimplemented!("Proto Cord should never be constructed"); + } + } + + impl<'msg> Proxy<'msg> for [< $t Cow>]<'msg> {} + + impl<'msg> ViewProxy<'msg> for [< $t Cow>]<'msg> {} + + impl<'msg> AsView for [< $t Cow>]<'msg> { + type Proxied = [< $t Cord>]; + + fn as_view(&self) -> [< $t Cow>]<'_> { + match self { + [< $t Cow>]::Owned(owned) => [< $t Cow>]::Borrowed((*owned).as_view()), + [< $t Cow>]::Borrowed(borrowed) => [< $t Cow>]::Borrowed(borrowed), + } + } + } + + impl<'msg> IntoView<'msg> for [< $t Cow>]<'msg> { + fn into_view<'shorter>(self) -> [< $t Cow>]<'shorter> + where + 'msg: 'shorter, { + match self { + [< $t Cow>]::Owned(owned) => [< $t Cow>]::Owned(owned), + [< $t Cow>]::Borrowed(borrow) => [< $t Cow>]::Borrowed(borrow.into_view()), + } + } + } + + impl IntoProxied<$t> for [< $t Cow>]<'_> { + fn into_proxied(self, _private: Private) -> $t { + match self { + [< $t Cow>]::Owned(owned) => owned, + [< $t Cow>]::Borrowed(borrowed) => borrowed.into_proxied(Private), + } + } + } + + impl<'a> Deref for [< $t Cow>]<'a> { + type Target = $vt; + + fn deref(&self) -> View<'_, $t> { + match self { + [< $t Cow>]::Borrowed(borrow) => borrow, + [< $t Cow>]::Owned(owned) => (*owned).as_view(), + } + } + } + + impl AsRef<[u8]> for [< $t Cow>]<'_> { + fn as_ref(&self) -> &[u8] { + match self { + [< $t Cow>]::Borrowed(borrow) => borrow.as_ref(), + [< $t Cow>]::Owned(owned) => owned.as_ref(), + } + } + } + )* + } + } +} + +impl_cord_types!( + ProtoString, ProtoStr; + ProtoBytes, [u8] +); + +macro_rules! impl_eq { + ($($t1:ty, $t2:ty);*) => { + paste! { $( + impl PartialEq<$t1> for $t2 { + fn eq(&self, rhs: &$t1) -> bool { + AsRef::<[u8]>::as_ref(self) == AsRef::<[u8]>::as_ref(rhs) + } + } + )* + } + } +} + +impl_eq!( + ProtoStringCow<'_>, ProtoStringCow<'_>; + str, ProtoStringCow<'_>; + ProtoStringCow<'_>, str; + ProtoBytesCow<'_>, ProtoBytesCow<'_>; + [u8], ProtoBytesCow<'_>; + ProtoBytesCow<'_>, [u8] +); diff --git a/rust/cpp.rs b/rust/cpp.rs index d8e4df041a..4d3358a8e8 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -95,6 +95,11 @@ pub struct InnerProtoString { owned_ptr: CppStdString, } +/// An opaque type matching MapNodeSizeInfoT from C++. +#[doc(hidden)] +#[repr(transparent)] +pub struct MapNodeSizeInfo(pub i32); + impl Drop for InnerProtoString { fn drop(&mut self) { // SAFETY: `self.owned_ptr` points to a valid std::string object. @@ -114,6 +119,12 @@ impl InnerProtoString { let s = ManuallyDrop::new(self); s.owned_ptr } + + /// # Safety + /// - `src` points to a valid CppStdString. + pub unsafe fn from_raw(_private: Private, src: CppStdString) -> InnerProtoString { + InnerProtoString { owned_ptr: src } + } } impl From<&[u8]> for InnerProtoString { @@ -188,6 +199,7 @@ impl From<&ProtoStr> for PtrAndLen { /// This struct is ABI-compatible with the equivalent struct on the C++ side. It /// owns (and drops) its data. #[repr(C)] +#[doc(hidden)] pub struct SerializedData { /// Owns the memory. data: NonNull, @@ -195,7 +207,7 @@ pub struct SerializedData { } impl SerializedData { - pub fn new() -> Self { + pub fn new(_private: Private) -> Self { Self { data: NonNull::dangling(), len: 0 } } @@ -683,15 +695,17 @@ impl UntypedMapIterator { _private: Private, iter_get_thunk: unsafe extern "C" fn( iter: &mut UntypedMapIterator, + size_info: MapNodeSizeInfo, key: *mut FfiKey, value: *mut FfiValue, ), + size_info: MapNodeSizeInfo, from_ffi_key: impl FnOnce(FfiKey) -> View<'a, K>, from_ffi_value: impl FnOnce(FfiValue) -> View<'a, V>, ) -> Option<(View<'a, K>, View<'a, V>)> where - K: Proxied + ?Sized + 'a, - V: ProxiedInMapValue + ?Sized + 'a, + K: Proxied + 'a, + V: ProxiedInMapValue + 'a, { if self.at_end() { return None; @@ -703,7 +717,7 @@ impl UntypedMapIterator { // - The iterator is not at the end (node is non-null). // - `ffi_key` and `ffi_value` are not read (as uninit) as promised by the // caller. - unsafe { (iter_get_thunk)(self, ffi_key.as_mut_ptr(), ffi_value.as_mut_ptr()) } + unsafe { (iter_get_thunk)(self, size_info, ffi_key.as_mut_ptr(), ffi_value.as_mut_ptr()) } // SAFETY: // - The backing map is alive as promised by the caller. @@ -726,8 +740,100 @@ impl UntypedMapIterator { } } +#[doc(hidden)] +#[repr(transparent)] +pub struct MapNodeSizeInfoIndex(i32); + +#[doc(hidden)] +pub trait MapNodeSizeInfoIndexForType { + const SIZE_INFO_INDEX: MapNodeSizeInfoIndex; +} + +macro_rules! generate_map_node_size_info_mapping { + ( $($key:ty, $index:expr;)* ) => { + $( + impl MapNodeSizeInfoIndexForType for $key { + const SIZE_INFO_INDEX: MapNodeSizeInfoIndex = MapNodeSizeInfoIndex($index); + } + )* + } +} + +// LINT.IfChange(size_info_mapping) +generate_map_node_size_info_mapping!( + i32, 0; + u32, 0; + i64, 1; + u64, 1; + bool, 2; + ProtoString, 3; +); +// LINT.ThenChange(//depot/google3/third_party/protobuf/compiler/rust/message. +// cc:size_info_mapping) + +macro_rules! impl_map_primitives { + (@impl $(($rust_type:ty, $cpp_type:ty) => [ + $insert_thunk:ident, + $get_thunk:ident, + $iter_get_thunk:ident, + $remove_thunk:ident, + ]),* $(,)?) => { + $( + extern "C" { + pub fn $insert_thunk( + m: RawMap, + size_info: MapNodeSizeInfo, + key: $cpp_type, + value: RawMessage, + placement_new: unsafe extern "C" fn(*mut c_void, m: RawMessage), + ) -> bool; + pub fn $get_thunk( + m: RawMap, + size_info: MapNodeSizeInfo, + key: $cpp_type, + value: *mut RawMessage, + ) -> bool; + pub fn $iter_get_thunk( + iter: &mut UntypedMapIterator, + size_info: MapNodeSizeInfo, + key: *mut $cpp_type, + value: *mut RawMessage, + ); + pub fn $remove_thunk(m: RawMap, size_info: MapNodeSizeInfo, key: $cpp_type) -> bool; + } + )* + }; + ($($rust_type:ty, $cpp_type:ty;)* $(,)?) => { + paste!{ + impl_map_primitives!(@impl $( + ($rust_type, $cpp_type) => [ + [< proto2_rust_map_insert_ $rust_type >], + [< proto2_rust_map_get_ $rust_type >], + [< proto2_rust_map_iter_get_ $rust_type >], + [< proto2_rust_map_remove_ $rust_type >], + ], + )*); + } + }; +} + +impl_map_primitives!( + i32, i32; + u32, u32; + i64, i64; + u64, u64; + bool, bool; + ProtoString, PtrAndLen; +); + extern "C" { fn proto2_rust_thunk_UntypedMapIterator_increment(iter: &mut UntypedMapIterator); + + pub fn proto2_rust_map_new() -> RawMap; + pub fn proto2_rust_map_free(m: RawMap, key_is_string: bool, size_info: MapNodeSizeInfo); + pub fn proto2_rust_map_clear(m: RawMap, key_is_string: bool, size_info: MapNodeSizeInfo); + pub fn proto2_rust_map_size(m: RawMap) -> usize; + pub fn proto2_rust_map_iter(m: RawMap) -> UntypedMapIterator; } macro_rules! impl_ProxiedInMapValue_for_non_generated_value_types { @@ -741,7 +847,7 @@ macro_rules! impl_ProxiedInMapValue_for_non_generated_value_types { fn [< proto2_rust_thunk_Map_ $key_t _ $t _insert >](m: RawMap, key: $ffi_key_t, value: $ffi_value_t) -> bool; fn [< proto2_rust_thunk_Map_ $key_t _ $t _get >](m: RawMap, key: $ffi_key_t, value: *mut $ffi_view_t) -> bool; fn [< proto2_rust_thunk_Map_ $key_t _ $t _iter >](m: RawMap) -> UntypedMapIterator; - fn [< proto2_rust_thunk_Map_ $key_t _ $t _iter_get >](iter: &mut UntypedMapIterator, key: *mut $ffi_key_t, value: *mut $ffi_view_t); + fn [< proto2_rust_thunk_Map_ $key_t _ $t _iter_get >](iter: &mut UntypedMapIterator, size_info: MapNodeSizeInfo, key: *mut $ffi_key_t, value: *mut $ffi_view_t); fn [< proto2_rust_thunk_Map_ $key_t _ $t _remove >](m: RawMap, key: $ffi_key_t, value: *mut $ffi_view_t) -> bool; } @@ -822,6 +928,7 @@ macro_rules! impl_ProxiedInMapValue_for_non_generated_value_types { iter.as_raw_mut(Private).next_unchecked::<$key_t, Self, _, _>( Private, [< proto2_rust_thunk_Map_ $key_t _ $t _iter_get >], + MapNodeSizeInfo(0), $from_ffi_key, $from_ffi_value, ) @@ -899,14 +1006,14 @@ mod tests { (content.as_mut_ptr(), content.len()) } - #[test] + #[googletest::test] fn test_serialized_data_roundtrip() { let (ptr, len) = allocate_byte_array(b"Hello world"); let serialized_data = SerializedData { data: NonNull::new(ptr).unwrap(), len }; assert_that!(&*serialized_data, eq(b"Hello world")); } - #[test] + #[googletest::test] fn test_empty_string() { let empty_str: String = RustStringRawParts { data: std::ptr::null(), len: 0 }.into(); assert_that!(empty_str, eq("")); diff --git a/rust/cpp_kernel/BUILD b/rust/cpp_kernel/BUILD index 7d0a256f1c..c1c1cdb7e1 100644 --- a/rust/cpp_kernel/BUILD +++ b/rust/cpp_kernel/BUILD @@ -27,6 +27,7 @@ cc_library( "//src/google/protobuf:protobuf_lite", "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/rust/cpp_kernel/map.cc b/rust/cpp_kernel/map.cc index 2418c7274c..60f53c7df0 100644 --- a/rust/cpp_kernel/map.cc +++ b/rust/cpp_kernel/map.cc @@ -1,12 +1,137 @@ #include "rust/cpp_kernel/map.h" +#include #include #include +#include #include #include "google/protobuf/map.h" +#include "google/protobuf/message_lite.h" #include "rust/cpp_kernel/strings.h" +namespace google { +namespace protobuf { +namespace rust { +namespace { + +template +struct FromViewType { + using type = T; +}; + +template <> +struct FromViewType { + using type = std::string; +}; + +template +using KeyMap = internal::KeyMapBase< + internal::KeyForBase::type>>; + +template +void DestroyMapNode(internal::UntypedMapBase* m, internal::NodeBase* node, + internal::MapNodeSizeInfoT size_info) { + if constexpr (std::is_same::value) { + static_cast(node->GetVoidKey())->~basic_string(); + } + internal::RustMapHelper::DestroyMessage( + static_cast(node->GetVoidValue(size_info))); + internal::RustMapHelper::DeallocNode(m, node, size_info); +} + +template +bool Insert(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + Key key, MessageLite* value, + void (*placement_new)(void*, MessageLite*)) { + internal::NodeBase* node = internal::RustMapHelper::AllocNode(m, size_info); + if constexpr (std::is_same::value) { + new (node->GetVoidKey()) std::string(key.ptr, key.len); + } else { + *static_cast(node->GetVoidKey()) = key; + } + void* value_ptr = node->GetVoidValue(size_info); + placement_new(value_ptr, value); + node = internal::RustMapHelper::InsertOrReplaceNode( + static_cast*>(m), node); + if (node == nullptr) { + return true; + } + DestroyMapNode(m, node, size_info); + return false; +} + +template ::value>::type> +internal::RustMapHelper::NodeAndBucket FindHelper(Map* m, Key key) { + return internal::RustMapHelper::FindHelper( + m, static_cast>(key)); +} + +template +internal::RustMapHelper::NodeAndBucket FindHelper(Map* m, + google::protobuf::rust::PtrAndLen key) { + return internal::RustMapHelper::FindHelper( + m, absl::string_view(key.ptr, key.len)); +} + +template +bool Get(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + Key key, MessageLite** value) { + auto* map_base = static_cast*>(m); + internal::RustMapHelper::NodeAndBucket result = FindHelper(map_base, key); + if (result.node == nullptr) { + return false; + } + *value = static_cast(result.node->GetVoidValue(size_info)); + return true; +} + +template +bool Remove(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + Key key) { + auto* map_base = static_cast*>(m); + internal::RustMapHelper::NodeAndBucket result = FindHelper(map_base, key); + if (result.node == nullptr) { + return false; + } + internal::RustMapHelper::EraseNoDestroy(map_base, result.bucket, result.node); + DestroyMapNode(m, result.node, size_info); + return true; +} + +template +void IterGet(const internal::UntypedMapIterator* iter, + internal::MapNodeSizeInfoT size_info, Key* key, + MessageLite** value) { + internal::NodeBase* node = iter->node_; + if constexpr (std::is_same::value) { + const std::string* s = static_cast(node->GetVoidKey()); + *key = PtrAndLen(s->data(), s->size()); + } else { + *key = *static_cast(node->GetVoidKey()); + } + *value = static_cast(node->GetVoidValue(size_info)); +} + +void ClearMap(internal::UntypedMapBase* m, internal::MapNodeSizeInfoT size_info, + bool key_is_string, bool reset_table) { + if (internal::RustMapHelper::IsGlobalEmptyTable(m)) return; + uint8_t bits = internal::RustMapHelper::kValueIsProto; + if (key_is_string) { + bits |= internal::RustMapHelper::kKeyIsString; + } + internal::RustMapHelper::ClearTable( + m, internal::RustMapHelper::ClearInput{size_info, bits, reset_table, + /* destroy_node = */ nullptr}); +} + +} // namespace +} // namespace rust +} // namespace protobuf +} // namespace google + extern "C" { void proto2_rust_thunk_UntypedMapIterator_increment( @@ -14,6 +139,69 @@ void proto2_rust_thunk_UntypedMapIterator_increment( iter->PlusPlus(); } +google::protobuf::internal::UntypedMapBase* proto2_rust_map_new() { + return new google::protobuf::internal::UntypedMapBase(/* arena = */ nullptr); +} + +void proto2_rust_map_free(google::protobuf::internal::UntypedMapBase* m, + bool key_is_string, + google::protobuf::internal::MapNodeSizeInfoT size_info) { + google::protobuf::rust::ClearMap(m, size_info, key_is_string, + /* reset_table = */ false); + delete m; +} + +void proto2_rust_map_clear(google::protobuf::internal::UntypedMapBase* m, + bool key_is_string, + google::protobuf::internal::MapNodeSizeInfoT size_info) { + google::protobuf::rust::ClearMap(m, size_info, key_is_string, /* reset_table = */ true); +} + +size_t proto2_rust_map_size(google::protobuf::internal::UntypedMapBase* m) { + return m->size(); +} + +google::protobuf::internal::UntypedMapIterator proto2_rust_map_iter( + google::protobuf::internal::UntypedMapBase* m) { + return m->begin(); +} + +#define DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(cpp_type, suffix) \ + bool proto2_rust_map_insert_##suffix( \ + google::protobuf::internal::UntypedMapBase* m, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type key, \ + google::protobuf::MessageLite* value, \ + void (*placement_new)(void*, google::protobuf::MessageLite*)) { \ + return google::protobuf::rust::Insert(m, size_info, key, value, placement_new); \ + } \ + \ + bool proto2_rust_map_get_##suffix( \ + google::protobuf::internal::UntypedMapBase* m, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type key, \ + google::protobuf::MessageLite** value) { \ + return google::protobuf::rust::Get(m, size_info, key, value); \ + } \ + \ + bool proto2_rust_map_remove_##suffix( \ + google::protobuf::internal::UntypedMapBase* m, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type key) { \ + return google::protobuf::rust::Remove(m, size_info, key); \ + } \ + \ + void proto2_rust_map_iter_get_##suffix( \ + const google::protobuf::internal::UntypedMapIterator* iter, \ + google::protobuf::internal::MapNodeSizeInfoT size_info, cpp_type* key, \ + google::protobuf::MessageLite** value) { \ + return google::protobuf::rust::IterGet(iter, size_info, key, value); \ + } + +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(int32_t, i32) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(uint32_t, u32) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(int64_t, i64) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(uint64_t, u64) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(bool, bool) +DEFINE_KEY_SPECIFIC_MAP_OPERATIONS(google::protobuf::rust::PtrAndLen, ProtoString) + __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE(int32_t, i32, int32_t, int32_t, value, cpp_value); __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE(uint32_t, u32, uint32_t, diff --git a/rust/cpp_kernel/map.h b/rust/cpp_kernel/map.h index c16f14b1f3..e568222edb 100644 --- a/rust/cpp_kernel/map.h +++ b/rust/cpp_kernel/map.h @@ -4,6 +4,10 @@ #include #include +#include "google/protobuf/map.h" +#include "google/protobuf/message_lite.h" +#include "rust/cpp_kernel/strings.h" + namespace google { namespace protobuf { namespace rust { @@ -74,8 +78,8 @@ auto MakeCleanup(T value) { return google::protobuf::internal::UntypedMapIterator::FromTyped(m->cbegin()); \ } \ void proto2_rust_thunk_Map_##rust_key_ty##_##rust_value_ty##_iter_get( \ - const google::protobuf::internal::UntypedMapIterator* iter, ffi_key_ty* key, \ - ffi_view_ty* value) { \ + const google::protobuf::internal::UntypedMapIterator* iter, int32_t, \ + ffi_key_ty* key, ffi_view_ty* value) { \ auto typed_iter = \ iter->ToTyped::const_iterator>(); \ const auto& cpp_key = typed_iter->first; \ diff --git a/rust/defs.bzl b/rust/defs.bzl index 23d841a187..15fa0be6d5 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -15,12 +15,6 @@ load( "rust_upb_proto_library_aspect", ) -visibility([ - "//experimental/...", - "//src/google/protobuf/...", - "//rust/...", -]) - def rust_proto_library(name, deps, **args): """Declares all the boilerplate needed to use Rust protobufs conveniently. diff --git a/rust/internal.rs b/rust/internal.rs index f8118c88d6..a2397bdbe7 100644 --- a/rust/internal.rs +++ b/rust/internal.rs @@ -21,4 +21,13 @@ pub use crate::ProtoStr; pub use crate::__runtime::{PtrAndLen, RawMap, RawMessage, RawRepeatedField}; /// Used to protect internal-only items from being used accidentally. +#[derive(Debug)] pub struct Private; + +/// A trait that is used as a subtrait of traits that we intend to be used but +/// not be implemented by users. +/// +/// This is slightly less 'sealed' than the typical sealed trait pattern would +/// permit in other crates; this trait is intended to be available to crates +/// which were generated by protoc, but not to application code. +pub trait SealedInternal {} diff --git a/rust/map.rs b/rust/map.rs index bbb5bfc677..ec72d9a49e 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -6,8 +6,9 @@ // https://developers.google.com/open-source/licenses/bsd use crate::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, - __internal::Private, + AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, View, + ViewProxy, + __internal::{Private, SealedInternal}, __runtime::{InnerMap, InnerMapMut, RawMap, RawMapIter}, }; use std::marker::PhantomData; @@ -59,19 +60,21 @@ impl<'msg, K: ?Sized, V: ?Sized> std::fmt::Debug for MapMut<'msg, K, V> { } } -pub struct Map> { +pub struct Map> { inner: InnerMap, _phantom: PhantomData<(PhantomData, PhantomData)>, } // SAFETY: `Map` is Sync because it does not implement interior mutability. -unsafe impl> Sync for Map {} +unsafe impl> Sync for Map {} // SAFETY: `Map` is Send because it's not bound to a specific thread e.g. // it does not use thread-local data or similar. -unsafe impl> Send for Map {} +unsafe impl> Send for Map {} -impl> Drop for Map { +impl> SealedInternal for Map {} + +impl> Drop for Map { fn drop(&mut self) { // SAFETY: // - `drop` is only called once. @@ -82,7 +85,7 @@ impl> Drop for Map { pub trait ProxiedInMapValue: Proxied where - K: Proxied + ?Sized, + K: Proxied, { fn map_new(_private: Private) -> Map; @@ -104,24 +107,44 @@ where fn map_iter_next<'a>(iter: &mut MapIter<'a, K, Self>) -> Option<(View<'a, K>, View<'a, Self>)>; } -impl + ?Sized> Proxied for Map { +impl> Proxied for Map { type View<'msg> = MapView<'msg, K, V> where K: 'msg, V: 'msg; } -impl + ?Sized> MutProxied for Map { +impl> AsView for Map { + type Proxied = Self; + + fn as_view(&self) -> MapView<'_, K, V> { + self.as_view() + } +} + +impl> MutProxied for Map { type Mut<'msg> = MapMut<'msg, K, V> where K: 'msg, V: 'msg; } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg> - for MapView<'msg, K, V> -{ +impl> AsMut for Map { + type MutProxied = Self; + + fn as_mut(&mut self) -> MapMut<'_, K, V> { + self.as_mut() + } +} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> SealedInternal for MapView<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapView<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapView<'msg, K, V> { type Proxied = Map; - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> MapView<'_, K, V> { *self } +} - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> +impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoView<'msg> for MapView<'msg, K, V> { + fn into_view<'shorter>(self) -> MapView<'shorter, K, V> where 'msg: 'shorter, { @@ -129,16 +152,22 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg } } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg> - for MapMut<'msg, K, V> -{ +impl<'msg, K: Proxied, V: ProxiedInMapValue> ViewProxy<'msg> for MapView<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> SealedInternal for MapMut<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> Proxy<'msg> for MapMut<'msg, K, V> {} + +impl<'msg, K: Proxied, V: ProxiedInMapValue> AsView for MapMut<'msg, K, V> { type Proxied = Map; - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> MapView<'_, K, V> { MapView { raw: self.inner.raw, _phantom: PhantomData } } +} - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> +impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoView<'msg> for MapMut<'msg, K, V> { + fn into_view<'shorter>(self) -> MapView<'shorter, K, V> where 'msg: 'shorter, { @@ -146,14 +175,16 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> ViewProxy<'msg } } -impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> MutProxy<'msg> - for MapMut<'msg, K, V> -{ - fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { +impl<'msg, K: Proxied, V: ProxiedInMapValue> AsMut for MapMut<'msg, K, V> { + type MutProxied = Map; + + fn as_mut(&mut self) -> MapMut<'_, K, V> { MapMut { inner: self.inner, _phantom: PhantomData } } +} - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> +impl<'msg, K: Proxied, V: ProxiedInMapValue> IntoMut<'msg> for MapMut<'msg, K, V> { + fn into_mut<'shorter>(self) -> MapMut<'shorter, K, V> where 'msg: 'shorter, { @@ -161,10 +192,12 @@ impl<'msg, K: Proxied + ?Sized, V: ProxiedInMapValue + ?Sized> MutProxy<'msg> } } +impl<'msg, K: Proxied, V: ProxiedInMapValue> MutProxy<'msg> for MapMut<'msg, K, V> {} + impl Map where - K: Proxied + ?Sized, - V: ProxiedInMapValue + ?Sized, + K: Proxied, + V: ProxiedInMapValue, { pub fn new() -> Self { V::map_new(Private) @@ -190,8 +223,8 @@ where impl Default for Map where - K: Proxied + ?Sized, - V: ProxiedInMapValue + ?Sized, + K: Proxied, + V: ProxiedInMapValue, { fn default() -> Self { Map::new() @@ -215,8 +248,8 @@ impl<'msg, K: ?Sized, V: ?Sized> MapView<'msg, K, V> { impl<'msg, K, V> MapView<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { pub fn get<'a>(self, key: impl Into>) -> Option> where @@ -273,8 +306,8 @@ impl<'msg, K: ?Sized, V: ?Sized> MapMut<'msg, K, V> { impl<'msg, K, V> MapMut<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { pub fn len(&self) -> usize { self.as_view().len() @@ -347,6 +380,30 @@ where } } +impl<'msg, K, V> IntoProxied> for MapView<'msg, K, V> +where + K: Proxied, + V: ProxiedInMapValue, + View<'msg, V>: IntoProxied, +{ + fn into_proxied(self, _private: Private) -> Map { + let mut m = Map::::new(); + m.as_mut().copy_from(self); + m + } +} + +impl<'msg, K, V> IntoProxied> for MapMut<'msg, K, V> +where + K: Proxied, + V: ProxiedInMapValue, + View<'msg, V>: IntoProxied, +{ + fn into_proxied(self, _private: Private) -> Map { + self.into_view().into_proxied(Private) + } +} + /// An iterator visiting all key-value pairs in arbitrary order. /// /// The iterator element type is `(View, View)`. @@ -373,8 +430,8 @@ impl<'msg, K: ?Sized, V: ?Sized> MapIter<'msg, K, V> { impl<'msg, K, V> Iterator for MapIter<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type Item = (View<'msg, K>, View<'msg, V>); @@ -385,8 +442,8 @@ where impl<'msg, K, V> IntoIterator for MapView<'msg, K, V> where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'msg, K, V>; type Item = (View<'msg, K>, View<'msg, V>); @@ -398,8 +455,8 @@ where impl<'msg, K, V> IntoIterator for &'msg Map where - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'msg, K, V>; type Item = (View<'msg, K>, View<'msg, V>); @@ -412,8 +469,8 @@ where impl<'a, 'msg, K, V> IntoIterator for &'a MapView<'msg, K, V> where 'msg: 'a, - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'msg, K, V>; type Item = (View<'msg, K>, View<'msg, V>); @@ -426,8 +483,8 @@ where impl<'a, 'msg, K, V> IntoIterator for &'a MapMut<'msg, K, V> where 'msg: 'a, - K: Proxied + ?Sized + 'msg, - V: ProxiedInMapValue + ?Sized + 'msg, + K: Proxied + 'msg, + V: ProxiedInMapValue + 'msg, { type IntoIter = MapIter<'a, K, V>; // The View's are valid for 'a instead of 'msg. @@ -441,8 +498,8 @@ where impl<'msg, 'k, 'v, KView, VView, K, V> Extend<(KView, VView)> for MapMut<'msg, K, V> where - K: Proxied + ?Sized + 'msg + 'k, - V: ProxiedInMapValue + ?Sized + 'msg + 'v, + K: Proxied + 'msg + 'k, + V: ProxiedInMapValue + 'msg + 'v, KView: Into>, VView: IntoProxied, { @@ -459,7 +516,7 @@ mod tests { use crate::{ProtoBytes, ProtoStr, ProtoString}; use googletest::prelude::*; - #[test] + #[googletest::test] fn test_proxied_scalar() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -485,7 +542,7 @@ mod tests { assert_that!(map_view_4.is_empty(), eq(false)); } - #[test] + #[googletest::test] fn test_proxied_str() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -510,7 +567,7 @@ mod tests { assert_that!(map_view_4.is_empty(), eq(false)); } - #[test] + #[googletest::test] fn test_proxied_iter() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -557,7 +614,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_overwrite_insert() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -567,7 +624,7 @@ mod tests { assert_that!(map.as_mut(), unordered_elements_are![eq((0, ProtoStr::from_str("buzz"))),]); } - #[test] + #[googletest::test] fn test_extend() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -604,7 +661,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_copy_from() { let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); @@ -635,7 +692,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_all_maps_can_be_constructed() { macro_rules! gen_proto_values { ($key_t:ty, $($value_t:ty),*) => { @@ -657,7 +714,7 @@ mod tests { gen_proto_keys!(i32, u32, i64, u64, bool, ProtoString); } - #[test] + #[googletest::test] fn test_dbg() { let mut map = Map::::new(); assert_that!(format!("{:?}", map.as_view()), eq("MapView(\"i32\", \"f64\")")); diff --git a/rust/optional.rs b/rust/optional.rs index 1f5cb9ad46..221f4d9ed2 100644 --- a/rust/optional.rs +++ b/rust/optional.rs @@ -23,14 +23,14 @@ use std::ptr; /// /// Two `Optional`s are equal if they match both presence and the field values. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Optional { +pub enum Optional { /// The field is set; it is present in the serialized message. /// /// - For an `_opt()` accessor, this contains a `View`. /// - For a `_mut()` accessor, this contains a [`PresentField`] that can be /// used to access the current value, convert to [`Mut`], clear presence, /// or set a new value. - Set(SetVal), + Set(T), /// The field is unset; it is absent in the serialized message. /// @@ -38,7 +38,7 @@ pub enum Optional { /// the default value. /// - For a `_mut()` accessor, this contains an [`AbsentField`] that can be /// used to access the default or set a new value. - Unset(UnsetVal), + Unset(T), } impl Optional { @@ -53,9 +53,7 @@ impl Optional { pub fn new(val: T, is_set: bool) -> Self { if is_set { Optional::Set(val) } else { Optional::Unset(val) } } -} -impl Optional { /// Converts into an `Option` of the set value, ignoring any unset value. pub fn into_option(self) -> Option { if let Optional::Set(x) = self { Some(x) } else { None } diff --git a/rust/prelude.rs b/rust/prelude.rs new file mode 100644 index 0000000000..d32f2ae298 --- /dev/null +++ b/rust/prelude.rs @@ -0,0 +1,16 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +//! Prelude for the Protobuf Rust API. +//! This module contains only non-struct items that are needed for extremely +//! common usages of the generated types from application code. Any struct +//! or less common items should be imported normally. + +pub use crate::{ + proto, AsMut, AsView, Clear, ClearAndParse, IntoMut, IntoView, Message, MessageMut, + MessageView, Parse, Serialize, +}; diff --git a/rust/primitive.rs b/rust/primitive.rs index b5f4b38108..bf5af332a9 100644 --- a/rust/primitive.rs +++ b/rust/primitive.rs @@ -4,34 +4,39 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::__internal::Private; -use crate::{IntoProxied, Proxied, View, ViewProxy}; +use crate::__internal::SealedInternal; +use crate::{AsView, IntoView, Proxied, Proxy, ViewProxy}; macro_rules! impl_singular_primitives { ($($t:ty),*) => { $( + impl SealedInternal for $t {} + impl Proxied for $t { type View<'msg> = $t; } - impl<'msg> ViewProxy<'msg> for $t { - type Proxied = $t; + impl<'msg> Proxy<'msg> for $t { + } - fn as_view(&self) -> View<'_, Self::Proxied> { - *self - } + impl AsView for $t { + type Proxied = $t; - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> { - self - } + fn as_view(&self) -> $t { + *self + } } - impl IntoProxied<$t> for $t { - fn into_proxied(self, _private: Private) -> $t { - self + impl<'msg> IntoView<'msg> for $t { + fn into_view<'shorter>(self) -> $t + where + 'msg: 'shorter { + self } } + impl<'msg> ViewProxy<'msg> for $t {} + // ProxiedInRepeated is implemented in {cpp,upb}.rs )* } diff --git a/rust/proto_macro.rs b/rust/proto_macro.rs index a70afffa62..b3f937d605 100644 --- a/rust/proto_macro.rs +++ b/rust/proto_macro.rs @@ -40,6 +40,8 @@ macro_rules! proto { #[macro_export(local_inner_macros)] #[doc(hidden)] macro_rules! proto_internal { + // @merge rules are used to find a trailing ..expr on the message and call merge_from on it + // before the fields of the message are set. (@merge $msg:ident $ident:ident : $expr:expr, $($rest:tt)*) => { proto_internal!(@merge $msg $($rest)*); }; @@ -51,52 +53,224 @@ macro_rules! proto_internal { $msg.merge_from($expr); }; - // nested message, - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $field:ident : $($value:tt)* }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : $($msgtype)::+ { $field : $($value)* }); + // @msg rules are used to set the fields of the message. There is a lot of duplication here + // because we need to parse the message type using a :: separated list of identifiers. + // nested message and trailing fields + (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@msg $msg $submsg : $($msgtype)::+ { $($value)* }); proto_internal!(@msg $msg $($rest)*); }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $field:ident : $($value:tt)* }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : ::$($msgtype)::+ { $field : $($value)* }); + // nested message with leading :: on type and trailing fields + (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@msg $msg $submsg : ::$($msgtype)::+ { $($value)* }); proto_internal!(@msg $msg $($rest)*); }; + // nested message using __ + (@msg $msg:ident $submsg:ident : __ { $($value:tt)* }) => { + { + let mut $msg = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + } + }; // nested message - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $field:ident : $($value:tt)* }) => { + (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { $($value:tt)* }) => { { let mut $msg: <$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); - proto_internal!(@merge $msg $field : $($value)*); - proto_internal!(@msg $msg $field : $($value)*); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); } }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $field:ident : $($value:tt)* }) => { + // nested message with leading :: + (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { $($value:tt)* }) => { { let mut $msg: <::$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); - proto_internal!(@merge $msg $field : $($value)*); - proto_internal!(@msg $msg $field : $($value)*); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); } }; - // empty nested message, - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : $($msgtype)::+ { }); - proto_internal!(@msg $msg $($rest)*); - }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { }, $($rest:tt)*) => { - proto_internal!(@msg $msg $submsg : ::$($msgtype)::+ { }); + // field with array literal and trailing fields + (@msg $msg:ident $ident:ident : [$($elems:tt)*], $($rest:tt)*) => { + proto_internal!(@msg $msg $ident : [$($elems)*]); proto_internal!(@msg $msg $($rest)*); }; - - // empty nested message - (@msg $msg:ident $submsg:ident : $($msgtype:ident)::+ { }) => { + // field with array literal, calls out to @array to look for nested messages + (@msg $msg:ident $ident:ident : [$($elems:tt)*]) => { { - let mut $msg: <$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); + let _repeated = $crate::__internal::paste!($msg.[<$ident>]()); + let elems = proto_internal!(@array $msg _repeated [] $($elems)*); + $crate::__internal::paste!($msg.[](elems.into_iter())); } }; - (@msg $msg:ident $submsg:ident : ::$($msgtype:ident)::+ { }) => { - { - let mut $msg: <::$($msgtype)::+ as $crate::MutProxied>::Mut<'_> = $crate::__internal::paste!($msg.[<$submsg _mut>]()); - } + + // @array searches through an array literal for nested messages. + // If a message is found then we recursively call the macro on it to set the fields. + // This will create an array literal of owned messages to be used while setting the field. + // For primitive types they should just be passed through as an $expr. + // The array literal is constructed recursively, so the [] case has to be handled separately so + // that we can properly insert commas. This leads to a lot of duplication. + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [$($vals:expr),+] __ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + $($vals),+ , + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + + // Message nested in [] literal + (@array $msg:ident $repeated:ident [$($vals:expr),+] __ { $($value:tt)* }) => { + [ + $($vals),+ , + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [] __ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + + // Message nested in [] literal + (@array $msg:ident $repeated:ident [] __ { $($value:tt)* }) => { + [ + { + let mut $msg = $crate::get_repeated_default_value($crate::__internal::Private, $repeated); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + // End of __ repeated, now we need to handle named types + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [$($vals:expr),+] $($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + $($vals),+ , + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // Message nested in [] literal with leading :: on type and trailing array items + (@array $msg:ident $repeated:ident [$($vals:expr),+] ::$($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + $($vals),+ , + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // Message nested in [] literal + (@array $msg:ident $repeated:ident [$($vals:expr),+] $($msgtype:ident)::+ { $($value:tt)* }) => { + [ + $($vals),+ , + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + // Message nested in [] literal with leading :: on type + (@array $msg:ident $repeated:ident [$($vals:expr),+] ::$($msgtype:ident)::+ { $($value:tt)* }) => { + [ + $($vals),+ , + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + // Message nested in array literal with trailing array items + (@array $msg:ident $repeated:ident [] $($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // with leading :: + (@array $msg:ident $repeated:ident [] ::$($msgtype:ident)::+ { $($value:tt)* }, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [ + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] $($rest)*) + }; + // Message nested in [] literal + (@array $msg:ident $repeated:ident [] $($msgtype:ident)::+ { $($value:tt)* }) => { + [ + { + let mut $msg = $($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + (@array $msg:ident $repeated:ident [] ::$($msgtype:ident)::+ { $($value:tt)* }) => { + [ + { + let mut $msg = ::$($msgtype)::+::new(); + proto_internal!(@merge $msg $($value)*); + proto_internal!(@msg $msg $($value)*); + $msg + } + ] + }; + + (@array $msg:ident $repeated:ident [$($vals:expr),+] $expr:expr, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [$($vals),+, $expr] $($rest)*) + }; + (@array $msg:ident $repeated:ident [$($vals:expr),+] $expr:expr) => { + [$($vals),+, $expr] + }; + (@array $msg:ident $repeated:ident [] $expr:expr, $($rest:tt)*) => { + proto_internal!(@array $msg $repeated [$expr] $($rest)*) + }; + (@array $msg:ident $repeated:ident [] $expr:expr) => { + [$expr] + }; + (@array $msg:ident $repeated:ident []) => { + [] }; // field: expr, diff --git a/rust/protobuf.rs b/rust/protobuf.rs index c83b94953a..13d0b5f14d 100644 --- a/rust/protobuf.rs +++ b/rust/protobuf.rs @@ -21,3 +21,5 @@ use protobuf_cpp as kernel; use protobuf_upb as kernel; pub use kernel::__public::*; + +pub use kernel::prelude; diff --git a/rust/proxied.rs b/rust/proxied.rs index b1903a4e43..32147908b5 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -44,7 +44,7 @@ //! implemented the concept of "proxy" types. Proxy types are a reference-like //! indirection between the user and the internal memory representation. -use crate::__internal::Private; +use crate::__internal::{Private, SealedInternal}; use std::fmt::Debug; /// A type that can be accessed through a reference-like proxy. @@ -52,11 +52,11 @@ use std::fmt::Debug; /// An instance of a `Proxied` can be accessed immutably via `Proxied::View`. /// /// All Protobuf field types implement `Proxied`. -pub trait Proxied: Sized { +pub trait Proxied: SealedInternal + AsView + Sized { /// The proxy type that provides shared access to a `T`, like a `&'msg T`. /// /// Most code should use the type alias [`View`]. - type View<'msg>: ViewProxy<'msg, Proxied = Self> + Copy + Send + type View<'msg>: ViewProxy<'msg, Proxied = Self> where Self: 'msg; } @@ -67,12 +67,12 @@ pub trait Proxied: Sized { /// and immutably via `MutProxied::View`. /// /// `MutProxied` is implemented by message, map and repeated field types. -pub trait MutProxied: Proxied { +pub trait MutProxied: SealedInternal + Proxied + AsMut { /// The proxy type that provides exclusive mutable access to a `T`, like a /// `&'msg mut T`. /// /// Most code should use the type alias [`Mut`]. - type Mut<'msg>: MutProxy<'msg, Proxied = Self> + type Mut<'msg>: MutProxy<'msg, MutProxied = Self> where Self: 'msg; } @@ -90,12 +90,13 @@ pub type View<'msg, T> = ::View<'msg>; #[allow(dead_code)] pub type Mut<'msg, T> = ::Mut<'msg>; -/// Declares conversion operations common to all views. +/// Used to semantically do a cheap "to-reference" conversion. This is +/// implemented on both owned `Proxied` types as well as ViewProxy and MutProxy +/// types. /// -/// This trait is intentionally made non-object-safe to prevent a potential -/// future incompatible change. -pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { - type Proxied: 'msg + Proxied + ?Sized; +/// On ViewProxy this will behave as a reborrow into a shorter lifetime. +pub trait AsView: SealedInternal { + type Proxied: Proxied; /// Converts a borrow into a `View` with the lifetime of that borrow. /// @@ -107,7 +108,7 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { /// /// For example, the call to `.as_view()` in the following snippet /// wouldn't be necessary in concrete code: - /// ``` + /// ```ignore /// fn reborrow<'a, 'b, T>(x: &'b View<'a, T>) -> View<'b, T> /// where 'a: 'b, T: Proxied /// { @@ -117,7 +118,16 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { /// /// [invariant]: https://doc.rust-lang.org/nomicon/subtyping.html#variance fn as_view(&self) -> View<'_, Self::Proxied>; +} +/// Used to turn another 'borrow' into a ViewProxy. +/// +/// On a MutProxy this borrows to a View (semantically matching turning a `&mut +/// T` into a `&T`). +/// +/// On a ViewProxy this will behave as a reborrow into a shorter lifetime +/// (semantically matching a `&'a T` into a `&'b T` where `'a: 'b`). +pub trait IntoView<'msg>: SealedInternal + AsView { /// Converts into a `View` with a potentially shorter lifetime. /// /// In non-generic code we don't need to use `into_view` because the proxy @@ -126,7 +136,7 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { /// `into_view` to explicitly perform the operation that in concrete /// code coercion would perform implicitly. /// - /// ``` + /// ```ignore /// fn reborrow_generic_view_into_view<'a, 'b, T>( /// x: View<'a, T>, /// y: View<'b, T>, @@ -148,38 +158,22 @@ pub trait ViewProxy<'msg>: 'msg + Sync + Unpin + Sized + Debug { 'msg: 'shorter; } -/// Declares operations common to all mutators. +/// Used to semantically do a cheap "to-mut-reference" conversion. This is +/// implemented on both owned `Proxied` types as well as MutProxy types. /// -/// This trait is intentionally made non-object-safe to prevent a potential -/// future incompatible change. -pub trait MutProxy<'msg>: ViewProxy<'msg> -where - Self::Proxied: MutProxied, -{ - /// Gets an immutable view of this field. This is shorthand for `as_view`. - /// - /// This provides a shorter lifetime than `into_view` but can also be called - /// multiple times - if the result of `get` is not living long enough - /// for your use, use that instead. - fn get(&self) -> View<'_, Self::Proxied> { - self.as_view() - } +/// On MutProxy this will behave as a reborrow into a shorter lifetime. +pub trait AsMut: SealedInternal + AsView { + type MutProxied: MutProxied; /// Converts a borrow into a `Mut` with the lifetime of that borrow. - /// - /// This function enables calling multiple methods consuming `self`, for - /// example: - /// - /// ```ignore - /// let mut sub: Mut = msg.submsg_mut(); - /// sub.as_mut().field_x_mut().set(10); // field_x_mut is fn(self) - /// sub.field_y_mut().set(20); // `sub` is now consumed - /// ``` - /// - /// `as_mut` is also useful in generic code to explicitly perform the - /// operation that in concrete code coercion would perform implicitly. - fn as_mut(&mut self) -> Mut<'_, Self::Proxied>; + fn as_mut(&mut self) -> Mut<'_, Self::MutProxied>; +} +/// Used to turn another 'borrow' into a MutProxy. +/// +/// On a MutProxy this will behave as a reborrow into a shorter lifetime +/// (semantically matching a `&mut 'a T` into a `&mut 'b T` where `'a: 'b`). +pub trait IntoMut<'msg>: SealedInternal + AsMut { /// Converts into a `Mut` with a potentially shorter lifetime. /// /// In non-generic code we don't need to use `into_mut` because the proxy @@ -188,7 +182,7 @@ where /// `into_mut` to explicitly perform the operation that in concrete code /// coercion would perform implicitly. /// - /// ``` + /// ```ignore /// fn reborrow_generic_mut_into_mut<'a, 'b, T>(x: Mut<'a, T>, y: Mut<'b, T>) -> [Mut<'b, T>; 2] /// where /// T: Proxied, @@ -202,11 +196,39 @@ where /// ``` /// /// [invariant]: https://doc.rust-lang.org/nomicon/subtyping.html#variance - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> + fn into_mut<'shorter>(self) -> Mut<'shorter, Self::MutProxied> where 'msg: 'shorter; } +/// Declares conversion operations common to all proxies (both views and mut +/// proxies). +/// +/// This trait is intentionally made non-object-safe to prevent a potential +/// future incompatible change. +pub trait Proxy<'msg>: + SealedInternal + 'msg + IntoView<'msg> + Sync + Unpin + Sized + Debug +{ +} + +/// Declares conversion operations common to view proxies. +pub trait ViewProxy<'msg>: SealedInternal + Proxy<'msg> + Send {} + +/// Declares operations common to all mut proxies. +/// +/// This trait is intentionally made non-object-safe to prevent a potential +/// future incompatible change. +pub trait MutProxy<'msg>: SealedInternal + Proxy<'msg> + AsMut + IntoMut<'msg> { + /// Gets an immutable view of this field. This is shorthand for `as_view`. + /// + /// This provides a shorter lifetime than `into_view` but can also be called + /// multiple times - if the result of `get` is not living long enough + /// for your use, use that instead. + fn get(&self) -> View<'_, Self::Proxied> { + self.as_view() + } +} + /// A value to `Proxied`-value conversion that consumes the input value. /// /// All setter functions accept types that implement `IntoProxied`. The purpose @@ -220,6 +242,12 @@ pub trait IntoProxied { fn into_proxied(self, _private: Private) -> T; } +impl IntoProxied for T { + fn into_proxied(self, _private: Private) -> T { + self + } +} + #[cfg(test)] mod tests { use super::*; @@ -240,33 +268,57 @@ mod tests { } } + impl SealedInternal for MyProxied {} + impl Proxied for MyProxied { type View<'msg> = MyProxiedView<'msg>; } + impl AsView for MyProxied { + type Proxied = Self; + fn as_view(&self) -> MyProxiedView<'_> { + self.as_view() + } + } + impl MutProxied for MyProxied { type Mut<'msg> = MyProxiedMut<'msg>; } + impl AsMut for MyProxied { + type MutProxied = Self; + fn as_mut(&mut self) -> MyProxiedMut<'_> { + self.as_mut() + } + } + #[derive(Debug, Clone, Copy)] struct MyProxiedView<'msg> { my_proxied_ref: &'msg MyProxied, } + impl<'msg> SealedInternal for MyProxiedView<'msg> {} + impl MyProxiedView<'_> { fn val(&self) -> &str { &self.my_proxied_ref.val } } - impl<'msg> ViewProxy<'msg> for MyProxiedView<'msg> { + impl<'msg> Proxy<'msg> for MyProxiedView<'msg> {} + + impl<'msg> ViewProxy<'msg> for MyProxiedView<'msg> {} + + impl<'msg> AsView for MyProxiedView<'msg> { type Proxied = MyProxied; - fn as_view(&self) -> View<'msg, MyProxied> { + fn as_view(&self) -> MyProxiedView<'msg> { *self } + } - fn into_view<'shorter>(self) -> View<'shorter, MyProxied> + impl<'msg> IntoView<'msg> for MyProxiedView<'msg> { + fn into_view<'shorter>(self) -> MyProxiedView<'shorter> where 'msg: 'shorter, { @@ -279,12 +331,19 @@ mod tests { my_proxied_ref: &'msg mut MyProxied, } - impl<'msg> ViewProxy<'msg> for MyProxiedMut<'msg> { + impl<'msg> SealedInternal for MyProxiedMut<'msg> {} + + impl<'msg> Proxy<'msg> for MyProxiedMut<'msg> {} + + impl<'msg> AsView for MyProxiedMut<'msg> { type Proxied = MyProxied; - fn as_view(&self) -> View<'_, MyProxied> { + fn as_view(&self) -> MyProxiedView<'_> { MyProxiedView { my_proxied_ref: self.my_proxied_ref } } + } + + impl<'msg> IntoView<'msg> for MyProxiedMut<'msg> { fn into_view<'shorter>(self) -> View<'shorter, MyProxied> where 'msg: 'shorter, @@ -293,12 +352,16 @@ mod tests { } } - impl<'msg> MutProxy<'msg> for MyProxiedMut<'msg> { - fn as_mut(&mut self) -> Mut<'_, MyProxied> { + impl<'msg> AsMut for MyProxiedMut<'msg> { + type MutProxied = MyProxied; + + fn as_mut(&mut self) -> MyProxiedMut<'_> { MyProxiedMut { my_proxied_ref: self.my_proxied_ref } } + } - fn into_mut<'shorter>(self) -> Mut<'shorter, MyProxied> + impl<'msg> IntoMut<'msg> for MyProxiedMut<'msg> { + fn into_mut<'shorter>(self) -> MyProxiedMut<'shorter> where 'msg: 'shorter, { @@ -306,7 +369,9 @@ mod tests { } } - #[test] + impl<'msg> MutProxy<'msg> for MyProxiedMut<'msg> {} + + #[googletest::test] fn test_as_view() { let my_proxied = MyProxied { val: "Hello World".to_string() }; @@ -321,7 +386,7 @@ mod tests { x.into_view() // OK: we return the same lifetime as we got in. } - #[test] + #[googletest::test] fn test_mut_into_view() { let mut my_proxied = MyProxied { val: "Hello World".to_string() }; reborrow_mut_into_view(my_proxied.as_mut()); @@ -329,7 +394,7 @@ mod tests { fn require_unified_lifetimes<'msg>(_x: Mut<'msg, MyProxied>, _y: View<'msg, MyProxied>) {} - #[test] + #[googletest::test] fn test_require_unified_lifetimes() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let my_mut = my_proxied.as_mut(); @@ -354,7 +419,7 @@ mod tests { [x.as_view(), y.as_view()] } - #[test] + #[googletest::test] fn test_reborrow_generic_as_view() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let mut my_mut = my_proxied.as_mut(); @@ -381,7 +446,7 @@ mod tests { [x.into_view(), y] } - #[test] + #[googletest::test] fn test_reborrow_generic_into_view() { let my_proxied = MyProxied { val: "Hello1".to_string() }; let my_view = my_proxied.as_view(); @@ -401,7 +466,7 @@ mod tests { [x.into_view(), y] } - #[test] + #[googletest::test] fn test_reborrow_generic_mut_into_view() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let my_mut = my_proxied.as_mut(); @@ -421,10 +486,11 @@ mod tests { // `[x, y]` fails to compile because `'a` is not the same as `'b` and the `Mut` // lifetime parameter is (conservatively) invariant. // `[x.as_mut(), y]` fails because that borrow cannot outlive `'b`. - [x.into_mut(), y] + let tmp: Mut<'b, T> = x.into_mut(); + [tmp, y] } - #[test] + #[googletest::test] fn test_reborrow_generic_mut_into_mut() { let mut my_proxied = MyProxied { val: "Hello1".to_string() }; let my_mut = my_proxied.as_mut(); diff --git a/rust/repeated.rs b/rust/repeated.rs index 99c935b4f4..1fd5e0e364 100644 --- a/rust/repeated.rs +++ b/rust/repeated.rs @@ -15,8 +15,9 @@ use std::iter::FusedIterator; use std::marker::PhantomData; use crate::{ - IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy, - __internal::Private, + AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, View, + ViewProxy, + __internal::{Private, SealedInternal}, __runtime::{InnerRepeated, InnerRepeatedMut, RawRepeatedField}, }; @@ -232,15 +233,6 @@ where } } -impl IntoProxied> for Repeated -where - T: ProxiedInRepeated, -{ - fn into_proxied(self, _private: Private) -> Repeated { - self - } -} - impl<'msg, T> IntoProxied> for RepeatedView<'msg, T> where T: 'msg + ProxiedInRepeated, @@ -390,6 +382,19 @@ where type View<'msg> = RepeatedView<'msg, T> where Repeated: 'msg; } +impl SealedInternal for Repeated where T: ProxiedInRepeated {} + +impl AsView for Repeated +where + T: ProxiedInRepeated, +{ + type Proxied = Self; + + fn as_view(&self) -> RepeatedView<'_, T> { + self.as_view() + } +} + impl MutProxied for Repeated where T: ProxiedInRepeated, @@ -397,17 +402,37 @@ where type Mut<'msg> = RepeatedMut<'msg, T> where Repeated: 'msg; } -impl<'msg, T> ViewProxy<'msg> for RepeatedView<'msg, T> +impl AsMut for Repeated +where + T: ProxiedInRepeated, +{ + type MutProxied = Self; + + fn as_mut(&mut self) -> RepeatedMut<'_, T> { + self.as_mut() + } +} + +impl<'msg, T> SealedInternal for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> Proxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> AsView for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg, { type Proxied = Repeated; #[inline] - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> View<'msg, Self::Proxied> { *self } +} +impl<'msg, T> IntoView<'msg> for RepeatedView<'msg, T> +where + T: ProxiedInRepeated + 'msg, +{ #[inline] fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> where @@ -417,19 +442,30 @@ where } } -impl<'msg, T> ViewProxy<'msg> for RepeatedMut<'msg, T> +impl<'msg, T> ViewProxy<'msg> for RepeatedView<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> SealedInternal for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> Proxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} + +impl<'msg, T> AsView for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg, { type Proxied = Repeated; #[inline] - fn as_view(&self) -> View<'_, Self::Proxied> { + fn as_view(&self) -> RepeatedView<'_, T> { RepeatedView { raw: self.inner.raw, _phantom: PhantomData } } +} +impl<'msg, T> IntoView<'msg> for RepeatedMut<'msg, T> +where + T: ProxiedInRepeated + 'msg, +{ #[inline] - fn into_view<'shorter>(self) -> View<'shorter, Self::Proxied> + fn into_view<'shorter>(self) -> RepeatedView<'shorter, T> where 'msg: 'shorter, { @@ -437,17 +473,24 @@ where } } -impl<'msg, T> MutProxy<'msg> for RepeatedMut<'msg, T> +impl<'msg, T> AsMut for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg, { + type MutProxied = Repeated; + #[inline] - fn as_mut(&mut self) -> Mut<'_, Self::Proxied> { + fn as_mut(&mut self) -> RepeatedMut<'_, T> { RepeatedMut { inner: self.inner, _phantom: PhantomData } } +} +impl<'msg, T> IntoMut<'msg> for RepeatedMut<'msg, T> +where + T: ProxiedInRepeated + 'msg, +{ #[inline] - fn into_mut<'shorter>(self) -> Mut<'shorter, Self::Proxied> + fn into_mut<'shorter>(self) -> RepeatedMut<'shorter, T> where 'msg: 'shorter, { @@ -455,6 +498,8 @@ where } } +impl<'msg, T> MutProxy<'msg> for RepeatedMut<'msg, T> where T: ProxiedInRepeated + 'msg {} + impl<'msg, T> iter::Iterator for RepeatedIter<'msg, T> where T: ProxiedInRepeated + 'msg, @@ -540,7 +585,7 @@ mod tests { use super::*; use googletest::prelude::*; - #[test] + #[googletest::test] fn test_primitive_repeated() { macro_rules! primitive_repeated_tests { ($($t:ty => [$($vals:expr),* $(,)?]),* $(,)?) => { @@ -581,7 +626,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn test_repeated_extend() { let mut r = Repeated::::new(); @@ -598,7 +643,7 @@ mod tests { assert_that!(r.as_mut(), elements_are![eq(0), eq(1), eq(2), eq(3)]); } - #[test] + #[googletest::test] fn test_repeated_iter_into_proxied() { let r: Repeated = [0, 1, 2, 3].into_iter().into_proxied(Private); assert_that!(r.as_view(), elements_are![eq(0), eq(1), eq(2), eq(3)]); diff --git a/rust/shared.rs b/rust/shared.rs index 403fcb0ea5..59e7bd8205 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -22,12 +22,22 @@ use std::fmt; /// These are the items protobuf users can access directly. #[doc(hidden)] pub mod __public { - pub use crate::codegen_traits::{Message, MessageMut, MessageView}; + pub use crate::codegen_traits::{ + create::Parse, + interop::{MessageMutInterop, MessageViewInterop, OwnedMessageInterop}, + read::Serialize, + write::{Clear, ClearAndParse}, + Message, MessageMut, MessageView, + }; + pub use crate::cord::{ProtoBytesCow, ProtoStringCow}; pub use crate::r#enum::{Enum, UnknownEnumValue}; pub use crate::map::{Map, MapIter, MapMut, MapView, ProxiedInMapValue}; pub use crate::optional::Optional; pub use crate::proto; - pub use crate::proxied::{IntoProxied, Mut, MutProxied, MutProxy, Proxied, View, ViewProxy}; + pub use crate::proxied::{ + AsMut, AsView, IntoMut, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Proxied, Proxy, + View, ViewProxy, + }; pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, }; @@ -36,6 +46,8 @@ pub mod __public { } pub use __public::*; +pub mod prelude; + /// Everything in `__internal` is allowed to change without it being considered /// a breaking change for the protobuf library. Nothing in here should be /// exported in `protobuf.rs`. @@ -45,14 +57,15 @@ pub mod __internal; /// Everything in `__runtime` is allowed to change without it being considered /// a breaking change for the protobuf library. Nothing in here should be /// exported in `protobuf.rs`. -#[cfg(cpp_kernel)] +#[cfg(all(bzl, cpp_kernel))] #[path = "cpp.rs"] pub mod __runtime; -#[cfg(upb_kernel)] +#[cfg(any(not(bzl), upb_kernel))] #[path = "upb.rs"] pub mod __runtime; mod codegen_traits; +mod cord; #[path = "enum.rs"] mod r#enum; mod map; @@ -63,6 +76,18 @@ mod proxied; mod repeated; mod string; +#[cfg(not(bzl))] +#[path = "upb/lib.rs"] +mod upb; + +#[cfg(not(bzl))] +mod utf8; + +// Forces the utf8 crate to be accessible from crate::. +#[cfg(bzl)] +#[allow(clippy::single_component_path_imports)] +use utf8; + // If the Upb and C++ kernels are both linked into the same binary, this symbol // will be defined twice and cause a link error. #[no_mangle] @@ -91,3 +116,10 @@ impl fmt::Display for SerializeError { write!(f, "Couldn't serialize proto into bytes (depth too deep or missing required fields)") } } + +pub fn get_repeated_default_value( + _: __internal::Private, + _: repeated::RepeatedView<'_, T>, +) -> T { + Default::default() +} diff --git a/rust/string.rs b/rust/string.rs index a94f4efc00..7c758783be 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -9,9 +9,12 @@ #![allow(dead_code)] #![allow(unused)] -use crate::__internal::Private; +use crate::__internal::{Private, SealedInternal}; use crate::__runtime::{InnerProtoString, PtrAndLen, RawMessage}; -use crate::{IntoProxied, Mut, MutProxied, MutProxy, Optional, Proxied, View, ViewProxy}; +use crate::{ + utf8::Utf8Chunks, AsView, IntoProxied, IntoView, Mut, MutProxied, MutProxy, Optional, Proxied, + Proxy, View, ViewProxy, +}; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use std::convert::{AsMut, AsRef}; @@ -23,7 +26,6 @@ use std::ops::{Deref, DerefMut}; use std::ptr; use std::rc::Rc; use std::sync::Arc; -use utf8::Utf8Chunks; pub struct ProtoBytes { pub(crate) inner: InnerProtoString, @@ -36,6 +38,15 @@ impl ProtoBytes { pub fn into_inner(self, _private: Private) -> InnerProtoString { self.inner } + + #[doc(hidden)] + pub fn from_inner(_private: Private, inner: InnerProtoString) -> ProtoBytes { + Self { inner } + } + + pub fn as_view(&self) -> &[u8] { + self.inner.as_bytes() + } } impl AsRef<[u8]> for ProtoBytes { @@ -56,13 +67,17 @@ impl From<&[u8; N]> for ProtoBytes { } } +impl SealedInternal for ProtoBytes {} + impl Proxied for ProtoBytes { type View<'msg> = &'msg [u8]; } -impl IntoProxied for ProtoBytes { - fn into_proxied(self, _private: Private) -> ProtoBytes { - self +impl AsView for ProtoBytes { + type Proxied = Self; + + fn as_view(&self) -> &[u8] { + self.as_view() } } @@ -114,13 +129,19 @@ impl IntoProxied for Arc<[u8]> { } } -impl<'msg> ViewProxy<'msg> for &'msg [u8] { +impl SealedInternal for &[u8] {} + +impl<'msg> Proxy<'msg> for &'msg [u8] {} + +impl AsView for &[u8] { type Proxied = ProtoBytes; fn as_view(&self) -> &[u8] { self } +} +impl<'msg> IntoView<'msg> for &'msg [u8] { fn into_view<'shorter>(self) -> &'shorter [u8] where 'msg: 'shorter, @@ -129,13 +150,24 @@ impl<'msg> ViewProxy<'msg> for &'msg [u8] { } } +impl<'msg> ViewProxy<'msg> for &'msg [u8] {} + /// The bytes were not valid UTF-8. #[derive(Debug, PartialEq)] -pub struct Utf8Error(pub(crate) ()); +pub struct Utf8Error { + pub(crate) inner: std::str::Utf8Error, +} +impl std::fmt::Display for Utf8Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.inner.fmt(f) + } +} + +impl std::error::Error for Utf8Error {} impl From for Utf8Error { - fn from(_: std::str::Utf8Error) -> Utf8Error { - Utf8Error(()) + fn from(inner: std::str::Utf8Error) -> Utf8Error { + Utf8Error { inner } } } @@ -168,6 +200,10 @@ pub struct ProtoString { } impl ProtoString { + pub fn as_view(&self) -> &ProtoStr { + unsafe { ProtoStr::from_utf8_unchecked(self.as_bytes()) } + } + pub fn as_bytes(&self) -> &[u8] { self.inner.as_bytes() } @@ -178,6 +214,19 @@ impl ProtoString { pub fn into_inner(self, _private: Private) -> InnerProtoString { self.inner } + + #[doc(hidden)] + pub fn from_inner(_private: Private, inner: InnerProtoString) -> ProtoString { + Self { inner } + } +} + +impl SealedInternal for ProtoString {} + +impl AsRef<[u8]> for ProtoString { + fn as_ref(&self) -> &[u8] { + self.inner.as_bytes() + } } impl From for ProtoBytes { @@ -198,11 +247,9 @@ impl From<&[u8]> for ProtoString { } } -impl IntoProxied for ProtoString { - fn into_proxied(self, _private: Private) -> ProtoString { - self - } -} +impl SealedInternal for &str {} + +impl SealedInternal for &ProtoStr {} impl IntoProxied for &str { fn into_proxied(self, _private: Private) -> ProtoString { @@ -479,13 +526,25 @@ impl Proxied for ProtoString { type View<'msg> = &'msg ProtoStr; } -impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { +impl AsView for ProtoString { + type Proxied = Self; + + fn as_view(&self) -> &ProtoStr { + self.as_view() + } +} + +impl<'msg> Proxy<'msg> for &'msg ProtoStr {} + +impl AsView for &ProtoStr { type Proxied = ProtoString; fn as_view(&self) -> &ProtoStr { self } +} +impl<'msg> IntoView<'msg> for &'msg ProtoStr { fn into_view<'shorter>(self) -> &'shorter ProtoStr where 'msg: 'shorter, @@ -494,6 +553,8 @@ impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { } } +impl<'msg> ViewProxy<'msg> for &'msg ProtoStr {} + /// Implements `PartialCmp` and `PartialEq` for the `lhs` against the `rhs` /// using `AsRef<[u8]>`. // TODO: consider improving to not require a `<()>` if no generics are @@ -524,6 +585,24 @@ impl_bytes_partial_cmp!( <()> str => ProtoStr, ); +impl std::fmt::Debug for ProtoString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + std::fmt::Debug::fmt(self.as_view(), f) + } +} + +impl std::fmt::Debug for ProtoBytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + std::fmt::Debug::fmt(self.as_view(), f) + } +} + +unsafe impl Sync for ProtoString {} +unsafe impl Send for ProtoString {} + +unsafe impl Send for ProtoBytes {} +unsafe impl Sync for ProtoBytes {} + #[cfg(test)] mod tests { use super::*; @@ -541,7 +620,7 @@ mod tests { // UTF-8 test cases copied from: // https://github.com/rust-lang/rust/blob/e8ee0b7/library/core/tests/str_lossy.rs - #[test] + #[googletest::test] fn proto_str_debug() { assert_eq!(&format!("{:?}", test_proto_str(b"Hello There")), "\"Hello There\""); assert_eq!( @@ -553,7 +632,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn proto_str_display() { assert_eq!(&test_proto_str(b"Hello There").to_string(), "Hello There"); assert_eq!( @@ -562,7 +641,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn proto_str_to_rust_str() { assert_eq!(test_proto_str(b"hello").to_str(), Ok("hello")); assert_eq!(test_proto_str("ศไทย中华Việt Nam".as_bytes()).to_str(), Ok("ศไทย中华Việt Nam")); @@ -575,11 +654,14 @@ mod tests { b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar", b"\xED\xA0\x80foo\xED\xBF\xBFbar", ] { - assert_eq!(test_proto_str(expect_fail).to_str(), Err(Utf8Error(())), "{expect_fail:?}"); + assert!( + matches!(test_proto_str(expect_fail).to_str(), Err(Utf8Error { inner: _ })), + "{expect_fail:?}" + ); } } - #[test] + #[googletest::test] fn proto_str_to_cow() { assert_eq!(test_proto_str(b"hello").to_cow_lossy(), Cow::Borrowed("hello")); assert_eq!( @@ -601,7 +683,7 @@ mod tests { } } - #[test] + #[googletest::test] fn proto_str_utf8_chunks() { macro_rules! assert_chunks { ($bytes:expr, $($chunks:expr),* $(,)?) => { @@ -680,7 +762,7 @@ mod tests { ); } - #[test] + #[googletest::test] fn proto_str_chars() { macro_rules! assert_chars { ($bytes:expr, $chars:expr) => { diff --git a/rust/test/cpp/debug_test.rs b/rust/test/cpp/debug_test.rs index 82b2d1b773..2b784a9a9b 100644 --- a/rust/test/cpp/debug_test.rs +++ b/rust/test/cpp/debug_test.rs @@ -3,7 +3,7 @@ use googletest::prelude::*; use optimize_for_lite_rust_proto::OptimizeForLiteTestMessage; #[cfg(not(lite_runtime))] -#[test] +#[googletest::test] fn test_debug() { let mut msg = DebugMsg::new(); msg.set_id(1); @@ -14,7 +14,7 @@ fn test_debug() { } #[cfg(lite_runtime)] -#[test] +#[googletest::test] fn test_debug_lite() { let mut msg = DebugMsg::new(); msg.set_id(1); @@ -27,7 +27,7 @@ fn test_debug_lite() { /// A message with the option set to optimize for lite will behave as a lite /// message regardless of the `lite_runtime` feature. Run this test not guarded /// by the cfg(lite_runtime) and ensure it functions as lite. -#[test] +#[googletest::test] fn test_optimize_for_lite_option() { let mut msg = OptimizeForLiteTestMessage::new(); msg.set_value("password"); diff --git a/rust/test/cpp/interop/main.rs b/rust/test/cpp/interop/main.rs index 812dab3a59..467f4559d0 100644 --- a/rust/test/cpp/interop/main.rs +++ b/rust/test/cpp/interop/main.rs @@ -6,7 +6,10 @@ // https://developers.google.com/open-source/licenses/bsd use googletest::prelude::*; +use protobuf_cpp::prelude::*; + use protobuf_cpp::__runtime::{PtrAndLen, RawMessage}; +use protobuf_cpp::{MessageMutInterop, MessageViewInterop, OwnedMessageInterop}; use unittest_rust_proto::{TestAllExtensions, TestAllTypes, TestAllTypesMut, TestAllTypesView}; macro_rules! proto_assert_eq { @@ -33,21 +36,19 @@ extern "C" { fn GetBytesExtension(msg: RawMessage) -> PtrAndLen; } -#[test] +#[googletest::test] fn send_to_cpp() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int32(7); - let i = unsafe { - TakeOwnershipAndGetOptionalInt32(msg1.__unstable_leak_cpp_repr_grant_permission_to_break()) - }; + let i = unsafe { TakeOwnershipAndGetOptionalInt32(msg1.__unstable_leak_raw_message()) }; assert_eq!(i, 7); } -#[test] +#[googletest::test] fn mutate_message_mut_in_cpp() { let mut msg1 = TestAllTypes::new(); unsafe { - MutateTestAllTypes(msg1.as_mut().__unstable_cpp_repr_grant_permission_to_break()); + MutateTestAllTypes(msg1.as_mut().__unstable_as_raw_message_mut()); } let mut msg2 = TestAllTypes::new(); @@ -58,20 +59,18 @@ fn mutate_message_mut_in_cpp() { proto_assert_eq!(msg1, msg2); } -#[test] +#[googletest::test] fn deserialize_in_rust() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); msg1.set_optional_bytes(b"some cool data I guess"); - let serialized = unsafe { - SerializeTestAllTypes(msg1.as_view().__unstable_cpp_repr_grant_permission_to_break()) - }; + let serialized = unsafe { SerializeTestAllTypes(msg1.as_view().__unstable_as_raw_message()) }; let msg2 = TestAllTypes::parse(&serialized).unwrap(); proto_assert_eq!(msg1, msg2); } -#[test] +#[googletest::test] fn deserialize_in_cpp() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -79,7 +78,7 @@ fn deserialize_in_cpp() { let data = msg1.serialize().unwrap(); let msg2 = unsafe { - TestAllTypes::__unstable_wrap_cpp_grant_permission_to_break(DeserializeTestAllTypes( + TestAllTypes::__unstable_take_ownership_of_raw_message(DeserializeTestAllTypes( (*data).as_ptr(), data.len(), )) @@ -88,7 +87,7 @@ fn deserialize_in_cpp() { proto_assert_eq!(msg1, msg2); } -#[test] +#[googletest::test] fn deserialize_in_cpp_into_mut() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -96,7 +95,7 @@ fn deserialize_in_cpp_into_mut() { let data = msg1.serialize().unwrap(); let mut raw_msg = unsafe { DeserializeTestAllTypes((*data).as_ptr(), data.len()) }; - let msg2 = TestAllTypesMut::__unstable_wrap_cpp_grant_permission_to_break(&mut raw_msg); + let msg2 = unsafe { TestAllTypesMut::__unstable_wrap_raw_message_mut(&mut raw_msg) }; proto_assert_eq!(msg1, msg2); @@ -106,7 +105,7 @@ fn deserialize_in_cpp_into_mut() { } } -#[test] +#[googletest::test] fn deserialize_in_cpp_into_view() { let mut msg1 = TestAllTypes::new(); msg1.set_optional_int64(-1); @@ -114,7 +113,7 @@ fn deserialize_in_cpp_into_view() { let data = msg1.serialize().unwrap(); let raw_msg = unsafe { DeserializeTestAllTypes((*data).as_ptr(), data.len()) }; - let msg2 = TestAllTypesView::__unstable_wrap_cpp_grant_permission_to_break(&raw_msg); + let msg2 = unsafe { TestAllTypesView::__unstable_wrap_raw_message(&raw_msg) }; proto_assert_eq!(msg1, msg2); @@ -126,16 +125,14 @@ fn deserialize_in_cpp_into_view() { // This test ensures that random fields we (Rust) don't know about don't // accidentally get destroyed by Rust. -#[test] +#[googletest::test] fn smuggle_extension() { - let msg1 = unsafe { - TestAllExtensions::__unstable_wrap_cpp_grant_permission_to_break(NewWithExtension()) - }; + let msg1 = + unsafe { TestAllExtensions::__unstable_take_ownership_of_raw_message(NewWithExtension()) }; let data = msg1.serialize().unwrap(); let mut msg2 = TestAllExtensions::parse(&data).unwrap(); - let bytes = unsafe { - GetBytesExtension(msg2.as_mut().__unstable_cpp_repr_grant_permission_to_break()).as_ref() - }; + let bytes = + unsafe { GetBytesExtension(msg2.as_mut().__unstable_as_raw_message_mut()).as_ref() }; assert_eq!(bytes, b"smuggled"); } diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index af009e9c08..4c789cdfba 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -16,10 +16,40 @@ load("@rules_rust//rust:defs.bzl", "rust_test") +rust_test( + name = "ctype_cord_upb_test", + srcs = ["ctype_cord_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, + deps = [ + "//rust:protobuf_upb_export", + "//rust/test:unittest_upb_rust_proto", + "@crate_index//:googletest", + ], +) + +rust_test( + name = "ctype_cord_cpp_test", + srcs = ["ctype_cord_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, + deps = [ + "//rust:protobuf_cpp_export", + "//rust/test:unittest_cpp_rust_proto", + "@crate_index//:googletest", + ], +) + rust_test( name = "child_parent_upb_test", srcs = ["child_parent_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, deps = [ + "//rust:protobuf_upb_export", "//rust/test:child_upb_rust_proto", "//rust/test:parent_upb_rust_proto", "@crate_index//:googletest", @@ -29,7 +59,11 @@ rust_test( rust_test( name = "child_parent_cpp_test", srcs = ["child_parent_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, deps = [ + "//rust:protobuf_cpp_export", "//rust/test:child_cpp_rust_proto", "//rust/test:parent_cpp_rust_proto", "@crate_index//:googletest", @@ -87,6 +121,7 @@ rust_test( srcs = ["import_public_test.rs"], deps = [ "//rust/test:import_public_cpp_rust_proto", + "@crate_index//:googletest", ], ) @@ -95,6 +130,7 @@ rust_test( srcs = ["import_public_test.rs"], deps = [ "//rust/test:import_public_upb_rust_proto", + "@crate_index//:googletest", ], ) @@ -105,6 +141,7 @@ rust_test( "//rust/test:dots_in_package_cpp_rust_proto", "//rust/test:no_package_cpp_rust_proto", "//rust/test:package_cpp_rust_proto", + "@crate_index//:googletest", ], ) @@ -115,6 +152,7 @@ rust_test( "//rust/test:dots_in_package_upb_rust_proto", "//rust/test:no_package_upb_rust_proto", "//rust/test:package_upb_rust_proto", + "@crate_index//:googletest", ], ) @@ -155,13 +193,19 @@ rust_test( rust_test( name = "nested_types_cpp_test", srcs = ["nested_types_test.rs"], - deps = ["//rust/test:unittest_cpp_rust_proto"], + deps = [ + "//rust/test:unittest_cpp_rust_proto", + "@crate_index//:googletest", + ], ) rust_test( name = "nested_types_upb_test", srcs = ["nested_types_test.rs"], - deps = ["//rust/test:unittest_upb_rust_proto"], + deps = [ + "//rust/test:unittest_upb_rust_proto", + "@crate_index//:googletest", + ], ) rust_test( @@ -227,10 +271,14 @@ rust_test( rust_test( name = "serialization_upb_test", srcs = ["serialization_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_upb_export", "//rust/test:unittest_edition_upb_rust_proto", "//rust/test:unittest_proto3_optional_upb_rust_proto", "//rust/test:unittest_proto3_upb_rust_proto", @@ -242,10 +290,14 @@ rust_test( rust_test( name = "serialization_cpp_test", srcs = ["serialization_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_cpp_export", "//rust/test:unittest_cpp_rust_proto", "//rust/test:unittest_edition_cpp_rust_proto", "//rust/test:unittest_proto3_cpp_rust_proto", @@ -315,10 +367,14 @@ rust_test( rust_test( name = "accessors_map_cpp_test", srcs = ["accessors_map_test.rs"], + aliases = { + "//rust:protobuf_cpp_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_cpp_export", "//rust/test:enums_cpp_rust_proto", "//rust/test:map_unittest_cpp_rust_proto", "//rust/test:unittest_cpp_rust_proto", @@ -329,10 +385,14 @@ rust_test( rust_test( name = "accessors_map_upb_test", srcs = ["accessors_map_test.rs"], + aliases = { + "//rust:protobuf_upb_export": "protobuf", + }, proc_macro_deps = [ "@crate_index//:paste", ], deps = [ + "//rust:protobuf_upb_export", "//rust/test:enums_upb_rust_proto", "//rust/test:map_unittest_upb_rust_proto", "//rust/test:unittest_upb_rust_proto", diff --git a/rust/test/shared/accessors_map_test.rs b/rust/test/shared/accessors_map_test.rs index 26a309de28..942efb6efd 100644 --- a/rust/test/shared/accessors_map_test.rs +++ b/rust/test/shared/accessors_map_test.rs @@ -9,6 +9,7 @@ use enums_rust_proto::{test_map_with_nested_enum, TestMapWithNestedEnum}; use googletest::prelude::*; use map_unittest_rust_proto::{MapEnum, TestMap, TestMapWithMessages}; use paste::paste; +use protobuf::ProtoString; use std::collections::HashMap; use unittest_rust_proto::TestAllTypes; @@ -19,7 +20,7 @@ macro_rules! generate_map_primitives_tests { $(,)? ) => { paste! { $( - #[test] + #[googletest::test] fn [< test_map_ $k_field _ $v_field >]() { let mut msg = TestMap::new(); assert_that!(msg.[< map_ $k_field _ $v_field >]().len(), eq(0)); @@ -95,7 +96,7 @@ generate_map_primitives_tests!( (i32, MapEnum, int32, enum, 1, MapEnum::Baz), ); -#[test] +#[googletest::test] fn collect_as_hashmap() { // Highlights conversion from protobuf map to hashmap. let mut msg = TestMap::new(); @@ -114,7 +115,7 @@ fn collect_as_hashmap() { ); } -#[test] +#[googletest::test] fn test_string_maps() { let mut msg = TestMap::new(); msg.map_string_string_mut().insert("hello", "world"); @@ -126,7 +127,7 @@ fn test_string_maps() { assert_that!(msg.map_string_string().len(), eq(0)); } -#[test] +#[googletest::test] fn test_nested_enum_maps() { // Verify that C++ thunks are generated and are with the right name for strings TestMapWithNestedEnum::new() @@ -134,7 +135,7 @@ fn test_nested_enum_maps() { .insert("foo", test_map_with_nested_enum::inner_nested::NestedEnum::Foo); } -#[test] +#[googletest::test] fn test_bytes_and_string_copied() { let mut msg = TestMap::new(); @@ -154,20 +155,110 @@ fn test_bytes_and_string_copied() { assert_that!(msg.map_int32_bytes_mut().get(1).unwrap(), eq(b"world")); } -#[test] +#[googletest::test] fn test_map_setter() { - let mut msg = TestMap::new(); - msg.map_string_string_mut().insert("hello", "world"); - msg.map_string_string_mut().insert("fizz", "buzz"); + // Set Map + { + let mut msg = TestMap::new(); + let mut map = protobuf::Map::::new(); + map.as_mut().copy_from([("hello", "world"), ("fizz", "buzz")]); + msg.set_map_string_string(map); + assert_that!( + msg.map_string_string(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + } - let mut msg2 = TestMap::new(); - msg2.set_map_string_string(msg.map_string_string()); - assert_that!( - msg2.map_string_string(), - unordered_elements_are![ - eq(("hello".into(), "world".into())), - eq(("fizz".into(), "buzz".into())) - ] + // Set MapView + { + let mut msg = TestMap::new(); + let mut map = protobuf::Map::::new(); + map.as_mut().copy_from([("hello", "world"), ("fizz", "buzz")]); + msg.set_map_string_string(map.as_view()); + assert_that!( + msg.map_string_string(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + } + + // Set MapMut + { + let mut msg = TestMap::new(); + let mut map = protobuf::Map::::new(); + map.as_mut().copy_from([("hello", "world"), ("fizz", "buzz")]); + msg.set_map_string_string(map.as_mut()); + assert_that!( + msg.map_string_string(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + + // The original map should remain unchanged. + assert_that!( + map.as_view(), + unordered_elements_are![ + eq(("hello".into(), "world".into())), + eq(("fizz".into(), "buzz".into())) + ] + ); + } +} + +#[test] +fn test_map_creation_with_message_values() { + // Maps are usually created and owned by a parent message, but let's verify that + // we can successfully create and destroy them independently. + macro_rules! test_for_each_key { + ($($key_t:ty, $key:expr;)*) => { + $( + let msg = TestAllTypes::new(); + let mut map = protobuf::Map::<$key_t, TestAllTypes>::new(); + map.as_mut().insert($key, msg); + assert_that!(map.as_view().len(), eq(1)); + )* + } + } + + test_for_each_key!( + i32, -5; + u32, 13u32; + i64, 7; + u64, 11u64; + bool, false; + ProtoString, "looooooooooooooooooooooooong string"; + ); +} + +#[test] +fn test_map_clearing_with_message_values() { + macro_rules! test_for_each_key { + ($($key_t:ty, $key:expr;)*) => { + $( + let msg = TestAllTypes::new(); + let mut map = protobuf::Map::<$key_t, TestAllTypes>::new(); + map.as_mut().insert($key, msg); + assert_that!(map.as_view().len(), eq(1)); + map.as_mut().clear(); + assert_that!(map.as_view().len(), eq(0)); + )* + } + } + + test_for_each_key!( + i32, -5; + u32, 13u32; + i64, 7; + u64, 11u64; + bool, false; + ProtoString, "looooooooooooooooooooooooong string"; ); } @@ -177,7 +268,7 @@ macro_rules! generate_map_with_msg_values_tests { $(,)? ) => { paste! { $( - #[test] + #[googletest::test] fn [< test_map_ $k_field _all_types >]() { // We need to cover the following upb/c++ thunks: // TODO - b/323883851: Add test once Map::new is public. diff --git a/rust/test/shared/accessors_proto3_test.rs b/rust/test/shared/accessors_proto3_test.rs index f4096ea43a..1df120d1a1 100644 --- a/rust/test/shared/accessors_proto3_test.rs +++ b/rust/test/shared/accessors_proto3_test.rs @@ -5,14 +5,17 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -/// Tests covering accessors for singular bool, int32, int64, and bytes fields -/// on proto3. +//! Tests covering accessors for singular bool, int32, int64, and bytes fields +//! on proto3. + use googletest::prelude::*; +use protobuf::prelude::*; + use protobuf::Optional; use unittest_proto3_optional_rust_proto::{test_proto3_optional, TestProto3Optional}; use unittest_proto3_rust_proto::{test_all_types, TestAllTypes}; -#[test] +#[googletest::test] fn test_fixed32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_fixed32(), eq(0)); @@ -27,7 +30,7 @@ fn test_fixed32_accessors() { assert_that!(msg.optional_fixed32(), eq(43)); } -#[test] +#[googletest::test] fn test_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_bool(), eq(false)); @@ -39,7 +42,7 @@ fn test_bool_accessors() { assert_that!(msg.optional_bool(), eq(false)); } -#[test] +#[googletest::test] fn test_bytes_accessors() { let mut msg = TestAllTypes::new(); // Note: even though it's named 'optional_bytes', the field is actually not @@ -59,7 +62,7 @@ fn test_bytes_accessors() { assert_that!(*msg.optional_bytes(), empty()); } -#[test] +#[googletest::test] fn test_optional_bytes_accessors() { let mut msg = TestProto3Optional::new(); assert_that!(*msg.optional_bytes(), empty()); @@ -81,7 +84,7 @@ fn test_optional_bytes_accessors() { assert_that!(msg.optional_bytes_opt(), eq(Optional::Set(&b"\xffbinary\x85non-utf8"[..]))); } -#[test] +#[googletest::test] fn test_string_accessors() { let mut msg = TestAllTypes::new(); // Note: even though it's named 'optional_string', the field is actually not @@ -101,7 +104,7 @@ fn test_string_accessors() { assert_that!(*msg.optional_string().as_bytes(), empty()); } -#[test] +#[googletest::test] fn test_optional_string_accessors() { let mut msg = TestProto3Optional::new(); assert_that!(*msg.optional_string().as_bytes(), empty()); @@ -123,7 +126,7 @@ fn test_optional_string_accessors() { assert_that!(msg.optional_string_opt(), eq(Optional::Set("".into()))); } -#[test] +#[googletest::test] fn test_nested_enum_accessors() { use test_all_types::NestedEnum; @@ -137,7 +140,7 @@ fn test_nested_enum_accessors() { assert_that!(msg.optional_nested_enum(), eq(NestedEnum::Zero)); } -#[test] +#[googletest::test] fn test_optional_nested_enum_accessors() { use test_proto3_optional::NestedEnum; @@ -154,7 +157,7 @@ fn test_optional_nested_enum_accessors() { assert_that!(msg.optional_nested_enum_opt(), eq(Optional::Set(NestedEnum::Unspecified))); } -#[test] +#[googletest::test] fn test_foreign_enum_accessors() { use unittest_proto3_rust_proto::ForeignEnum; @@ -168,7 +171,7 @@ fn test_foreign_enum_accessors() { assert_that!(msg.optional_foreign_enum(), eq(ForeignEnum::ForeignZero)); } -#[test] +#[googletest::test] fn test_oneof_accessors() { use test_all_types::OneofField::*; @@ -197,7 +200,7 @@ fn test_oneof_accessors() { assert_that!(msg.oneof_field(), matches_pattern!(not_set(_))); } -#[test] +#[googletest::test] fn test_oneof_accessors_view_long_lifetime() { use test_all_types::OneofField::*; @@ -213,7 +216,7 @@ fn test_oneof_accessors_view_long_lifetime() { assert_that!(oneof, matches_pattern!(OneofUint32(eq(7)))); } -#[test] +#[googletest::test] fn test_oneof_enum_accessors() { use unittest_proto3_rust_proto::{ test_oneof2::{Foo, FooCase, NestedEnum}, @@ -231,7 +234,7 @@ fn test_oneof_enum_accessors() { assert_that!(msg.foo_case(), matches_pattern!(FooCase::FooEnum)); } -#[test] +#[googletest::test] fn test_submsg_setter() { use test_all_types::*; @@ -244,10 +247,36 @@ fn test_submsg_setter() { assert_that!(parent.optional_nested_message().bb(), eq(7)); } -#[test] +#[googletest::test] fn test_ctype_stringpiece() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_string_piece(), eq("")); msg.set_optional_string_piece("hello"); assert_that!(msg.optional_string_piece(), eq("hello")); } + +#[googletest::test] +fn test_msg_clear() { + let mut m = TestAllTypes::new(); + m.set_optional_int32(42); + assert_that!(m.optional_int32(), eq(42)); + m.clear(); + assert_that!(m.optional_int32(), eq(0)); +} + +#[googletest::test] +fn test_submsg_clear() { + let mut m = TestAllTypes::new(); + let mut sub = m.optional_nested_message_mut(); + sub.set_bb(7); + + assert_that!(m.has_optional_nested_message(), eq(true)); + assert_that!(m.optional_nested_message().bb(), eq(7)); + + m.optional_nested_message_mut().clear(); + + // .clear() on the submsg doesn't affect its presence on the parent: + assert_that!(m.has_optional_nested_message(), eq(true)); + // ...but it does clear the submsg's value: + assert_that!(m.optional_nested_message().bb(), eq(0)); +} diff --git a/rust/test/shared/accessors_repeated_test.rs b/rust/test/shared/accessors_repeated_test.rs index 9f70074bf4..c1de003561 100644 --- a/rust/test/shared/accessors_repeated_test.rs +++ b/rust/test/shared/accessors_repeated_test.rs @@ -7,13 +7,13 @@ use googletest::prelude::*; use paste::paste; -use protobuf::ViewProxy; +use protobuf::AsView; use unittest_rust_proto::{test_all_types, test_all_types::NestedMessage, TestAllTypes}; macro_rules! generate_repeated_numeric_test { ($(($t: ty, $field: ident)),*) => { paste! { $( - #[test] + #[googletest::test] fn [< test_repeated_ $field _accessors >]() { let mut msg = TestAllTypes::new(); assert_that!(msg.[< repeated_ $field >](), empty()); @@ -52,7 +52,7 @@ macro_rules! generate_repeated_numeric_test { ); } - #[test] + #[googletest::test] fn [< test_repeated_ $field _set >]() { let mut msg = TestAllTypes::new(); let mut msg2 = TestAllTypes::new(); @@ -70,7 +70,7 @@ macro_rules! generate_repeated_numeric_test { ); } - #[test] + #[googletest::test] fn [< test_repeated_ $field _exact_size_iterator >]() { let mut msg = TestAllTypes::new(); let mut mutator = msg.[](); @@ -104,7 +104,7 @@ generate_repeated_numeric_test!( (f64, double) ); -#[test] +#[googletest::test] fn test_repeated_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_bool(), empty()); @@ -134,7 +134,7 @@ fn test_repeated_bool_accessors() { assert_that!(msg.repeated_bool(), each(eq(false))); } -#[test] +#[googletest::test] fn test_repeated_enum_accessors() { use test_all_types::NestedEnum; @@ -172,7 +172,7 @@ fn test_repeated_enum_accessors() { assert_that!(msg.repeated_nested_enum(), each(eq(NestedEnum::Foo))); } -#[test] +#[googletest::test] fn test_repeated_enum_set() { use test_all_types::NestedEnum; @@ -184,7 +184,7 @@ fn test_repeated_enum_set() { ); } -#[test] +#[googletest::test] fn test_repeated_bool_set() { let mut msg = TestAllTypes::new(); let mut msg2 = TestAllTypes::new(); @@ -199,7 +199,7 @@ fn test_repeated_bool_set() { assert_that!(&view.iter().collect::>(), eq(&mutator2.iter().collect::>())); } -#[test] +#[googletest::test] fn test_repeated_message() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_nested_message().len(), eq(0)); @@ -230,7 +230,7 @@ fn test_repeated_message() { assert_that!(msg2.repeated_nested_message().len(), eq(0)); } -#[test] +#[googletest::test] fn test_repeated_message_setter() { let mut msg = TestAllTypes::new(); let mut nested = NestedMessage::new(); @@ -239,7 +239,7 @@ fn test_repeated_message_setter() { assert_that!(msg.repeated_nested_message().get(0).unwrap().bb(), eq(1)); } -#[test] +#[googletest::test] fn test_repeated_strings() { let mut older_msg = TestAllTypes::new(); { @@ -274,7 +274,7 @@ fn test_repeated_strings() { assert_that!(older_msg.repeated_string(), empty()); } -#[test] +#[googletest::test] fn test_repeated_bytes() { let mut older_msg = TestAllTypes::new(); { diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index f108c889c1..af020f8ba4 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -8,6 +8,8 @@ //! Tests covering accessors for singular bool, int32, int64, and bytes fields. use googletest::prelude::*; +use protobuf::prelude::*; + use protobuf::{Optional, ProtoBytes, ProtoStr, ProtoString}; use std::borrow::Cow; use std::ffi::OsString; @@ -15,7 +17,7 @@ use std::rc::Rc; use std::sync::Arc; use unittest_rust_proto::{test_all_types, TestAllTypes}; -#[test] +#[googletest::test] fn test_default_accessors() { let msg: TestAllTypes = Default::default(); assert_that!( @@ -40,7 +42,7 @@ fn test_default_accessors() { assert_that!(msg.default_bytes(), eq("world".as_bytes())); } -#[test] +#[googletest::test] fn test_optional_fixed32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.has_optional_fixed32(), eq(false)); @@ -58,7 +60,7 @@ fn test_optional_fixed32_accessors() { assert_that!(msg.optional_fixed32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_fixed32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_fixed32(), eq(47)); @@ -76,7 +78,7 @@ fn test_default_fixed32_accessors() { assert_that!(msg.default_fixed32_opt(), eq(Optional::Unset(47))); } -#[test] +#[googletest::test] fn test_optional_fixed64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_fixed64_opt(), eq(Optional::Unset(0))); @@ -95,7 +97,7 @@ fn test_optional_fixed64_accessors() { assert_that!(msg.optional_fixed64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_fixed64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_fixed64(), eq(48)); @@ -114,7 +116,7 @@ fn test_default_fixed64_accessors() { assert_that!(msg.default_fixed64_opt(), eq(Optional::Unset(48))); } -#[test] +#[googletest::test] fn test_optional_int32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_int32_opt(), eq(Optional::Unset(0))); @@ -133,7 +135,7 @@ fn test_optional_int32_accessors() { assert_that!(msg.optional_int32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_int32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_int32(), eq(41)); @@ -156,7 +158,7 @@ fn test_default_int32_accessors() { assert_that!(msg.default_int32_opt(), eq(Optional::Unset(41))); } -#[test] +#[googletest::test] fn test_optional_int64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_int64_opt(), eq(Optional::Unset(0))); @@ -171,7 +173,7 @@ fn test_optional_int64_accessors() { assert_that!(msg.optional_int64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_int64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_int64(), eq(42)); @@ -186,7 +188,7 @@ fn test_default_int64_accessors() { assert_that!(msg.default_int64_opt(), eq(Optional::Unset(42))); } -#[test] +#[googletest::test] fn test_optional_sint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_sint32_opt(), eq(Optional::Unset(0))); @@ -201,7 +203,7 @@ fn test_optional_sint32_accessors() { assert_that!(msg.optional_sint32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_sint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_sint32(), eq(-45)); @@ -216,7 +218,7 @@ fn test_default_sint32_accessors() { assert_that!(msg.default_sint32_opt(), eq(Optional::Unset(-45))); } -#[test] +#[googletest::test] fn test_optional_sint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_sint64_opt(), eq(Optional::Unset(0))); @@ -231,7 +233,7 @@ fn test_optional_sint64_accessors() { assert_that!(msg.optional_sint64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_sint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_sint64(), eq(46)); @@ -246,7 +248,7 @@ fn test_default_sint64_accessors() { assert_that!(msg.default_sint64_opt(), eq(Optional::Unset(46))); } -#[test] +#[googletest::test] fn test_optional_uint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_uint32_opt(), eq(Optional::Unset(0))); @@ -261,7 +263,7 @@ fn test_optional_uint32_accessors() { assert_that!(msg.optional_uint32(), eq(0)); } -#[test] +#[googletest::test] fn test_default_uint32_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_uint32(), eq(43)); @@ -276,7 +278,7 @@ fn test_default_uint32_accessors() { assert_that!(msg.default_uint32_opt(), eq(Optional::Unset(43))); } -#[test] +#[googletest::test] fn test_optional_uint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_uint64_opt(), eq(Optional::Unset(0))); @@ -291,7 +293,7 @@ fn test_optional_uint64_accessors() { assert_that!(msg.optional_uint64(), eq(0)); } -#[test] +#[googletest::test] fn test_default_uint64_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_uint64(), eq(44)); @@ -306,7 +308,7 @@ fn test_default_uint64_accessors() { assert_that!(msg.default_uint64_opt(), eq(Optional::Unset(44))); } -#[test] +#[googletest::test] fn test_optional_float_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_float_opt(), eq(Optional::Unset(0.0))); @@ -321,7 +323,7 @@ fn test_optional_float_accessors() { assert_that!(msg.optional_float(), eq(0.0)); } -#[test] +#[googletest::test] fn test_default_float_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_float(), eq(51.5)); @@ -336,7 +338,7 @@ fn test_default_float_accessors() { assert_that!(msg.default_float_opt(), eq(Optional::Unset(51.5))); } -#[test] +#[googletest::test] fn test_optional_double_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_double_opt(), eq(Optional::Unset(0.0))); @@ -351,7 +353,7 @@ fn test_optional_double_accessors() { assert_that!(msg.optional_double(), eq(0.0)); } -#[test] +#[googletest::test] fn test_default_double_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_double(), eq(52e3)); @@ -366,7 +368,7 @@ fn test_default_double_accessors() { assert_that!(msg.default_double_opt(), eq(Optional::Unset(52e3))); } -#[test] +#[googletest::test] fn test_optional_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_bool_opt(), eq(Optional::Unset(false))); @@ -378,7 +380,7 @@ fn test_optional_bool_accessors() { assert_that!(msg.optional_bool_opt(), eq(Optional::Unset(false))); } -#[test] +#[googletest::test] fn test_default_bool_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_bool(), eq(true)); @@ -393,7 +395,7 @@ fn test_default_bool_accessors() { assert_that!(msg.default_bool_opt(), eq(Optional::Unset(true))); } -#[test] +#[googletest::test] fn test_optional_bytes_accessors() { let mut msg = TestAllTypes::new(); assert_that!(*msg.optional_bytes(), empty()); @@ -419,7 +421,7 @@ fn test_optional_bytes_accessors() { assert_that!(msg.optional_bytes_opt(), eq(Optional::Set(&b""[..]))); } -#[test] +#[googletest::test] fn test_into_proxied_for_bytes() { let mut msg = TestAllTypes::new(); @@ -461,7 +463,7 @@ fn test_into_proxied_for_bytes() { assert_that!(msg.optional_bytes(), eq(b"ninth")); } -#[test] +#[googletest::test] fn test_nonempty_default_bytes_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_bytes(), eq(b"world")); @@ -490,7 +492,7 @@ fn test_nonempty_default_bytes_accessors() { assert_that!(msg.default_bytes_opt(), eq(Optional::Unset(&b"world"[..]))); } -#[test] +#[googletest::test] fn test_optional_string_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_string(), eq("")); @@ -516,7 +518,7 @@ fn test_optional_string_accessors() { assert_that!(msg.optional_string_opt(), eq(Optional::Unset("".into()))); } -#[test] +#[googletest::test] fn test_into_proxied_for_string() { let mut msg = TestAllTypes::new(); @@ -565,7 +567,7 @@ fn test_into_proxied_for_string() { assert_that!(msg.optional_string(), eq("eleventh")); } -#[test] +#[googletest::test] fn test_nonempty_default_string_accessors() { let mut msg = TestAllTypes::new(); assert_that!(msg.default_string(), eq("hello")); @@ -591,7 +593,7 @@ fn test_nonempty_default_string_accessors() { assert_that!(msg.default_string_opt(), eq(Optional::Unset("hello".into()))); } -#[test] +#[googletest::test] fn test_singular_msg_field() { let mut msg = TestAllTypes::new(); let msg_view = msg.optional_nested_message(); @@ -606,18 +608,16 @@ fn test_singular_msg_field() { assert_that!(msg.has_optional_nested_message(), eq(true)); } -#[test] +#[googletest::test] fn test_message_opt() { let msg = TestAllTypes::new(); - let opt: Optional< - unittest_rust_proto::test_all_types::NestedMessageView<'_>, - unittest_rust_proto::test_all_types::NestedMessageView<'_>, - > = msg.optional_nested_message_opt(); + let opt: Optional> = + msg.optional_nested_message_opt(); assert_that!(opt.is_set(), eq(false)); assert_that!(opt.into_inner().bb(), eq(0)); } -#[test] +#[googletest::test] fn test_message_opt_set() { let mut msg = TestAllTypes::new(); let submsg = test_all_types::NestedMessage::new(); @@ -626,7 +626,7 @@ fn test_message_opt_set() { assert_that!(msg.optional_nested_message_opt().is_set(), eq(false)); } -#[test] +#[googletest::test] fn test_setting_submsg() { let mut msg = TestAllTypes::new(); let submsg = test_all_types::NestedMessage::new(); @@ -644,7 +644,7 @@ fn test_setting_submsg() { assert_that!(msg.optional_nested_message_opt().is_set(), eq(false)); } -#[test] +#[googletest::test] fn test_msg_mut_initializes() { let mut msg = TestAllTypes::new(); assert_that!(msg.has_optional_nested_message(), eq(false)); @@ -660,7 +660,7 @@ fn test_msg_mut_initializes() { assert_that!(msg.optional_nested_message_opt().is_set(), eq(false)); } -#[test] +#[googletest::test] fn test_optional_nested_enum_accessors() { use test_all_types::NestedEnum; @@ -680,7 +680,7 @@ fn test_optional_nested_enum_accessors() { assert_that!(msg.optional_nested_enum(), eq(NestedEnum::Foo)); } -#[test] +#[googletest::test] fn test_default_nested_enum_accessors() { use test_all_types::NestedEnum; @@ -697,7 +697,7 @@ fn test_default_nested_enum_accessors() { assert_that!(msg.default_nested_enum_opt(), eq(Optional::Unset(NestedEnum::Bar))); } -#[test] +#[googletest::test] fn test_optional_foreign_enum_accessors() { use unittest_rust_proto::ForeignEnum; @@ -714,7 +714,7 @@ fn test_optional_foreign_enum_accessors() { assert_that!(msg.optional_foreign_enum(), eq(ForeignEnum::ForeignFoo)); } -#[test] +#[googletest::test] fn test_default_foreign_enum_accessors() { use unittest_rust_proto::ForeignEnum; @@ -731,7 +731,7 @@ fn test_default_foreign_enum_accessors() { assert_that!(msg.default_foreign_enum_opt(), eq(Optional::Unset(ForeignEnum::ForeignBar))); } -#[test] +#[googletest::test] fn test_optional_import_enum_accessors() { use unittest_rust_proto::ImportEnum; @@ -748,7 +748,7 @@ fn test_optional_import_enum_accessors() { assert_that!(msg.optional_import_enum(), eq(ImportEnum::ImportFoo)); } -#[test] +#[googletest::test] fn test_default_import_enum_accessors() { use unittest_rust_proto::ImportEnum; @@ -765,7 +765,7 @@ fn test_default_import_enum_accessors() { assert_that!(msg.default_import_enum_opt(), eq(Optional::Unset(ImportEnum::ImportBar))); } -#[test] +#[googletest::test] fn test_oneof_accessors() { use unittest_rust_proto::test_oneof2::{Foo::*, FooCase, NestedEnum}; use unittest_rust_proto::TestOneof2; @@ -816,7 +816,7 @@ fn test_oneof_accessors() { // TODO: Add tests covering a message-type field in a oneof. } -#[test] +#[googletest::test] fn test_msg_oneof_default_accessors() { use unittest_rust_proto::test_oneof2::{Bar::*, BarCase, NestedEnum}; @@ -848,7 +848,7 @@ fn test_msg_oneof_default_accessors() { // TODO: Add tests covering a message-type field in a oneof. } -#[test] +#[googletest::test] fn test_group() { let mut m = TestAllTypes::new(); @@ -861,7 +861,7 @@ fn test_group() { assert_that!(m.optionalgroup().a(), eq(7)); } -#[test] +#[googletest::test] fn test_submsg_setter() { use test_all_types::*; @@ -874,7 +874,7 @@ fn test_submsg_setter() { assert_that!(parent.optional_nested_message().bb(), eq(7)); } -#[test] +#[googletest::test] fn test_clone() { let mut m = TestAllTypes::new(); m.set_optional_int32(42); @@ -886,7 +886,7 @@ fn test_clone() { assert_that!(clone.optional_int32(), eq(42)); } -#[test] +#[googletest::test] fn test_to_owned() { let mut m = TestAllTypes::new(); m.set_optional_int32(42); @@ -910,7 +910,7 @@ fn test_to_owned() { assert_that!(submsg_mut.bb(), eq(8)); } -#[test] +#[googletest::test] fn test_ctype_stringpiece() { let mut msg = TestAllTypes::new(); assert_that!(msg.optional_string_piece(), eq("")); @@ -919,3 +919,29 @@ fn test_ctype_stringpiece() { assert_that!(msg.optional_string_piece(), eq("hello")); assert_that!(msg.has_optional_string_piece(), eq(true)); } + +#[googletest::test] +fn test_msg_clear() { + let mut m = TestAllTypes::new(); + m.set_optional_int32(42); + assert_that!(m.has_optional_int32(), eq(true)); + m.clear(); + assert_that!(m.has_optional_int32(), eq(false)); +} + +#[googletest::test] +fn test_submsg_clear() { + let mut m = TestAllTypes::new(); + let mut sub = m.optional_nested_message_mut(); + sub.set_bb(7); + + assert_that!(m.has_optional_nested_message(), eq(true)); + assert_that!(m.optional_nested_message().bb(), eq(7)); + + m.optional_nested_message_mut().clear(); + + // .clear() on the submsg doesn't affect its presence on the parent: + assert_that!(m.has_optional_nested_message(), eq(true)); + // ...but it does clear the submsg's value: + assert_that!(m.optional_nested_message().bb(), eq(0)); +} diff --git a/rust/test/shared/bad_names_test.rs b/rust/test/shared/bad_names_test.rs index 19ba5076a7..a60bfb942b 100644 --- a/rust/test/shared/bad_names_test.rs +++ b/rust/test/shared/bad_names_test.rs @@ -8,20 +8,20 @@ use bad_names_rust_proto::*; use googletest::prelude::*; -#[test] +#[googletest::test] fn test_reserved_keyword_in_accessors() { let msg = Self__mangled_because_ident_isnt_a_legal_raw_identifier::new(); let res = msg.self__mangled_because_ident_isnt_a_legal_raw_identifier().r#for(); assert_that!(res, eq(0)); } -#[test] +#[googletest::test] fn test_reserved_keyword_in_messages() { let _ = r#enum::new(); let _ = Ref::new().r#const(); } -#[test] +#[googletest::test] fn test_collision_in_accessors() { let mut m = AccessorsCollide::new(); m.set_x_mut_5(false); diff --git a/rust/test/shared/child_parent_test.rs b/rust/test/shared/child_parent_test.rs index 1a269ec052..b1905a9cb4 100644 --- a/rust/test/shared/child_parent_test.rs +++ b/rust/test/shared/child_parent_test.rs @@ -6,8 +6,9 @@ // https://developers.google.com/open-source/licenses/bsd use googletest::prelude::*; +use protobuf::prelude::*; -#[test] +#[googletest::test] fn test_canonical_types() { let _child = child_rust_proto::Child::new(); let _parent = parent_rust_proto::Parent::new(); @@ -16,12 +17,12 @@ fn test_canonical_types() { let _parent_from_child: child_rust_proto::Parent = parent_rust_proto::Parent::new(); } -#[test] +#[googletest::test] fn test_parent_serialization() { assert_that!(*parent_rust_proto::Parent::new().serialize().unwrap(), empty()); } -#[test] +#[googletest::test] fn test_child_serialization() { assert_that!(*child_rust_proto::Child::new().serialize().unwrap(), empty()); } diff --git a/rust/test/shared/ctype_cord_test.rs b/rust/test/shared/ctype_cord_test.rs new file mode 100644 index 0000000000..2879af1906 --- /dev/null +++ b/rust/test/shared/ctype_cord_test.rs @@ -0,0 +1,38 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +use googletest::prelude::*; + +use unittest_rust_proto::{TestAllTypes, TestCord}; + +#[googletest::test] +fn test_string_cord() { + let mut msg = TestAllTypes::new(); + assert_that!(msg.has_optional_cord(), eq(false)); + assert_that!(msg.optional_cord(), eq("")); + msg.set_optional_cord("hello"); + assert_that!(msg.has_optional_cord(), eq(true)); + assert_that!(msg.optional_cord(), eq("hello")); + + let mut msg2 = TestAllTypes::new(); + msg2.set_optional_cord(msg.optional_cord()); + assert_that!(msg2.optional_cord(), eq("hello")); +} + +#[googletest::test] +fn test_bytes_cord() { + let mut msg = TestCord::new(); + assert_that!(msg.has_optional_bytes_cord(), eq(false)); + assert_that!(msg.optional_bytes_cord(), eq("".as_bytes())); + msg.set_optional_bytes_cord(b"hello"); + assert_that!(msg.has_optional_bytes_cord(), eq(true)); + assert_that!(msg.optional_bytes_cord(), eq("hello".as_bytes())); + + let mut msg2 = TestCord::new(); + msg2.set_optional_bytes_cord(msg.optional_bytes_cord()); + assert_that!(msg2.optional_bytes_cord(), eq("hello".as_bytes())); +} diff --git a/rust/test/shared/edition2023_test.rs b/rust/test/shared/edition2023_test.rs index 824e33b0be..ba31299dcc 100644 --- a/rust/test/shared/edition2023_test.rs +++ b/rust/test/shared/edition2023_test.rs @@ -11,14 +11,14 @@ use googletest::prelude::*; // is _not_ a test for Rust Edition 2023 (which doesn't exist) but instead // Protobuf Edition 2023 (which exists). -#[test] +#[googletest::test] fn check_edition2023_works() { let msg = edition2023_rust_proto::EditionsMessage::new(); assert_that!(msg.plain_field_opt().into_inner(), eq(0)); assert_that!(msg.implicit_presence_field(), eq(0)); } -#[test] +#[googletest::test] fn string_view_works() { let mut msg = edition2023_rust_proto::EditionsMessage::new(); assert_that!(msg.str_view(), eq("")); @@ -28,7 +28,7 @@ fn string_view_works() { assert_that!(msg.has_str_view(), eq(true)); } -#[test] +#[googletest::test] fn repeated_string_view_works() { let mut msg = edition2023_rust_proto::EditionsMessage::new(); assert_that!(msg.repeated_str_view().len(), eq(0)); diff --git a/rust/test/shared/enum_test.rs b/rust/test/shared/enum_test.rs index a19bcb97eb..b4d0900495 100644 --- a/rust/test/shared/enum_test.rs +++ b/rust/test/shared/enum_test.rs @@ -12,7 +12,7 @@ use googletest::prelude::*; use protobuf::Enum; use unittest_rust_proto::*; -#[test] +#[googletest::test] fn test_nested_enum_values() { assert_that!(i32::from(test_all_types::NestedEnum::Foo), eq(1)); assert_that!(i32::from(test_all_types::NestedEnum::Bar), eq(2)); @@ -20,19 +20,19 @@ fn test_nested_enum_values() { assert_that!(i32::from(test_all_types::NestedEnum::Neg), eq(-1)); } -#[test] +#[googletest::test] fn test_isolated_nested_enum() { // Ensure that the enum is generated even when it's the only nested type for the // message. assert_that!(i32::from(test_required_enum_no_mask::NestedEnum::Foo), eq(2)); } -#[test] +#[googletest::test] fn test_enum_value_name_same_as_enum() { assert_that!(i32::from(TestEnumValueNameSameAsEnum::TestEnumValueNameSameAsEnum), eq(1)); } -#[test] +#[googletest::test] fn test_enum_defaults() { assert_that!(TestSparseEnum::default(), eq(TestSparseEnum::SparseA)); assert_that!(TestEnumWithDupValue::default(), eq(TestEnumWithDupValue::Foo1)); @@ -44,7 +44,7 @@ fn test_enum_defaults() { assert_that!(test_all_types::NestedEnum::default(), eq(test_all_types::NestedEnum::Foo)); } -#[test] +#[googletest::test] #[deny(unreachable_patterns)] #[allow(clippy::let_unit_value)] fn test_closed_enum_is_nonexhaustive() { @@ -58,7 +58,7 @@ fn test_closed_enum_is_nonexhaustive() { }; } -#[test] +#[googletest::test] fn test_closed_enum_conversion() { assert_that!(i32::from(TestSparseEnum::SparseA), eq(123)); assert_that!(TestSparseEnum::try_from(123), ok(eq(&TestSparseEnum::SparseA))); @@ -70,7 +70,7 @@ fn test_closed_enum_conversion() { assert_that!(TestSparseEnum::try_from(1), err(anything())); } -#[test] +#[googletest::test] fn test_closed_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDupValue::Foo1), eq(1)); assert_that!(i32::from(TestEnumWithDupValue::Foo2), eq(1)); @@ -88,7 +88,7 @@ fn test_closed_aliased_enum_conversion() { assert_that!(TestEnumWithDupValue::Bar1, eq(TestEnumWithDupValue::Bar2)); } -#[test] +#[googletest::test] #[deny(unreachable_patterns)] #[allow(clippy::let_unit_value)] fn test_open_enum_is_nonexhaustive() { @@ -100,7 +100,7 @@ fn test_open_enum_is_nonexhaustive() { }; } -#[test] +#[googletest::test] fn test_open_enum_conversion() { assert_that!(i32::from(TestEnumWithNumericNames::Unknown), eq(0)); assert_that!(i32::from(TestEnumWithNumericNames::_2020), eq(1)); @@ -123,7 +123,7 @@ fn test_open_enum_conversion() { assert_that!(i32::from(TestEnumWithNumericNames::from(-1)), eq(-1)); } -#[test] +#[googletest::test] fn test_open_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDuplicateStrippedPrefixNames::Unknown), eq(0)); assert_that!(i32::from(TestEnumWithDuplicateStrippedPrefixNames::Foo), eq(1)); @@ -157,19 +157,19 @@ fn test_open_aliased_enum_conversion() { assert_that!(i32::from(TestEnumWithDuplicateStrippedPrefixNames::from(5)), eq(5)); } -#[test] +#[googletest::test] fn test_enum_conversion_failure_display() { let err = TestSparseEnum::try_from(1).unwrap_err(); assert_that!(format!("{err}"), eq("1 is not a known value for TestSparseEnum")); } -#[test] +#[googletest::test] fn test_enum_conversion_failure_impls_std_error() { let err = TestSparseEnum::try_from(1).unwrap_err(); let _test_compiles: &dyn std::error::Error = &err; } -#[test] +#[googletest::test] fn test_is_known_for_closed_enum() { assert_that!(test_all_types::NestedEnum::is_known(-2), eq(false)); assert_that!(test_all_types::NestedEnum::is_known(-1), eq(true)); @@ -180,7 +180,7 @@ fn test_is_known_for_closed_enum() { assert_that!(test_all_types::NestedEnum::is_known(4), eq(false)); } -#[test] +#[googletest::test] fn test_is_known_for_open_enum() { assert_that!(TestEnumWithNumericNames::is_known(-1), eq(false)); assert_that!(TestEnumWithNumericNames::is_known(0), eq(true)); diff --git a/rust/test/shared/fields_with_imported_types_test.rs b/rust/test/shared/fields_with_imported_types_test.rs index 6b1b17c45b..9fa6563da1 100644 --- a/rust/test/shared/fields_with_imported_types_test.rs +++ b/rust/test/shared/fields_with_imported_types_test.rs @@ -10,7 +10,7 @@ /// a separate proto_library target. use googletest::prelude::*; -#[test] +#[googletest::test] fn test_message_field_generated() { use fields_with_imported_types_rust_proto::MsgWithFieldsWithImportedTypes; use imported_types_rust_proto::ImportedMessageView; @@ -19,7 +19,7 @@ fn test_message_field_generated() { assert_that!(msg.imported_message_field(), matches_pattern!(ImportedMessageView { .. })); } -#[test] +#[googletest::test] fn test_enum_field_generated() { use fields_with_imported_types_rust_proto::MsgWithFieldsWithImportedTypes; use imported_types_rust_proto::ImportedEnum; @@ -28,7 +28,7 @@ fn test_enum_field_generated() { assert_that!(msg.imported_enum_field(), eq(ImportedEnum::Unknown)); } -#[test] +#[googletest::test] fn test_oneof_message_field_generated() { use fields_with_imported_types_rust_proto::msg_with_fields_with_imported_types::ImportedTypesOneof::not_set; use fields_with_imported_types_rust_proto::MsgWithFieldsWithImportedTypes; diff --git a/rust/test/shared/import_public_test.rs b/rust/test/shared/import_public_test.rs index 4d05ad8f02..6299219151 100644 --- a/rust/test/shared/import_public_test.rs +++ b/rust/test/shared/import_public_test.rs @@ -7,7 +7,7 @@ //! Tests covering codegen of import public statements. -#[test] +#[googletest::test] fn test_import_public_types_are_reexported() { let _: import_public_rust_proto::PrimarySrcPubliclyImportedMsg; let _: import_public_rust_proto::PrimarySrcPubliclyImportedMsgView; diff --git a/rust/test/shared/merge_from_test.rs b/rust/test/shared/merge_from_test.rs index b07f528fd9..26eb3998ed 100644 --- a/rust/test/shared/merge_from_test.rs +++ b/rust/test/shared/merge_from_test.rs @@ -2,7 +2,7 @@ use googletest::prelude::*; use protobuf::proto; use unittest_rust_proto::{NestedTestAllTypes, TestAllTypes}; -#[test] +#[googletest::test] fn merge_from_empty() { let mut dst = TestAllTypes::new(); let src = TestAllTypes::new(); @@ -10,7 +10,7 @@ fn merge_from_empty() { assert_that!(dst.as_view().has_optional_int32(), eq(false)); } -#[test] +#[googletest::test] fn merge_from_non_empty() { let mut dst = TestAllTypes::new(); let src = proto!(TestAllTypes { optional_int32: 42 }); @@ -18,7 +18,7 @@ fn merge_from_non_empty() { assert_eq!(dst.as_view().optional_int32(), 42); } -#[test] +#[googletest::test] fn merge_repeated_empty() { let mut dst = TestAllTypes::new(); let mut src = TestAllTypes::new(); @@ -30,7 +30,7 @@ fn merge_repeated_empty() { ); } -#[test] +#[googletest::test] fn merge_repeated_non_empty() { let mut dst = TestAllTypes::new(); let mut src = TestAllTypes::new(); @@ -43,7 +43,7 @@ fn merge_repeated_non_empty() { ); } -#[test] +#[googletest::test] fn merge_from_sub_message() { let mut dst = NestedTestAllTypes::new(); let src = proto!(NestedTestAllTypes { diff --git a/rust/test/shared/nested_types_test.rs b/rust/test/shared/nested_types_test.rs index d47929efb4..8f04d9f38d 100644 --- a/rust/test/shared/nested_types_test.rs +++ b/rust/test/shared/nested_types_test.rs @@ -7,7 +7,7 @@ //! Tests covering nested types. -#[test] +#[googletest::test] fn test_nested_messages_accessible() { let _parent: unittest_rust_proto::TestAllTypes; let _child: unittest_rust_proto::test_all_types::NestedMessage; @@ -15,7 +15,7 @@ fn test_nested_messages_accessible() { nested_test_all_extensions_data::NestedDynamicExtensions::new(); } -#[test] +#[googletest::test] fn test_nested_enums_accessible() { let _parent: unittest_rust_proto::TestAllTypes; let _child: unittest_rust_proto::test_all_types::NestedEnum; diff --git a/rust/test/shared/package_test.rs b/rust/test/shared/package_test.rs index 04f5b84d5f..67a854d6d3 100644 --- a/rust/test/shared/package_test.rs +++ b/rust/test/shared/package_test.rs @@ -7,7 +7,7 @@ //! Tests covering proto packages. -#[test] +#[googletest::test] fn test_message_packages() { // empty package, message declared in the first .proto source let _: no_package_rust_proto::MsgWithoutPackage; @@ -26,7 +26,7 @@ fn test_message_packages() { let _: package_rust_proto::ImportedMsgWithPackage; } -#[test] +#[googletest::test] fn test_enum_packages() { // empty package, enum declared in the first .proto source let _: no_package_rust_proto::EnumWithoutPackage; diff --git a/rust/test/shared/proto_macro_test.rs b/rust/test/shared/proto_macro_test.rs index 07c6f6bb80..5085cb73f5 100644 --- a/rust/test/shared/proto_macro_test.rs +++ b/rust/test/shared/proto_macro_test.rs @@ -18,7 +18,7 @@ struct TestValue { val: i64, } -#[test] +#[googletest::test] fn test_setting_literals() { let fixed64 = || 108; let test_ref = |x: &i64| *x; @@ -67,7 +67,7 @@ fn test_setting_literals() { assert_that!(msg.optional_nested_enum(), eq(test_all_types::NestedEnum::Baz)); } -#[test] +#[googletest::test] fn single_nested_message() { let msg = proto!(TestAllTypes { optional_nested_message: NestedMessage { bb: 42 } }); assert_that!(msg.optional_nested_message().bb(), eq(42)); @@ -94,6 +94,14 @@ fn single_nested_message() { }); assert_that!(msg.optional_nested_message().bb(), eq(42)); + // field above and below it + let msg = proto!(TestAllTypes { + optional_int32: 1, + optional_nested_message: __ { bb: 42 }, + optional_int64: 2 + }); + assert_that!(msg.optional_nested_message().bb(), eq(42)); + // test empty initializer let msg = proto!(TestAllTypes {}); assert_that!(msg.has_optional_nested_message(), eq(false)); @@ -104,9 +112,17 @@ fn single_nested_message() { optional_nested_message: unittest_rust_proto::test_all_types::NestedMessage {} }); assert_that!(msg.has_optional_nested_message(), eq(true)); + + let msg = proto!(::unittest_rust_proto::TestAllTypes { + optional_nested_message: ::unittest_rust_proto::test_all_types::NestedMessage {} + }); + assert_that!(msg.has_optional_nested_message(), eq(true)); + + let msg = proto!(::unittest_rust_proto::TestAllTypes { optional_nested_message: __ {} }); + assert_that!(msg.has_optional_nested_message(), eq(true)); } -#[test] +#[googletest::test] fn test_recursive_msg() { let msg = proto!(NestedTestAllTypes { child: NestedTestAllTypes { @@ -123,7 +139,7 @@ fn test_recursive_msg() { assert_that!(msg.child().child().child().payload().optional_int32(), eq(43)); } -#[test] +#[googletest::test] fn test_spread_msg() { let msg = proto!(TestAllTypes { optional_nested_message: NestedMessage { bb: 42 } }); let msg2 = proto!(TestAllTypes { ..msg.as_view() }); @@ -133,7 +149,7 @@ fn test_spread_msg() { assert_that!(msg3.optional_int32(), eq(1)); } -#[test] +#[googletest::test] fn test_spread_nested_msg() { let msg = proto!(NestedTestAllTypes { child: NestedTestAllTypes { @@ -151,3 +167,38 @@ fn test_spread_nested_msg() { assert_that!(msg2.child().child().payload().optional_int32(), eq(42)); assert_that!(msg2.child().child().child().payload().optional_int32(), eq(43)); } + +#[googletest::test] +fn test_repeated_i32() { + let msg = proto!(TestAllTypes { repeated_int32: [1, 1 + 1, 3] }); + assert_that!(msg.repeated_int32().len(), eq(3)); + assert_that!(msg.repeated_int32().get(0).unwrap(), eq(1)); + assert_that!(msg.repeated_int32().get(1).unwrap(), eq(2)); + assert_that!(msg.repeated_int32().get(2).unwrap(), eq(3)); +} + +#[googletest::test] +fn test_repeated_msg() { + let msg2 = proto!(NestedTestAllTypes { payload: TestAllTypes { optional_int32: 1 } }); + let msg = proto!(NestedTestAllTypes { + child: NestedTestAllTypes { + repeated_child: [ + NestedTestAllTypes { payload: TestAllTypes { optional_int32: 0 } }, + msg2, + __ { payload: TestAllTypes { optional_int32: 2 } } + ] + }, + repeated_child: [ + __ { payload: __ { optional_int32: 1 } }, + NestedTestAllTypes { payload: TestAllTypes { optional_int32: 2 } } + ] + }); + assert_that!(msg.child().repeated_child().len(), eq(3)); + assert_that!(msg.child().repeated_child().get(0).unwrap().payload().optional_int32(), eq(0)); + assert_that!(msg.child().repeated_child().get(1).unwrap().payload().optional_int32(), eq(1)); + assert_that!(msg.child().repeated_child().get(2).unwrap().payload().optional_int32(), eq(2)); + + assert_that!(msg.repeated_child().len(), eq(2)); + assert_that!(msg.repeated_child().get(0).unwrap().payload().optional_int32(), eq(1)); + assert_that!(msg.repeated_child().get(1).unwrap().payload().optional_int32(), eq(2)); +} diff --git a/rust/test/shared/serialization_test.rs b/rust/test/shared/serialization_test.rs index e5b614034f..b0e882271a 100644 --- a/rust/test/shared/serialization_test.rs +++ b/rust/test/shared/serialization_test.rs @@ -6,6 +6,8 @@ // https://developers.google.com/open-source/licenses/bsd use googletest::prelude::*; +use protobuf::prelude::*; + use paste::paste; use unittest_edition_rust_proto::TestAllTypes as TestAllTypesEditions; use unittest_proto3_optional_rust_proto::TestProto3Optional; @@ -15,7 +17,7 @@ use unittest_rust_proto::TestAllTypes as TestAllTypesProto2; macro_rules! generate_parameterized_serialization_test { ($(($type: ident, $name_ext: ident)),*) => { paste! { $( - #[test] + #[googletest::test] fn [< serialization_zero_length_ $name_ext >]() { let mut msg = [< $type >]::new(); @@ -29,7 +31,7 @@ macro_rules! generate_parameterized_serialization_test { assert_that!(serialized.len(), eq(0)); } - #[test] + #[googletest::test] fn [< serialize_deserialize_message_ $name_ext>]() { let mut msg = [< $type >]::new(); msg.set_optional_int64(42); @@ -44,17 +46,17 @@ macro_rules! generate_parameterized_serialization_test { assert_that!(msg.optional_bytes(), eq(msg2.optional_bytes())); } - #[test] + #[googletest::test] fn [< deserialize_empty_ $name_ext>]() { assert!([< $type >]::parse(&[]).is_ok()); } - #[test] + #[googletest::test] fn [< deserialize_error_ $name_ext>]() { assert!([< $type >]::parse(b"not a serialized proto").is_err()); } - #[test] + #[googletest::test] fn [< set_bytes_with_serialized_data_ $name_ext>]() { let mut msg = [< $type >]::new(); msg.set_optional_int64(42); @@ -64,7 +66,7 @@ macro_rules! generate_parameterized_serialization_test { assert_that!(msg2.optional_bytes(), eq(msg.serialize().unwrap())); } - #[test] + #[googletest::test] fn [< deserialize_on_previously_allocated_message_ $name_ext>]() { let mut msg = [< $type >]::new(); msg.set_optional_int64(42); @@ -95,7 +97,7 @@ macro_rules! generate_parameterized_int32_byte_size_test { ($(($type: ident, $name_ext: ident)),*) => { paste! { $( - #[test] + #[googletest::test] fn [< test_int32_byte_size_ $name_ext>]() { let args = vec![(0, 1), (127, 1), (128, 2), (-1, 10)]; for arg in args { diff --git a/rust/test/shared/simple_nested_test.rs b/rust/test/shared/simple_nested_test.rs index fe74786ec8..2fefc466de 100644 --- a/rust/test/shared/simple_nested_test.rs +++ b/rust/test/shared/simple_nested_test.rs @@ -10,7 +10,7 @@ use nested_rust_proto::outer::inner::InnerEnum; use nested_rust_proto::outer::InnerView; use nested_rust_proto::*; -#[test] +#[googletest::test] fn test_deeply_nested_message() { let deep = outer::inner::super_inner::duper_inner::even_more_inner::CantBelieveItsSoInner::new(); @@ -20,7 +20,7 @@ fn test_deeply_nested_message() { assert_that!(outermsg.deep().num(), eq(0)); } -#[test] +#[googletest::test] fn test_deeply_nested_enum() { use outer::inner::super_inner::duper_inner::even_more_inner::JustWayTooInner; let deep = JustWayTooInner::default(); @@ -30,7 +30,7 @@ fn test_deeply_nested_enum() { assert_that!(outermsg.deep_enum(), eq(JustWayTooInner::Unspecified)); } -#[test] +#[googletest::test] fn test_nested_views() { let outermsg = Outer::new(); let inner_msg: InnerView<'_> = outermsg.inner(); @@ -53,7 +53,7 @@ fn test_nested_views() { assert_that!(inner_msg.inner_enum(), eq(InnerEnum::Unspecified)); } -#[test] +#[googletest::test] fn test_nested_view_lifetimes() { // Ensure that views have the lifetime of the first layer of borrow, and don't // create intermediate borrows through nested accessors. @@ -79,7 +79,7 @@ fn test_nested_view_lifetimes() { assert_that!(string_map.len(), eq(0)); } -#[test] +#[googletest::test] fn test_msg_from_outside() { // let's make sure that we're not just working for messages nested inside // messages, messages from without and within should work @@ -87,7 +87,7 @@ fn test_msg_from_outside() { assert_that!(outer.notinside().num(), eq(0)); } -#[test] +#[googletest::test] fn test_recursive_view() { let rec = nested_rust_proto::Recursive::new(); assert_that!(rec.num(), eq(0)); @@ -100,7 +100,7 @@ fn test_recursive_view() { assert_that!(nested.num(), eq(0)); } -#[test] +#[googletest::test] fn test_recursive_mut() { let mut rec = nested_rust_proto::Recursive::new(); let mut one = rec.rec_mut(); diff --git a/rust/test/shared/utf8/BUILD b/rust/test/shared/utf8/BUILD index 8d09729017..51f3b4da01 100644 --- a/rust/test/shared/utf8/BUILD +++ b/rust/test/shared/utf8/BUILD @@ -8,13 +8,13 @@ rust_test( name = "utf8_cpp_test", srcs = ["utf8_test.rs"], aliases = { - "//rust:protobuf_cpp": "protobuf", + "//rust:protobuf_cpp_export": "protobuf", }, deps = [ ":feature_verify_cpp_rust_proto", ":no_features_proto2_cpp_rust_proto", ":no_features_proto3_cpp_rust_proto", - "//rust:protobuf_cpp", + "//rust:protobuf_cpp_export", "@crate_index//:googletest", ], ) @@ -23,13 +23,13 @@ rust_test( name = "utf8_upb_test", srcs = ["utf8_test.rs"], aliases = { - "//rust:protobuf_upb": "protobuf", + "//rust:protobuf_upb_export": "protobuf", }, deps = [ ":feature_verify_upb_rust_proto", ":no_features_proto2_upb_rust_proto", ":no_features_proto3_upb_rust_proto", - "//rust:protobuf_upb", + "//rust:protobuf_upb_export", "@crate_index//:googletest", ], ) diff --git a/rust/test/shared/utf8/utf8_test.rs b/rust/test/shared/utf8/utf8_test.rs index 8b46cf86dd..a213136cb1 100644 --- a/rust/test/shared/utf8/utf8_test.rs +++ b/rust/test/shared/utf8/utf8_test.rs @@ -13,8 +13,10 @@ // behavior. Do not assume that the Protobuf team is intentional about these // behaviors while b/304774814 is open. -use feature_verify_rust_proto::Verify; use googletest::prelude::*; +use protobuf::prelude::*; + +use feature_verify_rust_proto::Verify; use no_features_proto2_rust_proto::NoFeaturesProto2; use no_features_proto3_rust_proto::NoFeaturesProto3; use protobuf::{ParseError, ProtoStr}; @@ -34,7 +36,7 @@ fn make_non_utf8_proto_str() -> &'static ProtoStr { } } -#[test] +#[googletest::test] fn test_proto2() { let non_utf8_str = make_non_utf8_proto_str(); @@ -52,7 +54,7 @@ fn test_proto2() { assert_that!(parsed_result, ok(anything())); } -#[test] +#[googletest::test] fn test_proto3() { let non_utf8_str = make_non_utf8_proto_str(); @@ -70,7 +72,7 @@ fn test_proto3() { assert_that!(parsed_result, err(matches_pattern!(&ParseError))); } -#[test] +#[googletest::test] fn test_verify() { let non_utf8_str = make_non_utf8_proto_str(); diff --git a/rust/test/upb/debug_string_test.rs b/rust/test/upb/debug_string_test.rs index fa2f429c1c..9cc3457f1e 100644 --- a/rust/test/upb/debug_string_test.rs +++ b/rust/test/upb/debug_string_test.rs @@ -13,7 +13,7 @@ use unittest_rust_proto::{ test_all_types::NestedMessage as NestedMessageProto2, TestAllTypes as TestAllTypesProto2, }; -#[test] +#[googletest::test] fn test_debug_string() { let mut msg = proto!(TestAllTypesProto2 { optional_int32: 42, diff --git a/rust/test/upb/string_ctypes_test.rs b/rust/test/upb/string_ctypes_test.rs index 6bbea67a21..7999a58e45 100644 --- a/rust/test/upb/string_ctypes_test.rs +++ b/rust/test/upb/string_ctypes_test.rs @@ -8,7 +8,7 @@ use googletest::prelude::*; use unittest_proto3_rust_proto::TestAllTypes; -#[test] +#[googletest::test] fn test_stringpiece_repeated() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_string_piece().len(), eq(0)); @@ -17,15 +17,7 @@ fn test_stringpiece_repeated() { assert_that!(msg.repeated_string_piece().get(0), some(eq("hello"))); } -#[test] -fn test_cord() { - let mut msg = TestAllTypes::new(); - assert_that!(msg.optional_cord(), eq("")); - msg.set_optional_cord("hello"); - assert_that!(msg.optional_cord(), eq("hello")); -} - -#[test] +#[googletest::test] fn test_cord_repeated() { let mut msg = TestAllTypes::new(); assert_that!(msg.repeated_cord().len(), eq(0)); diff --git a/rust/upb.rs b/rust/upb.rs index a9c5581214..cc4db52bac 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -7,10 +7,10 @@ //! UPB FFI wrapper code for use by Rust Protobuf. -use crate::__internal::{Enum, Private}; +use crate::__internal::{Enum, Private, SealedInternal}; use crate::{ IntoProxied, Map, MapIter, MapMut, MapView, Mut, ProtoBytes, ProtoStr, ProtoString, Proxied, - ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, ViewProxy, + ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, }; use core::fmt::Debug; use std::alloc::Layout; @@ -19,7 +19,10 @@ use std::ptr::{self, NonNull}; use std::slice; use std::sync::OnceLock; +#[cfg(bzl)] extern crate upb; +#[cfg(not(bzl))] +use crate::upb; // Temporarily 'pub' since a lot of gencode is directly calling any of the ffi // fns. @@ -58,8 +61,11 @@ impl ScratchSpace { } } +#[doc(hidden)] pub type SerializedData = upb::OwnedArenaBox<[u8]>; +impl SealedInternal for SerializedData {} + impl IntoProxied for SerializedData { fn into_proxied(self, _private: Private) -> ProtoBytes { ProtoBytes { inner: InnerProtoString(self) } @@ -463,7 +469,7 @@ pub fn free_enum_repeated( } /// Returns a static empty RepeatedView. -pub fn empty_array() -> RepeatedView<'static, T> { +pub fn empty_array() -> RepeatedView<'static, T> { // TODO: Consider creating a static empty array in C. // Use `i32` for a shared empty repeated for all repeated types in the program. @@ -483,8 +489,8 @@ pub fn empty_array() -> RepeatedView<'static, T> /// Returns a static empty MapView. pub fn empty_map() -> MapView<'static, K, V> where - K: Proxied + ?Sized, - V: ProxiedInMapValue + ?Sized, + K: Proxied, + V: ProxiedInMapValue, { // TODO: Consider creating a static empty map in C. @@ -678,8 +684,7 @@ pub struct RawMapIter { impl RawMapIter { pub fn new(_private: Private, map: RawMap) -> Self { - // SAFETY: __rust_proto_kUpb_Map_Begin is never modified - RawMapIter { map, iter: unsafe { __rust_proto_kUpb_Map_Begin } } + RawMapIter { map, iter: UPB_MAP_BEGIN } } /// # Safety diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 8fa34f6237..8fd5505a04 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -4,6 +4,10 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +# begin:github_only +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") +# end:github_only + load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") # begin:google_only @@ -15,6 +19,7 @@ rust_library( srcs = [ "arena.rs", "array.rs", + "associated_mini_table.rs", "ctype.rs", "extension_registry.rs", "lib.rs", @@ -28,6 +33,7 @@ rust_library( "text.rs", "wire.rs", ], + rustc_flags = ["--cfg=bzl"], visibility = [ "//rust:__subpackages__", "//src/google/protobuf:__subpackages__", @@ -52,3 +58,16 @@ cc_library( "//upb/text:debug", ], ) + +# begin:github_only +pkg_files( + name = "rust_protobuf_upb_src", + srcs = glob( + [ + "*", + ], + ), + strip_prefix = strip_prefix.from_root("rust"), + visibility = ["//rust:__pkg__"], +) +# end:github_only diff --git a/rust/upb/arena.rs b/rust/upb/arena.rs index b1a90dc4e9..d4581eabd3 100644 --- a/rust/upb/arena.rs +++ b/rust/upb/arena.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; +use super::opaque_pointee::opaque_pointee; use std::alloc::{self, Layout}; use std::cell::UnsafeCell; use std::marker::PhantomData; diff --git a/rust/upb/array.rs b/rust/upb/array.rs index aeadd1bdfc..1c42cefbb0 100644 --- a/rust/upb/array.rs +++ b/rust/upb/array.rs @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; -use crate::{upb_MessageValue, upb_MutableMessageValue, CType, RawArena}; +use super::opaque_pointee::opaque_pointee; +use super::{upb_MessageValue, upb_MutableMessageValue, CType, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Array); @@ -27,13 +27,14 @@ extern "C" { #[cfg(test)] mod tests { + use super::super::Arena; use super::*; #[test] fn array_ffi_test() { // SAFETY: FFI unit test uses C API under expected patterns. unsafe { - let arena = crate::Arena::new(); + let arena = Arena::new(); let raw_arena = arena.raw(); let array = upb_Array_New(raw_arena, CType::Float); diff --git a/rust/upb/associated_mini_table.rs b/rust/upb/associated_mini_table.rs new file mode 100644 index 0000000000..382d9a63ac --- /dev/null +++ b/rust/upb/associated_mini_table.rs @@ -0,0 +1,30 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +use super::upb_MiniTable; + +/// A trait for types which have a constant associated MiniTable (e.g. +/// generated messages, and their mut and view proxy types). +/// +/// upb_Message in C is effectively a DST type, where instances are created with +/// a MiniTable (and have a size dependent on the given MiniTable). Many upb +/// operations on the upb_Message* will require the same upb_MiniTable* to be +/// passed in as a parameter, which is referred to as 'the associated MiniTable +/// for the upb_Message instance' in safety comments. +/// +/// This trait is a way to statically associate a MiniTable with Rust types +/// which hold upb_Message* to simplify ensuring the upb C invariants +/// are maintained. +/// +/// SAFETY: +/// - The MiniTable pointer must be from Protobuf code generation and follow the +/// corresponding invariants associated with upb's C API (the pointer should +/// always have the same non-null value, the underlying pointee should never +/// be modified and should have 'static lifetime). +pub unsafe trait AssociatedMiniTable { + const MINI_TABLE: *const upb_MiniTable; +} diff --git a/rust/upb/extension_registry.rs b/rust/upb/extension_registry.rs index c0400cd61e..ce7fd26b6e 100644 --- a/rust/upb/extension_registry.rs +++ b/rust/upb/extension_registry.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; +use super::opaque_pointee::opaque_pointee; use std::ptr::NonNull; opaque_pointee!(upb_ExtensionRegistry); diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index bbfc106b6b..125a0f277a 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -4,8 +4,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +#![deny(unsafe_op_in_unsafe_fn)] +#![cfg_attr(not(bzl), allow(unused_imports))] mod arena; + pub use arena::{upb_Arena, Arena, RawArena}; mod array; @@ -15,6 +18,9 @@ pub use array::{ upb_Array_Size, RawArray, }; +mod associated_mini_table; +pub use associated_mini_table::AssociatedMiniTable; + mod ctype; pub use ctype::CType; @@ -24,13 +30,14 @@ pub use extension_registry::{upb_ExtensionRegistry, RawExtensionRegistry}; mod map; pub use map::{ upb_Map, upb_Map_Clear, upb_Map_Delete, upb_Map_Get, upb_Map_Insert, upb_Map_New, upb_Map_Next, - upb_Map_Size, MapInsertStatus, RawMap, __rust_proto_kUpb_Map_Begin, + upb_Map_Size, MapInsertStatus, RawMap, UPB_MAP_BEGIN, }; mod message; pub use message::{ - upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_IsEqual, - upb_Message_MergeFrom, upb_Message_New, upb_Message_SetBaseField, RawMessage, + upb_Message, upb_Message_Clear, upb_Message_DeepClone, upb_Message_DeepCopy, + upb_Message_IsEqual, upb_Message_MergeFrom, upb_Message_New, upb_Message_SetBaseField, + RawMessage, }; mod message_value; diff --git a/rust/upb/map.rs b/rust/upb/map.rs index 5c97a21d64..5b3a8d7bbb 100644 --- a/rust/upb/map.rs +++ b/rust/upb/map.rs @@ -5,8 +5,8 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; -use crate::{upb_MessageValue, CType, RawArena}; +use super::opaque_pointee::opaque_pointee; +use super::{upb_MessageValue, CType, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Map); @@ -21,9 +21,9 @@ pub enum MapInsertStatus { OutOfMemory = 2, } -extern "C" { - pub static __rust_proto_kUpb_Map_Begin: usize; +pub const UPB_MAP_BEGIN: usize = usize::MAX; +extern "C" { pub fn upb_Map_New(arena: RawArena, key_type: CType, value_type: CType) -> RawMap; pub fn upb_Map_Size(map: RawMap) -> usize; pub fn upb_Map_Insert( @@ -49,13 +49,14 @@ extern "C" { #[cfg(test)] mod tests { + use super::super::Arena; use super::*; #[test] fn map_ffi_test() { // SAFETY: FFI unit test uses C API under expected patterns. unsafe { - let arena = crate::Arena::new(); + let arena = Arena::new(); let raw_arena = arena.raw(); let map = upb_Map_New(raw_arena, CType::Bool, CType::Double); assert_eq!(upb_Map_Size(map), 0); diff --git a/rust/upb/message.rs b/rust/upb/message.rs index 865c52ee9d..b78e495808 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -5,18 +5,28 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; -use crate::{upb_ExtensionRegistry, upb_MiniTable, upb_MiniTableField, RawArena}; +use super::opaque_pointee::opaque_pointee; +use super::{upb_ExtensionRegistry, upb_MiniTable, upb_MiniTableField, RawArena}; use std::ptr::NonNull; opaque_pointee!(upb_Message); pub type RawMessage = NonNull; extern "C" { - /// SAFETY: No constraints. + /// SAFETY: + /// - `mini_table` and `arena` must be valid to deref pub fn upb_Message_New(mini_table: *const upb_MiniTable, arena: RawArena) -> Option; + /// SAFETY: + /// - `m` and `mini_table` must be valid to deref + /// - `mini_table` must be the MiniTable associtaed with `m` + pub fn upb_Message_Clear(m: RawMessage, mini_table: *const upb_MiniTable); + + /// SAFETY: + /// - All four arguments must be valid to deref + /// - `mini_table` must be the MiniTable associated with both `dst` and + /// `src` pub fn upb_Message_DeepCopy( dst: RawMessage, src: RawMessage, @@ -24,18 +34,29 @@ extern "C" { arena: RawArena, ); + /// SAFETY: + /// - All three arguments must be valid to deref + /// - `mini_table` must be the MiniTable associated with `m` pub fn upb_Message_DeepClone( m: RawMessage, mini_table: *const upb_MiniTable, arena: RawArena, ) -> Option; + /// SAFETY: + /// - `m` and `mini_table` must be valid to deref + /// - `mini_table` must be the MiniTable associated with `m` + /// - `val` must be a pointer to legally readable memory of the correct type + /// for the field described by `mini_table` pub fn upb_Message_SetBaseField( m: RawMessage, mini_table: *const upb_MiniTableField, val: *const std::ffi::c_void, ); + /// SAFETY: + /// - All four arguments must be valid to deref + /// - `mini_table` must be the MiniTable associated with both `m1` and `m2` pub fn upb_Message_IsEqual( m1: RawMessage, m2: RawMessage, @@ -43,6 +64,11 @@ extern "C" { options: i32, ) -> bool; + /// SAFETY: + /// - `dst`, `src`, `mini_table` and `arena` must be valid to deref + /// - `extreg` must be valid to deref or nullptr + /// - `mini_table` must be the MiniTable associated with both `dst` and + /// `src` pub fn upb_Message_MergeFrom( dst: RawMessage, src: RawMessage, diff --git a/rust/upb/message_value.rs b/rust/upb/message_value.rs index 1bb5a8deb7..5cde4d6bb2 100644 --- a/rust/upb/message_value.rs +++ b/rust/upb/message_value.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::{RawArray, RawMap, RawMessage, StringView}; +use super::{RawArray, RawMap, RawMessage, StringView}; // Transcribed from google3/third_party/upb/upb/message/value.h #[repr(C)] diff --git a/rust/upb/mini_table.rs b/rust/upb/mini_table.rs index fe80a86a68..64b64ec192 100644 --- a/rust/upb/mini_table.rs +++ b/rust/upb/mini_table.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::opaque_pointee::opaque_pointee; +use super::opaque_pointee::opaque_pointee; use std::ptr::NonNull; opaque_pointee!(upb_MiniTable); diff --git a/rust/upb/owned_arena_box.rs b/rust/upb/owned_arena_box.rs index 4cd11d6666..c4ff1ac809 100644 --- a/rust/upb/owned_arena_box.rs +++ b/rust/upb/owned_arena_box.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::Arena; +use super::Arena; use std::fmt::{self, Debug}; use std::ops::{Deref, DerefMut}; use std::ptr::NonNull; diff --git a/rust/upb/text.rs b/rust/upb/text.rs index 2e6f84adaf..2f1cf09448 100644 --- a/rust/upb/text.rs +++ b/rust/upb/text.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::{upb_MiniTable, RawMessage}; +use super::{upb_MiniTable, RawMessage}; extern "C" { /// Returns the minimum needed length (excluding NULL) that `buf` has to be diff --git a/rust/upb/upb_api.c b/rust/upb/upb_api.c index 16cc9e081c..2b23a52e17 100644 --- a/rust/upb/upb_api.c +++ b/rust/upb/upb_api.c @@ -21,5 +21,3 @@ #include "upb/mini_table/message.h" // IWYU pragma: keep #include "upb/text/debug_string.h" // IWYU pragma: keep // go/keep-sorted end - -const size_t __rust_proto_kUpb_Map_Begin = kUpb_Map_Begin; diff --git a/rust/upb/wire.rs b/rust/upb/wire.rs index eaf53653a4..75e666f9d1 100644 --- a/rust/upb/wire.rs +++ b/rust/upb/wire.rs @@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -use crate::{upb_ExtensionRegistry, upb_MiniTable, Arena, RawArena, RawMessage}; +use super::{upb_ExtensionRegistry, upb_MiniTable, Arena, RawArena, RawMessage}; // LINT.IfChange(encode_status) #[repr(C)] @@ -56,12 +56,12 @@ pub unsafe fn encode( // SAFETY: // - `mini_table` is the one associated with `msg`. // - `buf` and `buf_size` are legally writable. - let status = upb_Encode(msg, mini_table, 0, arena.raw(), &mut buf, &mut len); + let status = unsafe { upb_Encode(msg, mini_table, 0, arena.raw(), &mut buf, &mut len) }; if status == EncodeStatus::Ok { assert!(!buf.is_null()); // EncodeStatus Ok should never return NULL data, even for len=0. // SAFETY: upb guarantees that `buf` is valid to read for `len`. - Ok((*std::ptr::slice_from_raw_parts(buf, len)).to_vec()) + Ok(unsafe { &*std::ptr::slice_from_raw_parts(buf, len) }.to_vec()) } else { Err(status) } @@ -87,7 +87,8 @@ pub unsafe fn decode( // - `mini_table` is the one associated with `msg` // - `buf` is legally readable for at least `buf_size` bytes. // - `extreg` is null. - let status = upb_Decode(buf, len, msg, mini_table, std::ptr::null(), options, arena.raw()); + let status = + unsafe { upb_Decode(buf, len, msg, mini_table, std::ptr::null(), options, arena.raw()) }; match status { DecodeStatus::Ok => Ok(()), _ => Err(status), diff --git a/rust/utf8.rs b/rust/utf8.rs index a452e8f916..7a169f0941 100644 --- a/rust/utf8.rs +++ b/rust/utf8.rs @@ -30,7 +30,7 @@ use std::str::from_utf8_unchecked; /// /// # Examples /// -/// ``` +/// ```ignore /// use googletest::prelude::*; /// use utf8::Utf8Chunks; /// @@ -133,7 +133,7 @@ impl fmt::Debug for Debug<'_> { /// This can be used to create functionality similar to /// [`String::from_utf8_lossy`] without allocating heap memory: /// -/// ``` +/// ```ignore /// use utf8::Utf8Chunks; /// /// fn from_utf8_lossy(input: &[u8], mut push: F) where F: FnMut(&str) { diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 07152d92d5..fa2eda48e1 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -394,6 +394,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/default_value.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/map.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/repeated_field.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_cord.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/rust/accessors/singular_string.cc diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 38c088e092..82a2dc6ea6 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -342,7 +342,7 @@ cc_library( "//src/google/protobuf:__subpackages__", ], deps = [ - "//src/google/protobuf:port", + ":port", "//src/google/protobuf/stubs:lite", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_check", @@ -479,6 +479,7 @@ cc_library( "repeated_ptr_field.cc", "wire_format_lite.cc", ], + # TODO Fix ODR violations across BUILD.bazel files. hdrs = [ "any.h", "arena.h", @@ -527,6 +528,7 @@ cc_library( ":arena_align", ":arena_allocation_policy", ":arena_cleanup", + ":endian", ":internal_visibility", ":port", ":string_block", @@ -1655,6 +1657,8 @@ cc_test( deps = [ ":cc_test_protos", ":protobuf", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings:string_view", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -2066,3 +2070,17 @@ filegroup( ], visibility = ["//pkg:__pkg__"], ) + +cc_library( + name = "endian", + hdrs = ["endian.h"], + strip_include_prefix = "/src", + visibility = [ + "//conformance:__subpackages__", + "//src/google/protobuf:__subpackages__", + ], + deps = [ + ":port", + "@com_google_absl//absl/base:config", + ], +) diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 00ef47fd6f..79bea6da18 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -315,7 +315,7 @@ class PROTOBUF_EXPORT Any final : public ::google::protobuf::Message const Any& from_msg); ::google::protobuf::internal::ArenaStringPtr type_url_; ::google::protobuf::internal::ArenaStringPtr value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::AnyMetadata _any_metadata_; PROTOBUF_TSAN_DECLARE_MEMBER }; diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index 3715740939..e9d38dcdf6 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -602,8 +602,7 @@ PROTOBUF_NOINLINE void Api::Clear() { } { // .google.protobuf.SourceContext source_context = 5; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.source_context_); diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 8c34fc7768..d05be84724 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -275,7 +275,7 @@ class PROTOBUF_EXPORT Mixin final : public ::google::protobuf::Message const Mixin& from_msg); ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr root_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -558,7 +558,7 @@ class PROTOBUF_EXPORT Method final : public ::google::protobuf::Message bool request_streaming_; bool response_streaming_; int syntax_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -848,7 +848,7 @@ class PROTOBUF_EXPORT Api final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Api& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method > methods_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin > mixins_; diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index a3cd7b7654..4ec1eaba96 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -29,6 +29,8 @@ using type_info = ::type_info; #include "absl/base/attributes.h" #include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/prefetch.h" #include "absl/log/absl_check.h" #include "absl/utility/internal/if_constexpr.h" #include "google/protobuf/arena_align.h" @@ -665,6 +667,12 @@ PROTOBUF_NOINLINE void* Arena::DefaultConstruct(Arena* arena) { template PROTOBUF_NOINLINE void* Arena::CopyConstruct(Arena* arena, const void* from) { + // If the object is larger than half a cache line, prefetch it. + // This way of prefetching is a little more aggressive than if we + // condition off a whole cache line, but benchmarks show better results. + if (sizeof(T) > ABSL_CACHELINE_SIZE / 2) { + PROTOBUF_PREFETCH_WITH_OFFSET(from, 64); + } static_assert(is_destructor_skippable::value, ""); void* mem = arena != nullptr ? arena->AllocateAligned(sizeof(T)) : ::operator new(sizeof(T)); diff --git a/src/google/protobuf/compiler/cpp/enum.cc b/src/google/protobuf/compiler/cpp/enum.cc index c40a5dff12..a8010b0f57 100644 --- a/src/google/protobuf/compiler/cpp/enum.cc +++ b/src/google/protobuf/compiler/cpp/enum.cc @@ -24,6 +24,7 @@ #include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" +#include "google/protobuf/compiler/cpp/generator.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/cpp/names.h" #include "google/protobuf/descriptor.h" @@ -49,6 +50,11 @@ absl::flat_hash_map EnumVars( enum_->containing_type() == nullptr ? "" : absl::StrCat(classname, "_")}, {"kMin", absl::StrCat(min->number())}, {"kMax", absl::StrCat(max->number())}, + {"return_type", CppGenerator::GetResolvedSourceFeatures(*enum_) + .GetExtension(::pb::cpp) + .enum_name_uses_string_view() + ? "::absl::string_view" + : "const std::string&"}, }; } @@ -182,7 +188,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { )cc"); } else { p->Emit(R"cc( - const std::string& $Msg_Enum$_Name($Msg_Enum$ value); + $return_type$ $Msg_Enum$_Name($Msg_Enum$ value); )cc"); } @@ -204,7 +210,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { if (should_cache_ || !has_reflection_) { p->Emit({{"static_assert", write_assert}}, R"cc( template - const std::string& $Msg_Enum$_Name(T value) { + $return_type$ $Msg_Enum$_Name(T value) { $static_assert$; return $Msg_Enum$_Name(static_cast<$Msg_Enum$>(value)); } @@ -216,7 +222,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { // pointers, so if the enum values are sparse, it's not worth it. p->Emit(R"cc( template <> - inline const std::string& $Msg_Enum$_Name($Msg_Enum$ value) { + inline $return_type$ $Msg_Enum$_Name($Msg_Enum$ value) { return ::$proto_ns$::internal::NameOfDenseEnum<$Msg_Enum$_descriptor, $kMin$, $kMax$>( static_cast(value)); @@ -226,7 +232,7 @@ void EnumGenerator::GenerateDefinition(io::Printer* p) { } else { p->Emit({{"static_assert", write_assert}}, R"cc( template - const std::string& $Msg_Enum$_Name(T value) { + $return_type$ $Msg_Enum$_Name(T value) { $static_assert$; return ::$proto_ns$::internal::NameOfEnum($Msg_Enum$_descriptor(), value); } @@ -322,7 +328,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* p) const { p->Emit(R"cc( template - static inline const std::string& $Enum$_Name(T value) { + static inline $return_type$ $Enum$_Name(T value) { return $Msg_Enum$_Name(value); } static inline bool $Enum$_Parse(absl::string_view name, $Enum_$* value) { @@ -514,7 +520,7 @@ void EnumGenerator::GenerateMethods(int idx, io::Printer* p) { $entries_by_number$, }; - const std::string& $Msg_Enum$_Name($Msg_Enum$ value) { + $return_type$ $Msg_Enum$_Name($Msg_Enum$ value) { static const bool kDummy = ::$proto_ns$::internal::InitializeEnumStrings( $Msg_Enum$_entries, $Msg_Enum$_entries_by_number, diff --git a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc index e8d894c700..b974f78463 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc @@ -241,7 +241,7 @@ class RepeatedEnum : public FieldGeneratorBase { if (has_cached_size_) { p->Emit(R"cc( - mutable $pbi$::CachedSize $cached_size_name$; + $pbi$::CachedSize $cached_size_name$; )cc"); } } @@ -518,41 +518,38 @@ void RepeatedEnum::GenerateSerializeWithCachedSizesToArray( } void RepeatedEnum::GenerateByteSize(io::Printer* p) const { + if (has_cached_size_) { + ABSL_CHECK(field_->is_packed()); + p->Emit(R"cc( + total_size += ::_pbi::WireFormatLite::EnumSizeWithPackedTagSize( + this_._internal_$name$(), $kTagBytes$, this_.$cached_size_$); + )cc"); + return; + } p->Emit( { - {"add_to_size", + {"tag_size", [&] { - if (!field_->is_packed()) { + if (field_->is_packed()) { p->Emit(R"cc( - total_size += std::size_t{$kTagBytes$} * count; + data_size == 0 + ? 0 + : $kTagBytes$ + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)); )cc"); - return; - } - - p->Emit(R"cc( - if (data_size > 0) { - total_size += $kTagBytes$; - total_size += ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)); - } - )cc"); - if (has_cached_size_) { + } else { p->Emit(R"cc( - this_.$cached_size_$.Set(::_pbi::ToCachedSize(data_size)); + std::size_t{$kTagBytes$} * + ::_pbi::FromIntSize(this_._internal_$name$_size()); )cc"); } }}, }, R"cc( - std::size_t data_size = 0; - auto count = static_cast(this_._internal_$name$_size()); - - for (std::size_t i = 0; i < count; ++i) { - data_size += ::_pbi::WireFormatLite::EnumSize( - this_._internal_$name$().Get(static_cast(i))); - } - total_size += data_size; - $add_to_size$; + std::size_t data_size = + ::_pbi::WireFormatLite::EnumSize(this_._internal_$name$()); + std::size_t tag_size = $tag_size$; + total_size += data_size + tag_size; )cc"); } } // namespace diff --git a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc index 918d708554..7590512e5a 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/field_generators/primitive_field.cc @@ -437,7 +437,7 @@ void RepeatedPrimitive::GeneratePrivateMembers(io::Printer* p) const { if (HasCachedSize()) { p->Emit({{"_cached_size_", MakeVarintCachedSizeName(field_)}}, R"cc( - mutable $pbi$::CachedSize $_cached_size_$; + $pbi$::CachedSize $_cached_size_$; )cc"); } } @@ -590,54 +590,53 @@ void RepeatedPrimitive::GenerateSerializeWithCachedSizesToArray( } void RepeatedPrimitive::GenerateByteSize(io::Printer* p) const { + if (HasCachedSize()) { + ABSL_CHECK(field_->is_packed()); + p->Emit( + R"cc( + total_size += + ::_pbi::WireFormatLite::$DeclaredType$SizeWithPackedTagSize( + this_._internal_$name$(), $kTagBytes$, + this_.$_field_cached_byte_size_$); + )cc"); + return; + } p->Emit( { - Sub{"data_size", - [&] { - auto fixed_size = FixedSize(field_->type()); - if (fixed_size.has_value()) { - p->Emit({{"kFixed", *fixed_size}}, R"cc( - std::size_t{$kFixed$} * - ::_pbi::FromIntSize(this_._internal_$name$_size()) - )cc"); - } else { - p->Emit(R"cc( - ::_pbi::WireFormatLite::$DeclaredType$Size( - this_._internal_$name$()) - )cc"); - } - }} // Here and below, we need to disable the default ;-chomping - // that closure substitutions do. - .WithSuffix(""), - {"maybe_cache_data_size", + {"data_size", + [&] { + auto fixed_size = FixedSize(field_->type()); + if (fixed_size.has_value()) { + p->Emit({{"kFixed", *fixed_size}}, R"cc( + std::size_t{$kFixed$} * + ::_pbi::FromIntSize(this_._internal_$name$_size()); + )cc"); + } else { + p->Emit(R"cc( + ::_pbi::WireFormatLite::$DeclaredType$Size( + this_._internal_$name$()); + )cc"); + } + }}, + {"tag_size", [&] { - if (!HasCachedSize()) return; - p->Emit(R"cc( - this_.$_field_cached_byte_size_$.Set( - ::_pbi::ToCachedSize(data_size)); - )cc"); + if (field_->is_packed()) { + p->Emit(R"cc( + data_size == 0 + ? 0 + : $kTagBytes$ + ::_pbi::WireFormatLite::Int32Size( + static_cast(data_size)); + )cc"); + } else { + p->Emit(R"cc( + std::size_t{$kTagBytes$} * + ::_pbi::FromIntSize(this_._internal_$name$_size()); + )cc"); + } }}, - Sub{"tag_size", - [&] { - if (field_->is_packed()) { - p->Emit(R"cc( - data_size == 0 - ? 0 - : $kTagBytes$ + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - )cc"); - } else { - p->Emit(R"cc( - std::size_t{$kTagBytes$} * - ::_pbi::FromIntSize(this_._internal_$name$_size()); - )cc"); - } - }} - .WithSuffix(""), }, R"cc( std::size_t data_size = $data_size$; - $maybe_cache_data_size$; std::size_t tag_size = $tag_size$; total_size += tag_size + data_size; )cc"); diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 1f868e5897..8384a5c70e 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -227,10 +227,16 @@ bool MayEmitIfNonDefaultCheck(io::Printer* p, const std::string& prefix, ABSL_CHECK(!HasHasbit(field)); if (!ShouldEmitNonDefaultCheck(field)) return false; + // SUBTLE: |format| must be a raw string without newline. + // io::Printer::Emit treats the format string as a "raw string" if it doesn't + // contain multiple lines. Note that a string of the form "\n($condition$)\n" + // (i.e. newline characters are present; there is only one non-empty line) + // will still be treated as a multi-line string. + // + // io::Printer::Emit will print a newline if the input is a multi-line string. + // In this case, we prefer to let the caller handle if-statement braces. p->Emit({{"condition", [&] { EmitNonDefaultCheck(p, prefix, field); }}}, - R"cc( - if ($condition$) { - )cc"); + /*format=*/"if ($condition$)"); return true; } @@ -1207,6 +1213,55 @@ class AccessorVerifier { } // namespace +void MessageGenerator::EmitUpdateByteSizeForField( + const FieldDescriptor* field, io::Printer* p, + int& cached_has_word_index) const { + p->Emit( + {{"comment", [&] { PrintFieldComment(Formatter{p}, field, options_); }}, + {"update_byte_size_for_field", + [&] { field_generators_.get(field).GenerateByteSize(p); }}, + {"update_cached_has_bits", + [&] { + if (!HasHasbit(field) || field->options().weak()) return; + + int has_bit_index = has_bit_indices_[field->index()]; + + if (cached_has_word_index == (has_bit_index / 32)) return; + + cached_has_word_index = (has_bit_index / 32); + p->Emit({{"index", cached_has_word_index}}, + R"cc( + cached_has_bits = this_.$has_bits$[$index$]; + )cc"); + }}, + {"check_if_field_present", + [&] { + if (!HasHasbit(field)) { + MayEmitIfNonDefaultCheck(p, "this_.", field); + return; + } + + if (field->options().weak()) { + p->Emit("if (has_$name$())"); + return; + } + + int has_bit_index = has_bit_indices_[field->index()]; + p->Emit( + {{"mask", absl::StrFormat("0x%08xu", + uint32_t{1} << (has_bit_index % 32))}}, + "if (cached_has_bits & $mask$)"); + }}}, + R"cc( + $comment$; + $update_cached_has_bits$; + $check_if_field_present$ { + //~ Force newline. + $update_byte_size_for_field$; + } + )cc"); +} + void MessageGenerator::GenerateFieldAccessorDefinitions(io::Printer* p) { p->Emit("// $classname$\n\n"); @@ -1367,7 +1422,7 @@ void MessageGenerator::GenerateImplDefinition(io::Printer* p) { )cc"); if (need_to_emit_cached_size) { p->Emit(R"cc( - mutable ::$proto_ns$::internal::CachedSize _cached_size_; + ::$proto_ns$::internal::CachedSize _cached_size_; )cc"); need_to_emit_cached_size = false; } @@ -1437,7 +1492,7 @@ void MessageGenerator::GenerateImplDefinition(io::Printer* p) { need_to_emit_cached_size = false; p->Emit(R"cc( - mutable ::$proto_ns$::internal::CachedSize _cached_size_; + ::$proto_ns$::internal::CachedSize _cached_size_; )cc"); }}, {"oneof_case", @@ -3952,12 +4007,15 @@ void MessageGenerator::GenerateClassSpecificMergeImpl(io::Printer* p) { } else if (field->is_optional() && !HasHasbit(field)) { // Merge semantics without true field presence: primitive fields are // merged only if non-zero (numeric) or non-empty (string). - bool have_enclosing_if = MayEmitIfNonDefaultCheck(p, "from.", field); - if (have_enclosing_if) format.Indent(); + bool emitted_check = MayEmitIfNonDefaultCheck(p, "from.", field); + if (emitted_check) { + p->Emit(" {\n"); + p->Indent(); + } generator.GenerateMergingCode(p); - if (have_enclosing_if) { - format.Outdent(); - format("}\n"); + if (emitted_check) { + p->Outdent(); + p->Emit("}\n"); } } else if (field->options().weak() || cached_has_word_index != HasWordIndex(field)) { @@ -4225,14 +4283,15 @@ void MessageGenerator::GenerateSerializeOneField(io::Printer* p, } )cc"); } else if (field->is_optional()) { - bool have_enclosing_if = MayEmitIfNonDefaultCheck(p, "this_.", field); - if (have_enclosing_if) p->Indent(); + bool emitted_check = MayEmitIfNonDefaultCheck(p, "this_.", field); + if (emitted_check) { + p->Emit(" {\n"); + p->Indent(); + } emit_body(); - if (have_enclosing_if) { + if (emitted_check) { p->Outdent(); - p->Emit(R"cc( - } - )cc"); + p->Emit("}\n"); } } else { emit_body(); @@ -4738,69 +4797,8 @@ void MessageGenerator::GenerateByteSize(io::Printer* p) { // Go back and emit checks for each of the fields we // processed. for (const auto* field : fields) { - p->Emit( - {{"comment", - [&] { - PrintFieldComment(Formatter{p}, field, - options_); - }}, - {"update_byte_size_for_field", - [&] { - field_generators_.get(field).GenerateByteSize( - p); - }}, - {"update_cached_has_bits", - [&] { - if (!HasHasbit(field) || - field->options().weak()) - return; - int has_bit_index = - has_bit_indices_[field->index()]; - if (cached_has_word_index == - (has_bit_index / 32)) - return; - cached_has_word_index = (has_bit_index / 32); - p->Emit({{"index", cached_has_word_index}}, - R"cc( - cached_has_bits = - this_.$has_bits$[$index$]; - )cc"); - }}, - {"check_if_field_present", - [&] { - if (HasHasbit(field)) { - if (field->options().weak()) { - p->Emit("if (has_$name$())"); - return; - } - - int has_bit_index = - has_bit_indices_[field->index()]; - p->Emit({{"mask", - absl::StrFormat( - "0x%08xu", - 1u << (has_bit_index % 32))}}, - "if (cached_has_bits & $mask$)"); - } else if (ShouldEmitNonDefaultCheck(field)) { - // Without field presence: field is - // serialized only if it has a non-default - // value. - p->Emit({{"non_default_check", - [&] { - EmitNonDefaultCheck(p, "this_.", - field); - }}}, - "if ($non_default_check$)"); - } - }}}, - R"cc( - $comment$; - $update_cached_has_bits$; - $check_if_field_present$ { - //~ Force newline. - $update_byte_size_for_field$; - } - )cc"); + EmitUpdateByteSizeForField(field, p, + cached_has_word_index); } }}, {"may_update_cached_has_word_index", diff --git a/src/google/protobuf/compiler/cpp/message.h b/src/google/protobuf/compiler/cpp/message.h index 4e5afb1084..11db273a08 100644 --- a/src/google/protobuf/compiler/cpp/message.h +++ b/src/google/protobuf/compiler/cpp/message.h @@ -186,6 +186,10 @@ class MessageGenerator { int HasWordIndex(const FieldDescriptor* field) const; std::vector RequiredFieldsBitMask() const; + // Helper function to reduce nesting levels of deep Emit calls. + void EmitUpdateByteSizeForField(const FieldDescriptor* field, io::Printer* p, + int& cached_has_word_index) const; + const Descriptor* descriptor_; int index_in_file_messages_; Options options_; diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index f188e07ca8..3aca276057 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -197,6 +197,7 @@ cc_library( "//src/google/protobuf", "//src/google/protobuf:port", "//src/google/protobuf/compiler:code_generator", + "@com_google_absl//absl/strings", ], ) diff --git a/src/google/protobuf/compiler/java/context.cc b/src/google/protobuf/compiler/java/context.cc index e37afbcb18..1469b59a98 100644 --- a/src/google/protobuf/compiler/java/context.cc +++ b/src/google/protobuf/compiler/java/context.cc @@ -153,6 +153,7 @@ void Context::InitializeFieldGeneratorInfoForFields( absl::StrAppend(&info.capitalized_name, field->number()); info.disambiguated_reason = conflict_reason[i]; } + info.options = options_; field_generator_info_map_[field] = info; } } diff --git a/src/google/protobuf/compiler/java/field_common.cc b/src/google/protobuf/compiler/java/field_common.cc index d8d7255e7b..7ff7f0cd48 100644 --- a/src/google/protobuf/compiler/java/field_common.cc +++ b/src/google/protobuf/compiler/java/field_common.cc @@ -41,6 +41,7 @@ void SetCommonFieldVariables( (*variables)["kt_capitalized_name"] = IsForbiddenKotlin(info->name) ? absl::StrCat(info->capitalized_name, "_") : info->capitalized_name; + (*variables)["jvm_synthetic"] = JvmSynthetic(info->options); if (!descriptor->is_repeated()) { (*variables)["annotation_field_type"] = std::string(FieldTypeName(descriptor->type())); diff --git a/src/google/protobuf/compiler/java/field_common.h b/src/google/protobuf/compiler/java/field_common.h index c29d1c25e9..1f4602634c 100644 --- a/src/google/protobuf/compiler/java/field_common.h +++ b/src/google/protobuf/compiler/java/field_common.h @@ -3,6 +3,7 @@ #include +#include "google/protobuf/compiler/java/options.h" #include "google/protobuf/descriptor.h" namespace google { @@ -15,6 +16,7 @@ struct FieldGeneratorInfo { std::string name; std::string capitalized_name; std::string disambiguated_reason; + Options options; }; // Oneof information used in OneofFieldGenerators. diff --git a/src/google/protobuf/compiler/java/full/message.cc b/src/google/protobuf/compiler/java/full/message.cc index 433177312a..887dfe2b2b 100644 --- a/src/google/protobuf/compiler/java/full/message.cc +++ b/src/google/protobuf/compiler/java/full/message.cc @@ -447,6 +447,10 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { "}\n" "\n"); } + if (!context_->options().opensource_runtime) { + printer->Print( + "@com.google.protobuf.Internal.ProtoMethodMayReturnNull\n"); + } printer->Print( vars, "public static $oneof_capitalized_name$Case forNumber(int value) {\n" @@ -1330,6 +1334,11 @@ void ImmutableMessageGenerator::GenerateKotlinOrNull( for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); if (field->has_presence() && GetJavaType(field) == JAVATYPE_MESSAGE) { + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } printer->Print( "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " "$full_name$?\n" diff --git a/src/google/protobuf/compiler/java/full/message_field.cc b/src/google/protobuf/compiler/java/full/message_field.cc index 54ae8dea17..f26c57b072 100644 --- a/src/google/protobuf/compiler/java/full/message_field.cc +++ b/src/google/protobuf/compiler/java/full/message_field.cc @@ -409,6 +409,7 @@ void ImmutableMessageFieldGenerator::GenerateKotlinOrNull(io::Printer* printer) if (descriptor_->has_presence() && descriptor_->real_containing_oneof() == nullptr) { printer->Print(variables_, + "$kt_deprecation$\n" "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" " get() = $kt_dsl_builder$.$name$OrNull\n"); } diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index 3cd40e2d0e..fa359f9120 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -380,6 +380,23 @@ const FieldDescriptor* MapKeyField(const FieldDescriptor* descriptor); const FieldDescriptor* MapValueField(const FieldDescriptor* descriptor); +inline std::string JvmSynthetic(const Options& options) { + return options.jvm_dsl ? "@kotlin.jvm.JvmSynthetic\n" : ""; +} + +struct JvmNameContext { + const Options& options; + io::Printer* printer; +}; + +inline void JvmName(absl::string_view name, const JvmNameContext& context) { + if (!context.options.jvm_dsl) return; + context.printer->Emit("@kotlin.jvm.JvmName(\""); + // Note: `name` will likely have vars in it that we do want to interpolate. + context.printer->Emit(name); + context.printer->Emit("\")\n"); +} + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/java_features.pb.h b/src/google/protobuf/compiler/java/java_features.pb.h index 5604c67cc5..2f851642ca 100644 --- a/src/google/protobuf/compiler/java/java_features.pb.h +++ b/src/google/protobuf/compiler/java/java_features.pb.h @@ -309,7 +309,7 @@ class PROTOC_EXPORT JavaFeatures final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const JavaFeatures& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; bool legacy_closed_enum_; int utf8_validation_; PROTOBUF_TSAN_DECLARE_MEMBER diff --git a/src/google/protobuf/compiler/java/kotlin_generator.cc b/src/google/protobuf/compiler/java/kotlin_generator.cc index c9848b390d..9c44c7acf1 100644 --- a/src/google/protobuf/compiler/java/kotlin_generator.cc +++ b/src/google/protobuf/compiler/java/kotlin_generator.cc @@ -7,6 +7,7 @@ #include "google/protobuf/compiler/java/kotlin_generator.h" +#include "absl/strings/str_cat.h" #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/java/file.h" #include "google/protobuf/compiler/java/generator.h" @@ -57,6 +58,8 @@ bool KotlinGenerator::Generate(const FileDescriptor* file, file_options.annotation_list_file = option.second; } else if (option.first == "experimental_strip_nonfunctional_codegen") { file_options.strip_nonfunctional_codegen = true; + } else if (option.first == "no_jvm_dsl") { + file_options.jvm_dsl = false; } else { *error = absl::StrCat("Unknown generator option: ", option.first); return false; @@ -81,8 +84,8 @@ bool KotlinGenerator::Generate(const FileDescriptor* file, return std::unique_ptr(context->Open(filename)); }; std::string package_dir = JavaPackageToDir(file_generator->java_package()); - std::string kotlin_filename = - absl::StrCat(package_dir, file_generator->GetKotlinClassname(), ".kt"); + std::string kotlin_filename = absl::StrCat( + package_dir, file_generator->GetKotlinClassname(), ".proto.kt"); all_files.push_back(kotlin_filename); std::string info_full_path = absl::StrCat(kotlin_filename, ".pb.meta"); if (file_options.annotate_code) { diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index bffec06347..db2d5ef658 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -303,24 +303,37 @@ void ImmutableEnumFieldLiteGenerator::GenerateBuilderMembers( void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); if (SupportUnknownEnumValue(descriptor_)) { - printer->Print( - variables_, + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$Value$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$Value$}$", name_ctx); }}, + }, "$kt_deprecation$public var $kt_name$Value: kotlin.Int\n" - " @JvmName(\"${$get$kt_capitalized_name$Value$}$\")\n" + " $jvm_name_get$" " get() = $kt_dsl_builder$.${$$kt_property_name$Value$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$Value$}$\")\n" + " $jvm_name_set$" " set(value) {\n" " $kt_dsl_builder$.${$$kt_property_name$Value$}$ = value\n" " }\n"); @@ -329,17 +342,16 @@ void ImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -894,8 +906,9 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateInitializationCode( void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -907,59 +920,73 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$ public val $kt_name$: " - "com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" - " )\n"); + printer->Print( + "$kt_deprecation$ public val $kt_name$: " + "com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" + "$ jvm_synthetic$" + " get() = com.google.protobuf.kotlin.DslList(\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "add(value: $kt_type$) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "add(value: $kt_type$) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "plusAssign(value: $kt_type$) {\n" - " add(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "plusAssign(value: $kt_type$) {\n" + " add(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" - " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" + " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." @@ -970,10 +997,12 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." "set(index: kotlin.Int, value: $kt_type$) {\n" @@ -983,14 +1012,18 @@ void RepeatedImmutableEnumFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}"); } std::string RepeatedImmutableEnumFieldLiteGenerator::GetBoxedType() const { diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 7ab58c50ba..29d285c2e3 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -837,8 +837,9 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -850,22 +851,27 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, + printer->Emit( + { + {"jvm_name", + [&] { JvmName("get$kt_capitalized_name$Map", name_ctx); }}, + }, "$kt_deprecation$ public val $kt_name$: " "com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " @JvmName(\"get$kt_capitalized_name$Map\")\n" + "$ jvm_synthetic$" + "$jvm_name$" " get() = com.google.protobuf.kotlin.DslMap(\n" " $kt_dsl_builder$.${$$kt_property_name$Map$}$\n" " )\n"); WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@JvmName(\"put$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("put$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .put(key: $kt_key_type$, value: $kt_value_type$) {\n" @@ -874,10 +880,12 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" @@ -887,10 +895,13 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"remove$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("remove$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .remove(key: $kt_key_type$) {\n" @@ -899,10 +910,13 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"putAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("putAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .putAll(map: kotlin.collections.Map<$kt_key_type$, $kt_value_type$>) " @@ -912,10 +926,13 @@ void ImmutableMapFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@JvmName(\"clear$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslMap" "<$kt_key_type$, $kt_value_type$, ${$$kt_capitalized_name$Proxy$}$>\n" " .clear() {\n" diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index 39222bbb9b..2e1ecd38a0 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -260,6 +260,10 @@ void ImmutableMessageLiteGenerator::Generate(io::Printer* printer) { "}\n" "\n"); } + if (!context_->options().opensource_runtime) { + printer->Print( + "@com.google.protobuf.Internal.ProtoMethodMayReturnNull\n"); + } printer->Print( vars, "public static $oneof_capitalized_name$Case forNumber(int value) {\n" @@ -751,16 +755,16 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( " private val _builder: $message$.Builder\n" ") {\n" " public companion object {\n" - " @kotlin.jvm.JvmSynthetic\n" + "$ jvm_synthetic$" " @kotlin.PublishedApi\n" " internal fun _create(builder: $message$.Builder): Dsl = " "Dsl(builder)\n" " }\n" "\n" - " @kotlin.jvm.JvmSynthetic\n" + "$ jvm_synthetic$" " @kotlin.PublishedApi\n" " internal fun _build(): $message$ = _builder.build()\n", - "message", + "jvm_synthetic", JvmSynthetic(context_->options()), "message", EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true))); printer->Indent(); @@ -772,19 +776,26 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( } for (auto& kv : oneofs_) { + JvmNameContext name_ctx = {context_->options(), printer}; const OneofDescriptor* oneof = kv.second; auto oneof_name = context_->GetOneofGeneratorInfo(oneof)->name; - printer->Print( + printer->Emit( + { + {"jvm_name", + [&] { JvmName("get$oneof_capitalized_name$Case", name_ctx); }}, + {"oneof_name", oneof_name}, + {"oneof_capitalized_name", + context_->GetOneofGeneratorInfo(oneof)->capitalized_name}, + {"oneof_property_name", GetKotlinPropertyName(oneof_name)}, + {"message", EscapeKotlinKeywords( + name_resolver_->GetClassName(descriptor_, true))}, + }, "public val $oneof_name$Case: $message$.$oneof_capitalized_name$Case\n" - " @JvmName(\"get$oneof_capitalized_name$Case\")\n" + "$jvm_name$" " get() = _builder.$oneof_property_name$Case\n\n" "public fun clear$oneof_capitalized_name$() {\n" " _builder.clear$oneof_capitalized_name$()\n" - "}\n", - "oneof_name", oneof_name, "oneof_capitalized_name", - context_->GetOneofGeneratorInfo(oneof)->capitalized_name, - "oneof_property_name", GetKotlinPropertyName(oneof_name), "message", - EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true))); + "}\n"); } if (descriptor_->extension_range_count() > 0) { @@ -797,9 +808,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( void ImmutableMessageLiteGenerator::GenerateKotlinMembers( io::Printer* printer) const { - printer->Print("@kotlin.jvm.JvmName(\"-initialize$camelcase_name$\")\n", - "camelcase_name", - name_resolver_->GetKotlinFactoryName(descriptor_)); + if (context_->options().jvm_dsl) { + printer->Print("@kotlin.jvm.JvmName(\"-initialize$camelcase_name$\")\n", + "camelcase_name", + name_resolver_->GetKotlinFactoryName(descriptor_)); + } printer->Print( "public inline fun $camelcase_name$(block: $message_kt$.Dsl.() -> " @@ -850,10 +863,19 @@ void ImmutableMessageLiteGenerator::GenerateTopLevelKotlinMembers( void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( io::Printer* printer) const { - // Generate getFieldOrNull getters for all optional message fields. for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); - if (field->has_presence() && GetJavaType(field) == JAVATYPE_MESSAGE) { + if (!field->has_presence() || GetJavaType(field) != JAVATYPE_MESSAGE) { + continue; + } + if (context_->options().jvm_dsl) { + // On the JVM, we can use `FooOrBuilder`, and it saves code size to + // generate only one method instead of two. + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } printer->Print( "public val $full_classname$OrBuilder.$camelcase_name$OrNull: " "$full_name$?\n" @@ -865,18 +887,60 @@ void ImmutableMessageLiteGenerator::GenerateKotlinOrNull( EscapeKotlinKeywords( name_resolver_->GetImmutableClassName(field->message_type())), "name", context_->GetFieldGeneratorInfo(field)->capitalized_name); + } else { + // Non-JVM platforms don't have `FooOrBuilder`, so we generate `Foo` + // and `Foo.Builder` methods. + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } + printer->Print( + "public val $full_classname$.$camelcase_name$OrNull: " + "$full_name$?\n" + " get() = if (has$capitalized_name$()) this.$name$ else null\n\n", + "full_classname", + EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)), + "camelcase_name", context_->GetFieldGeneratorInfo(field)->name, + "full_name", + EscapeKotlinKeywords( + name_resolver_->GetImmutableClassName(field->message_type())), + "capitalized_name", + context_->GetFieldGeneratorInfo(field)->capitalized_name, "name", + EscapeKotlinKeywords(GetKotlinPropertyName( + context_->GetFieldGeneratorInfo(field)->capitalized_name))); + if (field->options().deprecated()) { + printer->Print( + "@kotlin.Deprecated(message = \"Field $name$ is deprecated\")\n", + "name", context_->GetFieldGeneratorInfo(field)->name); + } + printer->Print( + "public val $full_classname$.Builder.$camelcase_name$OrNull: " + "$full_name$?\n" + " get() = if (has$capitalized_name$()) this.$name$ else null\n\n", + "full_classname", + EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)), + "camelcase_name", context_->GetFieldGeneratorInfo(field)->name, + "full_name", + EscapeKotlinKeywords( + name_resolver_->GetImmutableClassName(field->message_type())), + "capitalized_name", + context_->GetFieldGeneratorInfo(field)->capitalized_name, "name", + EscapeKotlinKeywords(GetKotlinPropertyName( + context_->GetFieldGeneratorInfo(field)->capitalized_name))); } } } void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( io::Printer* printer) const { + JvmNameContext name_ctx = {context_->options(), printer}; std::string message_name = EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)); printer->Print( "@Suppress(\"UNCHECKED_CAST\")\n" - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public operator fun get(extension: " "com.google.protobuf.ExtensionLite<$message$, T>): T {\n" " return if (extension.isRepeated) {\n" @@ -886,51 +950,63 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( " _builder.getExtension(extension)\n" " }\n" "}\n\n", - "message", message_name); - - printer->Print( - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.OptIn" - "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" - "@kotlin.jvm.JvmName(\"-getRepeatedExtension\")\n" - "public operator fun get(\n" - " extension: com.google.protobuf.ExtensionLite<$message$, " - "kotlin.collections.List>\n" - "): com.google.protobuf.kotlin.ExtensionList {\n" - " return com.google.protobuf.kotlin.ExtensionList(extension, " - "_builder.getExtension(extension))\n" - "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); + + if (context_->options().jvm_dsl) { + // TODO: generate this on Kotlin Native once the [Mutable]List + // issue is resolved. + printer->Emit( + { + {"jvm_name", [&] { JvmName("-getRepeatedExtension", name_ctx); }}, + {"jvm_synthetic", JvmSynthetic(context_->options())}, + {"message", message_name}, + }, + "$jvm_synthetic$" + "@kotlin.OptIn" + "(com.google.protobuf.kotlin.OnlyForUseByGeneratedProtoCode::class)\n" + "$jvm_name$" + "public operator fun get(\n" + " extension: com.google.protobuf.ExtensionLite<$message$, " + "kotlin.collections.List>\n" + "): com.google.protobuf.kotlin.ExtensionList {\n" + " return com.google.protobuf.kotlin.ExtensionList(extension, " + "_builder.getExtension(extension))\n" + "}\n\n"); + } printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public operator fun contains(extension: " "com.google.protobuf.ExtensionLite<$message$, *>): " "Boolean {\n" " return _builder.hasExtension(extension)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun clear(extension: " "com.google.protobuf.ExtensionLite<$message$, *>) " "{\n" " _builder.clearExtension(extension)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun setExtension(extension: " "com.google.protobuf.ExtensionLite<$message$, T>, " "value: T) {\n" " _builder.setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun > set(\n" " extension: com.google.protobuf.ExtensionLite<$message$, T>,\n" @@ -938,10 +1014,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( ") {\n" " setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun set(\n" " extension: com.google.protobuf.ExtensionLite<$message$, " @@ -950,10 +1027,11 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( ") {\n" " setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun set(\n" " extension: com.google.protobuf.ExtensionLite<$message$, T>,\n" @@ -961,18 +1039,24 @@ void ImmutableMessageLiteGenerator::GenerateKotlinExtensions( ") {\n" " setExtension(extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); + + // TODO: generate these methods on Kotlin Native once the + // [Mutable]List issue is resolved. + if (!context_->options().jvm_dsl) return; printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun com.google.protobuf.kotlin.ExtensionList.add(value: E) {\n" " _builder.addExtension(this.extension, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun " "com.google.protobuf.kotlin.ExtensionListoptions()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public fun com.google.protobuf.kotlin.ExtensionList.addAll(values: Iterable) {\n" " for (value in values) {\n" " add(value)\n" " }\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun " "com.google.protobuf.kotlin.ExtensionList) {\n" " addAll(values)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "public operator fun " "com.google.protobuf.kotlin.ExtensionList.set(index: Int, value: " "E) {\n" " _builder.setExtension(this.extension, index, value)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); printer->Print( - "@kotlin.jvm.JvmSynthetic\n" + "$jvm_synthetic$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline fun com.google.protobuf.kotlin.ExtensionList<*, " "$message$>.clear() {\n" " clear(extension)\n" "}\n\n", - "message", message_name); + "jvm_synthetic", JvmSynthetic(context_->options()), "message", + message_name); } } // namespace java diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index 84a10b9888..d719357318 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -279,42 +279,51 @@ void ImmutableMessageFieldLiteGenerator::GenerateBuilderMembers( void ImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); GenerateKotlinOrNull(printer); } -void ImmutableMessageFieldLiteGenerator::GenerateKotlinOrNull(io::Printer* printer) const { +void ImmutableMessageFieldLiteGenerator::GenerateKotlinOrNull( + io::Printer* printer) const { if (descriptor_->has_presence() && descriptor_->real_containing_oneof() == nullptr) { - printer->Print(variables_, - "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" - " get() = $kt_dsl_builder$.$name$OrNull\n"); + printer->Print( + "$kt_deprecation$\n" + "public val $classname$Kt.Dsl.$name$OrNull: $kt_type$?\n" + " get() = $kt_dsl_builder$.$name$OrNull\n"); } } @@ -800,8 +809,9 @@ std::string RepeatedImmutableMessageFieldLiteGenerator::GetBoxedType() const { void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -813,59 +823,73 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$ public val $kt_name$: " - "com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" - " )\n"); + printer->Print( + "$kt_deprecation$ public val $kt_name$: " + "com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" + "$ jvm_synthetic$" + " get() = com.google.protobuf.kotlin.DslList(\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "add(value: $kt_type$) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "add(value: $kt_type$) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "plusAssign(value: $kt_type$) {\n" - " add(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "plusAssign(value: $kt_type$) {\n" + " add(value)\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" - " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" + " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" + "}\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." @@ -876,10 +900,12 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." "set(index: kotlin.Int, value: $kt_type$) {\n" @@ -889,14 +915,19 @@ void RepeatedImmutableMessageFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); } } // namespace java diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index 838df002ce..3e0d174671 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -328,43 +328,55 @@ void ImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void ImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options()); if (descriptor_->name() == "is_initialized") { - printer->Print( - variables_, + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, "// TODO: b/336400327 - remove this hack; we should access properties\n" "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" + " $jvm_name_get$" " get() = $kt_dsl_builder$.get${$$kt_capitalized_name$$}$()\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" + " $jvm_name_set$" " set(value) {\n" " $kt_dsl_builder$.${$set$kt_capitalized_name$$}$(value)\n" " }\n"); } else { - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: $kt_type$\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: $kt_type$\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); } WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -713,8 +725,9 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateBuilderMembers( void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -725,59 +738,73 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( " : com.google.protobuf.kotlin.DslProxy()\n"); WriteFieldDocComment(printer, descriptor_, context_->options()); - printer->Print(variables_, - "$kt_deprecation$ public val $kt_name$: " - "com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" - " @kotlin.jvm.JvmSynthetic\n" - " get() = com.google.protobuf.kotlin.DslList(\n" - " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" - " )\n"); + printer->Print( + "$kt_deprecation$ public val $kt_name$: " + "com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>\n" + "$ jvm_synthetic$" + " get() = com.google.protobuf.kotlin.DslList(\n" + " $kt_dsl_builder$.${$$kt_property_name$List$}$\n" + " )\n"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "add(value: $kt_type$) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "add(value: $kt_type$) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "plusAssign(value: $kt_type$) {\n" - " add(value)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "plusAssign(value: $kt_type$) {\n" + " add(value)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" - " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "addAll(values: kotlin.collections.Iterable<$kt_type$>) {\n" + " $kt_dsl_builder$.${$addAll$capitalized_name$$}$(values)\n" + "}"); WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." @@ -788,10 +815,12 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." "set(index: kotlin.Int, value: $kt_type$) {\n" @@ -801,14 +830,18 @@ void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("clear$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "<$kt_type$, ${$$kt_capitalized_name$Proxy$}$>." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}"); } void RepeatedImmutablePrimitiveFieldLiteGenerator::GenerateFieldInfo( diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index 5e926422bc..e3717b3245 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -332,31 +332,38 @@ void ImmutableStringFieldLiteGenerator::GenerateBuilderMembers( void ImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; WriteFieldDocComment(printer, descriptor_, context_->options(), /* kdoc */ true); - printer->Print(variables_, - "$kt_deprecation$public var $kt_name$: kotlin.String\n" - " @JvmName(\"${$get$kt_capitalized_name$$}$\")\n" - " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" - " @JvmName(\"${$set$kt_capitalized_name$$}$\")\n" - " set(value) {\n" - " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" - " }\n"); + printer->Emit( + { + {"jvm_name_get", + [&] { JvmName("${$get$kt_capitalized_name$$}$", name_ctx); }}, + {"jvm_name_set", + [&] { JvmName("${$set$kt_capitalized_name$$}$", name_ctx); }}, + }, + "$kt_deprecation$public var $kt_name$: kotlin.String\n" + " $jvm_name_get$" + " get() = $kt_dsl_builder$.${$$kt_safe_name$$}$\n" + " $jvm_name_set$" + " set(value) {\n" + " $kt_dsl_builder$.${$$kt_safe_name$$}$ = value\n" + " }\n"); WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "public fun ${$clear$kt_capitalized_name$$}$() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}\n"); + printer->Print( + "public fun ${$clear$kt_capitalized_name$$}$() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}\n"); if (descriptor_->has_presence()) { WriteFieldAccessorDocComment(printer, descriptor_, HAZZER, context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "public fun ${$has$kt_capitalized_name$$}$(): kotlin.Boolean {\n" " return $kt_dsl_builder$.${$has$capitalized_name$$}$()\n" "}\n"); @@ -803,8 +810,9 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateBuilderMembers( void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( io::Printer* printer) const { + auto vars = printer->WithVars(variables_); + JvmNameContext name_ctx = {context_->options(), printer}; printer->Print( - variables_, "/**\n" " * An uninstantiable, behaviorless type to represent the field in\n" " * generics.\n" @@ -819,7 +827,6 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( context_->options(), /* builder */ false, /* kdoc */ true); printer->Print( - variables_, "$kt_deprecation$public val $kt_name$: " "com.google.protobuf.kotlin.DslList" "\n" @@ -833,37 +840,47 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"add$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "." - "add(value: kotlin.String) {\n" - " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("add$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "." + "add(value: kotlin.String) {\n" + " $kt_dsl_builder$.${$add$capitalized_name$$}$(value)\n" + "}\n"); // List += String WriteFieldAccessorDocComment(printer, descriptor_, LIST_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssign$kt_capitalized_name$\")\n" - "@Suppress(\"NOTHING_TO_INLINE\")\n" - "public inline operator fun com.google.protobuf.kotlin.DslList" - "." - "plusAssign(value: kotlin.String) {\n" - " add(value)\n" - "}\n"); + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssign$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "@Suppress(\"NOTHING_TO_INLINE\")\n" + "public inline operator fun com.google.protobuf.kotlin.DslList" + "." + "plusAssign(value: kotlin.String) {\n" + " add(value)\n" + "}\n"); // List.addAll(Iterable) WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"addAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("addAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public fun com.google.protobuf.kotlin.DslList" "." "addAll(values: kotlin.collections.Iterable) {\n" @@ -874,10 +891,13 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_MULTI_ADDER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"plusAssignAll$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", + [&] { JvmName("plusAssignAll$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "@Suppress(\"NOTHING_TO_INLINE\")\n" "public inline operator fun com.google.protobuf.kotlin.DslList" "." @@ -889,10 +909,12 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, LIST_INDEXED_SETTER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print( - variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"set$kt_capitalized_name$\")\n" + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" "public operator fun com.google.protobuf.kotlin.DslList" "." "set(index: kotlin.Int, value: kotlin.String) {\n" @@ -902,14 +924,17 @@ void RepeatedImmutableStringFieldLiteGenerator::GenerateKotlinDslMembers( WriteFieldAccessorDocComment(printer, descriptor_, CLEARER, context_->options(), /* builder */ false, /* kdoc */ true); - printer->Print(variables_, - "@kotlin.jvm.JvmSynthetic\n" - "@kotlin.jvm.JvmName(\"clear$kt_capitalized_name$\")\n" - "public fun com.google.protobuf.kotlin.DslList" - "." - "clear() {\n" - " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" - "}"); + printer->Emit( + { + {"jvm_name", [&] { JvmName("set$kt_capitalized_name$", name_ctx); }}, + }, + "$jvm_synthetic$" + "$jvm_name$" + "public fun com.google.protobuf.kotlin.DslList" + "." + "clear() {\n" + " $kt_dsl_builder$.${$clear$capitalized_name$$}$()\n" + "}"); } void RepeatedImmutableStringFieldLiteGenerator::GenerateFieldInfo( diff --git a/src/google/protobuf/compiler/java/options.h b/src/google/protobuf/compiler/java/options.h index fbf116bdc6..0328af4c9c 100644 --- a/src/google/protobuf/compiler/java/options.h +++ b/src/google/protobuf/compiler/java/options.h @@ -25,8 +25,8 @@ struct Options { generate_shared_code(false), enforce_lite(false), annotate_code(false), - strip_nonfunctional_codegen(false) { - } + strip_nonfunctional_codegen(false), + jvm_dsl(true) {} bool generate_immutable_code; bool generate_mutable_code; @@ -46,6 +46,10 @@ struct Options { std::string output_list_file; // If true, strip out nonfunctional codegen. bool strip_nonfunctional_codegen; + + // If true, generate JVM-specific DSL code. This defaults to true for + // compatibility with the old behavior. + bool jvm_dsl; }; } // namespace java diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index ded79ec209..107bbbf8bd 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -336,7 +336,7 @@ class PROTOC_EXPORT Version final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Version& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr suffix_; ::int32_t major_; ::int32_t minor_; @@ -585,7 +585,7 @@ class PROTOC_EXPORT CodeGeneratorResponse_File final : public ::google::protobuf ::google::protobuf::Arena* arena, const Impl_& from, const CodeGeneratorResponse_File& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr insertion_point_; ::google::protobuf::internal::ArenaStringPtr content_; @@ -857,7 +857,7 @@ class PROTOC_EXPORT CodeGeneratorResponse final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const CodeGeneratorResponse& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::compiler::CodeGeneratorResponse_File > file_; ::google::protobuf::internal::ArenaStringPtr error_; ::uint64_t supported_features_; @@ -1135,7 +1135,7 @@ class PROTOC_EXPORT CodeGeneratorRequest final : public ::google::protobuf::Mess ::google::protobuf::Arena* arena, const Impl_& from, const CodeGeneratorRequest& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField file_to_generate_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FileDescriptorProto > proto_file_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FileDescriptorProto > source_file_descriptors_; diff --git a/src/google/protobuf/compiler/python/pyi_generator.cc b/src/google/protobuf/compiler/python/pyi_generator.cc index 323f93eba4..f86777c2c2 100644 --- a/src/google/protobuf/compiler/python/pyi_generator.cc +++ b/src/google/protobuf/compiler/python/pyi_generator.cc @@ -567,7 +567,7 @@ bool PyiGenerator::Generate(const FileDescriptor* file, import_map_.clear(); // Calculate file name. file_ = file; - // In google3, devtools/python/blaze/pytype/pytype_impl.bzl uses --pyi_out to + // In google3, devtools/python/bazel/pytype/pytype_impl.bzl uses --pyi_out to // directly set the output file name. std::vector > options; ParseGeneratorParameter(parameter, &options); diff --git a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel index d7bf99e34e..15bc15fab6 100644 --- a/src/google/protobuf/compiler/rust/accessors/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/accessors/BUILD.bazel @@ -13,6 +13,7 @@ cc_library( "default_value.cc", "map.cc", "repeated_field.cc", + "singular_cord.cc", "singular_message.cc", "singular_scalar.cc", "singular_string.cc", diff --git a/src/google/protobuf/compiler/rust/accessors/accessors.cc b/src/google/protobuf/compiler/rust/accessors/accessors.cc index 333260e126..12a5fd72c3 100644 --- a/src/google/protobuf/compiler/rust/accessors/accessors.cc +++ b/src/google/protobuf/compiler/rust/accessors/accessors.cc @@ -30,8 +30,8 @@ std::unique_ptr AccessorGeneratorFor( // ctype=STRING_PIECE fields on cpp kernel yet (upb doesn't care about ctype). auto ctype = field.options().ctype(); if (ctx.is_cpp() && - (ctype == FieldOptions::CORD || - (ctype == FieldOptions::STRING_PIECE && field.is_repeated()))) { + (ctype == FieldOptions::CORD || ctype == FieldOptions::STRING_PIECE) && + field.is_repeated()) { return std::make_unique( "fields has an unsupported ctype"); } @@ -56,6 +56,9 @@ std::unique_ptr AccessorGeneratorFor( return std::make_unique(); case RustFieldType::BYTES: case RustFieldType::STRING: + if (ctype == FieldOptions::CORD) { + return std::make_unique(); + } return std::make_unique(); case RustFieldType::MESSAGE: return std::make_unique(); diff --git a/src/google/protobuf/compiler/rust/accessors/generator.h b/src/google/protobuf/compiler/rust/accessors/generator.h index 8b0cb791de..31761502b2 100644 --- a/src/google/protobuf/compiler/rust/accessors/generator.h +++ b/src/google/protobuf/compiler/rust/accessors/generator.h @@ -94,6 +94,15 @@ class SingularString final : public AccessorGenerator { void InThunkCc(Context& ctx, const FieldDescriptor& field) const override; }; +class SingularCord final : public AccessorGenerator { + public: + ~SingularCord() override = default; + void InMsgImpl(Context& ctx, const FieldDescriptor& field, + AccessorCase accessor_case) const override; + void InExternC(Context& ctx, const FieldDescriptor& field) const override; + void InThunkCc(Context& ctx, const FieldDescriptor& field) const override; +}; + class SingularMessage final : public AccessorGenerator { public: ~SingularMessage() override = default; diff --git a/src/google/protobuf/compiler/rust/accessors/map.cc b/src/google/protobuf/compiler/rust/accessors/map.cc index e38f03da5b..2bd16ff79c 100644 --- a/src/google/protobuf/compiler/rust/accessors/map.cc +++ b/src/google/protobuf/compiler/rust/accessors/map.cc @@ -107,9 +107,9 @@ void Map::InMsgImpl(Context& ctx, const FieldDescriptor& field, return; } ctx.Emit({}, R"rs( - pub fn set_$raw_field_name$(&mut self, src: $pb$::MapView<'_, $Key$, $Value$>) { - // TODO: Implement IntoProxied and avoid copying. - self.$field$_mut().copy_from(src); + pub fn set_$raw_field_name$(&mut self, src: impl $pb$::IntoProxied<$pb$::Map<$Key$, $Value$>>) { + // TODO: b/355493062 - Fix this extra copy. + self.$field$_mut().copy_from(src.into_proxied($pbi$::Private).as_view()); } )rs"); }}}, diff --git a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc index 966f835608..a8f6e2b13b 100644 --- a/src/google/protobuf/compiler/rust/accessors/repeated_field.cc +++ b/src/google/protobuf/compiler/rust/accessors/repeated_field.cc @@ -114,7 +114,7 @@ void RepeatedField::InMsgImpl(Context& ctx, const FieldDescriptor& field, pub fn set_$raw_field_name$(&mut self, src: impl $pb$::IntoProxied<$pb$::Repeated<$RsType$>>) { let minitable_field = unsafe { $pbr$::upb_MiniTable_FindFieldByNumber( - Self::raw_minitable($pbi$::Private), + ::MINI_TABLE, $field_number$ ) }; diff --git a/src/google/protobuf/compiler/rust/accessors/singular_cord.cc b/src/google/protobuf/compiler/rust/accessors/singular_cord.cc new file mode 100644 index 0000000000..900f5b4de3 --- /dev/null +++ b/src/google/protobuf/compiler/rust/accessors/singular_cord.cc @@ -0,0 +1,288 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include + +#include "absl/strings/string_view.h" +#include "google/protobuf/compiler/cpp/helpers.h" +#include "google/protobuf/compiler/rust/accessors/accessor_case.h" +#include "google/protobuf/compiler/rust/accessors/generator.h" +#include "google/protobuf/compiler/rust/context.h" +#include "google/protobuf/compiler/rust/naming.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace rust { + +void SingularCord::InMsgImpl(Context& ctx, const FieldDescriptor& field, + AccessorCase accessor_case) const { + std::string field_name = FieldNameWithCollisionAvoidance(field); + bool is_string_type = field.type() == FieldDescriptor::TYPE_STRING; + ctx.Emit( + {{"field", RsSafeName(field_name)}, + {"raw_field_name", field_name}, + {"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"borrowed_getter_thunk", ThunkName(ctx, field, "get_cord_borrowed")}, + {"owned_getter_thunk", ThunkName(ctx, field, "get_cord_owned")}, + {"is_flat_thunk", ThunkName(ctx, field, "cord_is_flat")}, + {"setter_thunk", ThunkName(ctx, field, "set")}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"getter_thunk", ThunkName(ctx, field, "get")}, + {"proxied_type", RsTypePath(ctx, field)}, + {"borrowed_type", + [&] { + if (is_string_type) { + ctx.Emit("$pb$::ProtoStr"); + } else { + ctx.Emit("[u8]"); + } + }}, + {"transform_borrowed", + [&] { + if (is_string_type) { + ctx.Emit(R"rs( + $pb$::ProtoStringCow::Borrowed( + // SAFETY: The runtime doesn't require ProtoStr to be UTF-8. + unsafe { $pb$::ProtoStr::from_utf8_unchecked(view) } + ) + )rs"); + } else { + ctx.Emit(R"rs( + $pb$::ProtoBytesCow::Borrowed( + view + ) + )rs"); + } + }}, + {"transform_owned", + [&] { + if (is_string_type) { + ctx.Emit(R"rs( + $pb$::ProtoStringCow::Owned( + $pb$::ProtoString::from_inner($pbi$::Private, inner) + ) + )rs"); + } else { + ctx.Emit(R"rs( + $pb$::ProtoBytesCow::Owned( + $pb$::ProtoBytes::from_inner($pbi$::Private, inner) + ) + )rs"); + } + }}, + {"view_lifetime", ViewLifetime(accessor_case)}, + {"view_type", + [&] { + if (is_string_type) { + ctx.Emit("$pb$::ProtoStringCow<$view_lifetime$>"); + } else { + ctx.Emit("$pb$::ProtoBytesCow<$view_lifetime$>"); + } + }}, + {"view_self", ViewReceiver(accessor_case)}, + {"getter_impl", + [&] { + if (ctx.is_cpp()) { + ctx.Emit(R"rs( + let cord_is_flat = unsafe { $is_flat_thunk$(self.raw_msg()) }; + if cord_is_flat { + let view = unsafe { $borrowed_getter_thunk$(self.raw_msg()).as_ref() }; + return $transform_borrowed$; + } + + let owned = unsafe { $owned_getter_thunk$(self.raw_msg()) }; + let inner = unsafe { $pbr$::InnerProtoString::from_raw($pbi$::Private, owned) }; + + $transform_owned$ + )rs"); + } else { + ctx.Emit(R"rs( + let view = unsafe { $getter_thunk$(self.raw_msg()).as_ref() }; + $transform_borrowed$ + )rs"); + } + }}, + {"getter", + [&] { + ctx.Emit(R"rs( + pub fn $field$($view_self$) -> $view_type$ { + $getter_impl$ + } + )rs"); + }}, + {"setter_impl", + [&] { + if (ctx.is_cpp()) { + ctx.Emit({}, + R"rs( + let s = val.into_proxied($pbi$::Private); + unsafe { + $setter_thunk$( + self.as_mutator_message_ref($pbi$::Private).msg(), + s.into_inner($pbi$::Private).into_raw($pbi$::Private) + ); + } + )rs"); + } else { + ctx.Emit(R"rs( + let s = val.into_proxied($pbi$::Private); + let (view, arena) = + s.into_inner($pbi$::Private).into_raw_parts($pbi$::Private); + + let mm_ref = + self.as_mutator_message_ref($pbi$::Private); + let parent_arena = mm_ref.arena($pbi$::Private); + + parent_arena.fuse(&arena); + + unsafe { + $setter_thunk$( + self.as_mutator_message_ref($pbi$::Private).msg(), + view + ); + } + )rs"); + } + }}, + {"setter", + [&] { + if (accessor_case == AccessorCase::VIEW) return; + ctx.Emit({}, + R"rs( + pub fn set_$raw_field_name$(&mut self, val: impl $pb$::IntoProxied<$proxied_type$>) { + $setter_impl$ + } + )rs"); + }}, + {"hazzer", + [&] { + if (!field.has_presence()) return; + ctx.Emit({}, R"rs( + pub fn has_$raw_field_name$($view_self$) -> bool { + unsafe { $hazzer_thunk$(self.raw_msg()) } + })rs"); + }}, + {"clearer", + [&] { + if (accessor_case == AccessorCase::VIEW) return; + if (!field.has_presence()) return; + ctx.Emit({}, R"rs( + pub fn clear_$raw_field_name$(&mut self) { + unsafe { $clearer_thunk$(self.raw_msg()) } + })rs"); + }}}, + R"rs( + $getter$ + $setter$ + $hazzer$ + $clearer$ + )rs"); +} + +void SingularCord::InExternC(Context& ctx, const FieldDescriptor& field) const { + ctx.Emit( + {{"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"borrowed_getter_thunk", ThunkName(ctx, field, "get_cord_borrowed")}, + {"owned_getter_thunk", ThunkName(ctx, field, "get_cord_owned")}, + {"is_flat_thunk", ThunkName(ctx, field, "cord_is_flat")}, + {"getter_thunk", ThunkName(ctx, field, "get")}, + {"setter_thunk", ThunkName(ctx, field, "set")}, + {"setter", + [&] { + if (ctx.is_cpp()) { + ctx.Emit(R"rs( + fn $setter_thunk$(raw_msg: $pbr$::RawMessage, val: $pbr$::CppStdString); + )rs"); + } else { + ctx.Emit(R"rs( + fn $setter_thunk$(raw_msg: $pbr$::RawMessage, val: $pbr$::PtrAndLen); + )rs"); + } + }}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"getter_thunks", + [&] { + if (ctx.is_cpp()) { + ctx.Emit(R"rs( + fn $is_flat_thunk$(raw_msg: $pbr$::RawMessage) -> bool; + fn $borrowed_getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::PtrAndLen; + fn $owned_getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::CppStdString; + )rs"); + } else { + ctx.Emit(R"rs( + fn $getter_thunk$(raw_msg: $pbr$::RawMessage) -> $pbr$::PtrAndLen; + )rs"); + } + }}, + {"with_presence_fields_thunks", + [&] { + if (field.has_presence()) { + ctx.Emit(R"rs( + fn $hazzer_thunk$(raw_msg: $pbr$::RawMessage) -> bool; + fn $clearer_thunk$(raw_msg: $pbr$::RawMessage); + )rs"); + } + }}}, + R"rs( + $with_presence_fields_thunks$ + $getter_thunks$ + $setter$ + )rs"); +} + +void SingularCord::InThunkCc(Context& ctx, const FieldDescriptor& field) const { + ctx.Emit( + {{"field", cpp::FieldName(&field)}, + {"QualifiedMsg", cpp::QualifiedClassName(field.containing_type())}, + {"hazzer_thunk", ThunkName(ctx, field, "has")}, + {"setter_thunk", ThunkName(ctx, field, "set")}, + {"clearer_thunk", ThunkName(ctx, field, "clear")}, + {"borrowed_getter_thunk", ThunkName(ctx, field, "get_cord_borrowed")}, + {"owned_getter_thunk", ThunkName(ctx, field, "get_cord_owned")}, + {"is_flat_thunk", ThunkName(ctx, field, "cord_is_flat")}, + {"with_presence_fields_thunks", + [&] { + if (field.has_presence()) { + ctx.Emit(R"cc( + bool $hazzer_thunk$($QualifiedMsg$* msg) { + return msg->has_$field$(); + } + void $clearer_thunk$($QualifiedMsg$* msg) { msg->clear_$field$(); } + )cc"); + } + }}}, + R"cc( + $with_presence_fields_thunks$; + + bool $is_flat_thunk$($QualifiedMsg$* msg) { + const absl::Cord& cord = msg->$field$(); + return cord.TryFlat().has_value(); + } + ::google::protobuf::rust::PtrAndLen $borrowed_getter_thunk$($QualifiedMsg$* msg) { + const absl::Cord& cord = msg->$field$(); + absl::string_view s = cord.TryFlat().value(); + return ::google::protobuf::rust::PtrAndLen(s.data(), s.size()); + } + std::string* $owned_getter_thunk$($QualifiedMsg$* msg) { + const absl::Cord& cord = msg->$field$(); + std::string* owned = new std::string(); + absl::CopyCordToString(cord, owned); + return owned; + } + void $setter_thunk$($QualifiedMsg$* msg, std::string* s) { + msg->set_$field$(absl::Cord(std::move(*s))); + delete s; + } + )cc"); +} + +} // namespace rust +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/rust/enum.cc b/src/google/protobuf/compiler/rust/enum.cc index 74613aa8f2..8d5ccb54e9 100644 --- a/src/google/protobuf/compiler/rust/enum.cc +++ b/src/google/protobuf/compiler/rust/enum.cc @@ -81,7 +81,7 @@ void EnumProxiedInMapValue(Context& ctx, const EnumDescriptor& desc) { fn $map_get_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $name$) -> bool; fn $map_remove_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $name$) -> bool; fn $map_iter_thunk$(m: $pbr$::RawMap) -> $pbr$::UntypedMapIterator; - fn $map_iter_get_thunk$(iter: &mut $pbr$::UntypedMapIterator, key: *mut $ffi_key_t$, value: *mut $name$); + fn $map_iter_get_thunk$(iter: &mut $pbr$::UntypedMapIterator, size_info: $pbr$::MapNodeSizeInfo, key: *mut $ffi_key_t$, value: *mut $name$); } impl $pb$::ProxiedInMapValue<$key_t$> for $name$ { fn map_new(_private: $pbi$::Private) -> $pb$::Map<$key_t$, Self> { @@ -149,6 +149,7 @@ void EnumProxiedInMapValue(Context& ctx, const EnumDescriptor& desc) { iter.as_raw_mut($pbi$::Private).next_unchecked::<$key_t$, Self, _, _>( $pbi$::Private, $map_iter_get_thunk$, + $pbr$::MapNodeSizeInfo(0), // Ignored |ffi_key| $from_ffi_key_expr$, $std$::convert::identity, ) @@ -389,30 +390,31 @@ void GenerateEnumDefinition(Context& ctx, const EnumDescriptor& desc) { } } - impl $pb$::IntoProxied<$name$> for $name$ { - fn into_proxied(self, _: $pbi$::Private) -> Self { - self - } - } - impl $pb$::IntoProxied for $name$ { fn into_proxied(self, _: $pbi$::Private) -> i32 { self.0 } } + impl $pbi$::SealedInternal for $name$ {} + impl $pb$::Proxied for $name$ { type View<'a> = $name$; } - impl $pb$::ViewProxy<'_> for $name$ { + impl $pb$::Proxy<'_> for $name$ {} + impl $pb$::ViewProxy<'_> for $name$ {} + + impl $pb$::AsView for $name$ { type Proxied = $name$; fn as_view(&self) -> $name$ { *self } + } - fn into_view<'shorter>(self) -> $pb$::View<'shorter, $name$> { + impl<'msg> $pb$::IntoView<'msg> for $name$ { + fn into_view<'shorter>(self) -> $name$ where 'msg: 'shorter { self } } diff --git a/src/google/protobuf/compiler/rust/generator.cc b/src/google/protobuf/compiler/rust/generator.cc index dc49e325b0..52a120f409 100644 --- a/src/google/protobuf/compiler/rust/generator.cc +++ b/src/google/protobuf/compiler/rust/generator.cc @@ -177,6 +177,16 @@ bool RustGenerator::Generate(const FileDescriptor* file, {"Phantom", "::__std::marker::PhantomData"}, }); + // On upb we need to enable this feature to be able to have const pointers to + // upb_MiniTables while using the upb C minitable codegen. This + // feature appears that it'll imminently be considered stabilized + // (https://github.com/rust-lang/rust/issues/128183), and if we emit the + // MiniTables as const directly in .rs rather than using the upb minitable .h + // this could be avoided regardless. + if (ctx.is_upb() && file == &rust_generator_context.primary_file()) { + ctx.Emit("#![feature(const_refs_to_static)]\n"); + } + ctx.Emit({{"kernel", KernelRsName(ctx.opts().kernel)}}, R"rs( extern crate protobuf_$kernel$ as __pb; extern crate std as __std; diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 54cac79093..4665f44bf6 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -11,6 +11,7 @@ #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/cpp/names.h" @@ -61,7 +62,7 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: ctx.Emit({{"serialize_thunk", ThunkName(ctx, msg, "serialize")}}, R"rs( - let mut serialized_data = $pbr$::SerializedData::new(); + let mut serialized_data = $pbr$::SerializedData::new($pbi$::Private); let success = unsafe { $serialize_thunk$(self.raw_msg(), &mut serialized_data) }; @@ -76,11 +77,10 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( - // SAFETY: $minitable$ is a static of a const object. - let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; - // SAFETY: $minitable$ is the one associated with raw_msg(). + // SAFETY: `MINI_TABLE` is the one associated with `self.raw_msg()`. let encoded = unsafe { - $pbr$::wire::encode(self.raw_msg(), mini_table) + $pbr$::wire::encode(self.raw_msg(), + ::MINI_TABLE) }; //~ TODO: This discards the info we have about the reason //~ of the failure, we should try to keep it instead. @@ -92,6 +92,28 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { ABSL_LOG(FATAL) << "unreachable"; } +void MessageMutClear(Context& ctx, const Descriptor& msg) { + switch (ctx.opts().kernel) { + case Kernel::kCpp: + ctx.Emit({{"clear_thunk", ThunkName(ctx, msg, "clear")}}, + R"rs( + unsafe { $clear_thunk$(self.raw_msg()) } + )rs"); + return; + case Kernel::kUpb: + ctx.Emit( + { + {"minitable", UpbMinitableName(msg)}, + }, + R"rs( + unsafe { + $pbr$::upb_Message_Clear(self.raw_msg(), $std$::ptr::addr_of!($minitable$)) + } + )rs"); + return; + } +} + void MessageClearAndParse(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: @@ -117,8 +139,6 @@ void MessageClearAndParse(Context& ctx, const Descriptor& msg) { ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( let mut msg = Self::new(); - // SAFETY: $minitable$ is a static of a const object. - let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; // SAFETY: // - `data.as_ptr()` is valid to read for `data.len()` @@ -126,8 +146,10 @@ void MessageClearAndParse(Context& ctx, const Descriptor& msg) { // - `msg.arena().raw()` is held for the same lifetime as `msg`. let status = unsafe { $pbr$::wire::decode( - data, msg.raw_msg(), - mini_table, msg.arena()) + data, + msg.raw_msg(), + ::MINI_TABLE, + msg.arena()) }; match status { Ok(_) => { @@ -156,13 +178,12 @@ void MessageDebug(Context& ctx, const Descriptor& msg) { return; case Kernel::kUpb: - ctx.Emit({{"minitable", UpbMinitableName(msg)}}, - R"rs( - let mini_table = unsafe { $std$::ptr::addr_of!($minitable$) }; + ctx.Emit( + R"rs( let string = unsafe { $pbr$::debug_string( self.raw_msg(), - mini_table, + ::MINI_TABLE ) }; write!(f, "{}", string) @@ -177,29 +198,30 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { switch (ctx.opts().kernel) { case Kernel::kCpp: ctx.Emit( - { - {"new_thunk", ThunkName(ctx, msg, "new")}, - {"delete_thunk", ThunkName(ctx, msg, "delete")}, - {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, - {"parse_thunk", ThunkName(ctx, msg, "parse")}, - {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, - {"merge_from_thunk", ThunkName(ctx, msg, "merge_from")}, - {"repeated_new_thunk", ThunkName(ctx, msg, "repeated_new")}, - {"repeated_free_thunk", ThunkName(ctx, msg, "repeated_free")}, - {"repeated_len_thunk", ThunkName(ctx, msg, "repeated_len")}, - {"repeated_get_thunk", ThunkName(ctx, msg, "repeated_get")}, - {"repeated_get_mut_thunk", - ThunkName(ctx, msg, "repeated_get_mut")}, - {"repeated_add_thunk", ThunkName(ctx, msg, "repeated_add")}, - {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, - {"repeated_copy_from_thunk", - ThunkName(ctx, msg, "repeated_copy_from")}, - {"repeated_reserve_thunk", - ThunkName(ctx, msg, "repeated_reserve")}, - }, + {{"new_thunk", ThunkName(ctx, msg, "new")}, + {"placement_new_thunk", ThunkName(ctx, msg, "placement_new")}, + {"delete_thunk", ThunkName(ctx, msg, "delete")}, + {"clear_thunk", ThunkName(ctx, msg, "clear")}, + {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, + {"parse_thunk", ThunkName(ctx, msg, "parse")}, + {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, + {"merge_from_thunk", ThunkName(ctx, msg, "merge_from")}, + {"repeated_new_thunk", ThunkName(ctx, msg, "repeated_new")}, + {"repeated_free_thunk", ThunkName(ctx, msg, "repeated_free")}, + {"repeated_len_thunk", ThunkName(ctx, msg, "repeated_len")}, + {"repeated_get_thunk", ThunkName(ctx, msg, "repeated_get")}, + {"repeated_get_mut_thunk", ThunkName(ctx, msg, "repeated_get_mut")}, + {"repeated_add_thunk", ThunkName(ctx, msg, "repeated_add")}, + {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, + {"repeated_copy_from_thunk", + ThunkName(ctx, msg, "repeated_copy_from")}, + {"repeated_reserve_thunk", ThunkName(ctx, msg, "repeated_reserve")}, + {"map_size_info_thunk", ThunkName(ctx, msg, "size_info")}}, R"rs( fn $new_thunk$() -> $pbr$::RawMessage; + fn $placement_new_thunk$(ptr: *mut std::ffi::c_void, m: $pbr$::RawMessage); fn $delete_thunk$(raw_msg: $pbr$::RawMessage); + fn $clear_thunk$(raw_msg: $pbr$::RawMessage); fn $serialize_thunk$(raw_msg: $pbr$::RawMessage, out: &mut $pbr$::SerializedData) -> bool; fn $parse_thunk$(raw_msg: $pbr$::RawMessage, data: $pbr$::SerializedData) -> bool; fn $copy_from_thunk$(dst: $pbr$::RawMessage, src: $pbr$::RawMessage); @@ -213,6 +235,7 @@ void MessageExterns(Context& ctx, const Descriptor& msg) { fn $repeated_clear_thunk$(raw: $pbr$::RawRepeatedField); fn $repeated_copy_from_thunk$(dst: $pbr$::RawRepeatedField, src: $pbr$::RawRepeatedField); fn $repeated_reserve_thunk$(raw: $pbr$::RawRepeatedField, additional: usize); + fn $map_size_info_thunk$(i: $pbr$::MapNodeSizeInfoIndex) -> $pbr$::MapNodeSizeInfo; )rs"); return; @@ -260,13 +283,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - $pb$::IntoProxied::into_proxied($pb$::ViewProxy::into_view(self), _private) - } - } - - impl $pb$::IntoProxied<$Msg$> for $Msg$ { - fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - self + $pb$::IntoProxied::into_proxied($pb$::IntoView::into_view(self), _private) } } )rs"); @@ -280,7 +297,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { unsafe { $pbr$::upb_Message_DeepCopy( dst.inner.msg, self.msg, - $std$::ptr::addr_of!($minitable$), + ::MINI_TABLE, dst.inner.arena.raw(), ) }; dst @@ -289,13 +306,7 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { impl<'msg> $pb$::IntoProxied<$Msg$> for $Msg$Mut<'msg> { fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - $pb$::IntoProxied::into_proxied($pb$::ViewProxy::into_view(self), _private) - } - } - - impl $pb$::IntoProxied<$Msg$> for $Msg$ { - fn into_proxied(self, _private: $pbi$::Private) -> $Msg$ { - self + $pb$::IntoProxied::into_proxied($pb$::IntoView::into_view(self), _private) } } )rs"); @@ -305,11 +316,19 @@ void IntoProxiedForMessage(Context& ctx, const Descriptor& msg) { ABSL_LOG(FATAL) << "unreachable"; } -void MessageGetMinitable(Context& ctx, const Descriptor& msg) { +void UpbGeneratedMessageTraitImpls(Context& ctx, const Descriptor& msg) { if (ctx.opts().kernel == Kernel::kUpb) { ctx.Emit({{"minitable", UpbMinitableName(msg)}}, R"rs( - pub fn raw_minitable(_private: $pbi$::Private) -> *const $pbr$::upb_MiniTable { - unsafe { $std$::ptr::addr_of!($minitable$) } + unsafe impl $pbr$::AssociatedMiniTable for $Msg$ { + const MINI_TABLE: *const $pbr$::upb_MiniTable = unsafe { $std$::ptr::addr_of!($minitable$) }; + } + + unsafe impl $pbr$::AssociatedMiniTable for $Msg$View<'_> { + const MINI_TABLE: *const $pbr$::upb_MiniTable = unsafe { $std$::ptr::addr_of!($minitable$) }; + } + + unsafe impl $pbr$::AssociatedMiniTable for $Msg$Mut<'_> { + const MINI_TABLE: *const $pbr$::upb_MiniTable = unsafe { $std$::ptr::addr_of!($minitable$) }; } )rs"); } @@ -323,7 +342,7 @@ void MessageMergeFrom(Context& ctx, const Descriptor& msg) { {"merge_from_thunk", ThunkName(ctx, msg, "merge_from")}, }, R"rs( - pub fn merge_from<'src>(&mut self, src: impl $pb$::ViewProxy<'src, Proxied = $Msg$>) { + pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { // SAFETY: self and src are both valid `$Msg$`s. unsafe { $merge_from_thunk$(self.raw_msg(), src.as_view().raw_msg()); @@ -333,17 +352,14 @@ void MessageMergeFrom(Context& ctx, const Descriptor& msg) { return; case Kernel::kUpb: ctx.Emit( - { - {"minitable", UpbMinitableName(msg)}, - }, R"rs( - pub fn merge_from<'src>(&mut self, src: impl $pb$::ViewProxy<'src, Proxied = $Msg$>) { + pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { // SAFETY: self and src are both valid `$Msg$`s. unsafe { assert!( - $pbr$::upb_Message_MergeFrom(self.raw_msg(), + $pbr$::upb_Message_MergeFrom(self.raw_msg(), src.as_view().raw_msg(), - $std$::ptr::addr_of!($minitable$), + ::MINI_TABLE, // Use a nullptr for the ExtensionRegistry. $std$::ptr::null(), self.arena().raw()) @@ -469,7 +485,6 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit( { - {"minitable", UpbMinitableName(msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, }, R"rs( @@ -547,9 +562,9 @@ void MessageProxiedInRepeated(Context& ctx, const Descriptor& msg) { dest: $pb$::Mut<$pb$::Repeated>, ) { // SAFETY: - // - Elements of `src` and `dest` have message minitable `$minitable$`. + // - Elements of `src` and `dest` have message minitable `MINI_TABLE`. unsafe { - $pbr$::repeated_message_copy_from(src, dest, $std$::ptr::addr_of!($minitable$)); + $pbr$::repeated_message_copy_from(src, dest, ::MINI_TABLE); } } @@ -576,19 +591,18 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { case Kernel::kCpp: for (const auto& t : kMapKeyTypes) { ctx.Emit( - {{"map_new_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "new")}, - {"map_free_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "free")}, - {"map_clear_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "clear")}, - {"map_size_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "size")}, - {"map_insert_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "insert")}, - {"map_get_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "get")}, - {"map_remove_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "remove")}, - {"map_iter_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "iter")}, - {"map_iter_get_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "iter_get")}, + {{"map_size_info_thunk", ThunkName(ctx, msg, "size_info")}, + {"placement_new_thunk", ThunkName(ctx, msg, "placement_new")}, + {"map_insert", + absl::StrCat("proto2_rust_map_insert_", t.thunk_ident)}, + {"map_remove", + absl::StrCat("proto2_rust_map_remove_", t.thunk_ident)}, + {"map_get", absl::StrCat("proto2_rust_map_get_", t.thunk_ident)}, + {"map_iter_get", + absl::StrCat("proto2_rust_map_iter_get_", t.thunk_ident)}, {"key_expr", t.rs_to_ffi_key_expr}, + {"key_is_string", + t.thunk_ident == "ProtoString" ? "true" : "false"}, io::Printer::Sub("ffi_key_t", [&] { ctx.Emit(t.rs_ffi_key_t); }) .WithSuffix(""), io::Printer::Sub("key_t", [&] { ctx.Emit(t.rs_key_t); }) @@ -597,47 +611,52 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { [&] { ctx.Emit(t.rs_from_ffi_key_expr); }) .WithSuffix("")}, R"rs( - extern "C" { - fn $map_new_thunk$() -> $pbr$::RawMap; - fn $map_free_thunk$(m: $pbr$::RawMap); - fn $map_clear_thunk$(m: $pbr$::RawMap); - fn $map_size_thunk$(m: $pbr$::RawMap) -> usize; - fn $map_insert_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: $pbr$::RawMessage) -> bool; - fn $map_get_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $pbr$::RawMessage) -> bool; - fn $map_remove_thunk$(m: $pbr$::RawMap, key: $ffi_key_t$, value: *mut $pbr$::RawMessage) -> bool; - fn $map_iter_thunk$(m: $pbr$::RawMap) -> $pbr$::UntypedMapIterator; - fn $map_iter_get_thunk$(iter: &mut $pbr$::UntypedMapIterator, key: *mut $ffi_key_t$, value: *mut $pbr$::RawMessage); - } impl $pb$::ProxiedInMapValue<$key_t$> for $Msg$ { fn map_new(_private: $pbi$::Private) -> $pb$::Map<$key_t$, Self> { unsafe { $pb$::Map::from_inner( $pbi$::Private, - $pbr$::InnerMap::new($pbi$::Private, $map_new_thunk$()) + $pbr$::InnerMap::new($pbi$::Private, $pbr$::proto2_rust_map_new()) ) } } unsafe fn map_free(_private: $pbi$::Private, map: &mut $pb$::Map<$key_t$, Self>) { - unsafe { $map_free_thunk$(map.as_raw($pbi$::Private)); } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { $pbr$::proto2_rust_map_free(map.as_raw($pbi$::Private), $key_is_string$, $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX)); } } fn map_clear(mut map: $pb$::Mut<'_, $pb$::Map<$key_t$, Self>>) { - unsafe { $map_clear_thunk$(map.as_raw($pbi$::Private)); } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { $pbr$::proto2_rust_map_clear(map.as_raw($pbi$::Private), $key_is_string$, $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX)); } } fn map_len(map: $pb$::View<'_, $pb$::Map<$key_t$, Self>>) -> usize { - unsafe { $map_size_thunk$(map.as_raw($pbi$::Private)) } + unsafe { $pbr$::proto2_rust_map_size(map.as_raw($pbi$::Private)) } } fn map_insert(mut map: $pb$::Mut<'_, $pb$::Map<$key_t$, Self>>, key: $pb$::View<'_, $key_t$>, value: impl $pb$::IntoProxied) -> bool { - unsafe { $map_insert_thunk$(map.as_raw($pbi$::Private), $key_expr$, value.into_proxied($pbi$::Private).raw_msg()) } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { + $pbr$::$map_insert$( + map.as_raw($pbi$::Private), + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), + $key_expr$, + value.into_proxied($pbi$::Private).raw_msg(), $placement_new_thunk$) + } } fn map_get<'a>(map: $pb$::View<'a, $pb$::Map<$key_t$, Self>>, key: $pb$::View<'_, $key_t$>) -> Option<$pb$::View<'a, Self>> { + use $pbr$::MapNodeSizeInfoIndexForType; let key = $key_expr$; let mut value = $std$::mem::MaybeUninit::uninit(); - let found = unsafe { $map_get_thunk$(map.as_raw($pbi$::Private), key, value.as_mut_ptr()) }; + let found = unsafe { + $pbr$::$map_get$( + map.as_raw($pbi$::Private), + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), + key, + value.as_mut_ptr()) + }; if !found { return None; } @@ -645,8 +664,13 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { } fn map_remove(mut map: $pb$::Mut<'_, $pb$::Map<$key_t$, Self>>, key: $pb$::View<'_, $key_t$>) -> bool { - let mut value = $std$::mem::MaybeUninit::uninit(); - unsafe { $map_remove_thunk$(map.as_raw($pbi$::Private), $key_expr$, value.as_mut_ptr()) } + use $pbr$::MapNodeSizeInfoIndexForType; + unsafe { + $pbr$::$map_remove$( + map.as_raw($pbi$::Private), + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), + $key_expr$) + } } fn map_iter(map: $pb$::View<'_, $pb$::Map<$key_t$, Self>>) -> $pb$::MapIter<'_, $key_t$, Self> { @@ -658,12 +682,13 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { unsafe { $pb$::MapIter::from_raw( $pbi$::Private, - $map_iter_thunk$(map.as_raw($pbi$::Private)) + $pbr$::proto2_rust_map_iter(map.as_raw($pbi$::Private)) ) } } fn map_iter_next<'a>(iter: &mut $pb$::MapIter<'a, $key_t$, Self>) -> Option<($pb$::View<'a, $key_t$>, $pb$::View<'a, Self>)> { + use $pbr$::MapNodeSizeInfoIndexForType; // SAFETY: // - The `MapIter` API forbids the backing map from being mutated for 'a, // and guarantees that it's the correct key and value types. @@ -673,7 +698,8 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { unsafe { iter.as_raw_mut($pbi$::Private).next_unchecked::<$key_t$, Self, _, _>( $pbi$::Private, - $map_iter_get_thunk$, + $pbr$::$map_iter_get$, + $map_size_info_thunk$($key_t$::SIZE_INFO_INDEX), |ffi_key| $from_ffi_key_expr$, |raw_msg| $Msg$View::new($pbi$::Private, raw_msg) ) @@ -686,7 +712,6 @@ void MessageProxiedInMapValue(Context& ctx, const Descriptor& msg) { case Kernel::kUpb: ctx.Emit( { - {"minitable", UpbMinitableName(msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, }, R"rs( @@ -824,6 +849,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { {{"Msg", RsSafeName(msg.name())}, {"Msg::new", [&] { MessageNew(ctx, msg); }}, {"Msg::serialize", [&] { MessageSerialize(ctx, msg); }}, + {"MsgMut::clear", [&] { MessageMutClear(ctx, msg); }}, {"Msg::clear_and_parse", [&] { MessageClearAndParse(ctx, msg); }}, {"Msg::drop", [&] { MessageDrop(ctx, msg); }}, {"Msg::debug", [&] { MessageDebug(ctx, msg); }}, @@ -927,7 +953,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } }}, {"into_proxied_impl", [&] { IntoProxiedForMessage(ctx, msg); }}, - {"get_upb_minitable", [&] { MessageGetMinitable(ctx, msg); }}, + {"upb_generated_message_trait_impls", + [&] { UpbGeneratedMessageTraitImpls(ctx, msg); }}, {"msg_merge_from", [&] { MessageMergeFrom(ctx, msg); }}, {"repeated_impl", [&] { MessageProxiedInRepeated(ctx, msg); }}, {"map_value_impl", [&] { MessageProxiedInMapValue(ctx, msg); }}, @@ -953,15 +980,39 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { impl $pb$::Message for $Msg$ {} + impl std::default::Default for $Msg$ { + fn default() -> Self { + Self::new() + } + } + + impl $pb$::Parse for $Msg$ { + fn parse(serialized: &[u8]) -> Result { + Self::parse(serialized) + } + } + impl std::fmt::Debug for $Msg$ { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { $Msg::debug$ } } - impl std::default::Default for $Msg$ { - fn default() -> Self { - Self::new() + impl $pb$::Serialize for $Msg$ { + fn serialize(&self) -> Result, $pb$::SerializeError> { + $pb$::AsView::as_view(self).serialize() + } + } + + impl $pb$::Clear for $Msg$ { + fn clear(&mut self) { + self.as_mut().clear() + } + } + + impl $pb$::ClearAndParse for $Msg$ { + fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { + self.clear_and_parse(data) } } @@ -979,6 +1030,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { type View<'msg> = $Msg$View<'msg>; } + impl $pbi$::SealedInternal for $Msg$ {} + impl $pb$::MutProxied for $Msg$ { type Mut<'msg> = $Msg$Mut<'msg>; } @@ -990,7 +1043,11 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { _phantom: $Phantom$<&'msg ()>, } - impl $pb$::MessageView for $Msg$View<'_> {} + impl<'msg> $pbi$::SealedInternal for $Msg$View<'msg> {} + + impl<'msg> $pb$::MessageView<'msg> for $Msg$View<'msg> { + type Message = $Msg$; + } impl std::fmt::Debug for $Msg$View<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -998,6 +1055,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::Serialize for $Msg$View<'_> { + fn serialize(&self) -> Result, $pb$::SerializeError> { + $Msg::serialize$ + } + } + #[allow(dead_code)] impl<'msg> $Msg$View<'msg> { #[doc(hidden)] @@ -1009,10 +1072,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.msg } - pub fn serialize(&self) -> Result, $pb$::SerializeError> { - $Msg::serialize$ - } - pub fn to_owned(&self) -> $Msg$ { $pb$::IntoProxied::into_proxied(*self, $pbi$::Private) } @@ -1029,13 +1088,20 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { // - `$Msg$View` does not use thread-local data. unsafe impl Send for $Msg$View<'_> {} - impl<'msg> $pb$::ViewProxy<'msg> for $Msg$View<'msg> { - type Proxied = $Msg$; + impl<'msg> $pb$::Proxy<'msg> for $Msg$View<'msg> {} + impl<'msg> $pb$::ViewProxy<'msg> for $Msg$View<'msg> {} + impl<'msg> $pb$::AsView for $Msg$View<'msg> { + type Proxied = $Msg$; fn as_view(&self) -> $pb$::View<'msg, $Msg$> { *self } - fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> where 'msg: 'shorter { + } + + impl<'msg> $pb$::IntoView<'msg> for $Msg$View<'msg> { + fn into_view<'shorter>(self) -> $Msg$View<'shorter> + where + 'msg: 'shorter { self } } @@ -1051,7 +1117,11 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { inner: $pbr$::MutatorMessageRef<'msg>, } - impl $pb$::MessageMut for $Msg$Mut<'_> {} + impl<'msg> $pbi$::SealedInternal for $Msg$Mut<'msg> {} + + impl<'msg> $pb$::MessageMut<'msg> for $Msg$Mut<'msg> { + type Message = $Msg$; + } impl std::fmt::Debug for $Msg$Mut<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -1059,6 +1129,18 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::Serialize for $Msg$Mut<'_> { + fn serialize(&self) -> Result, $pb$::SerializeError> { + $pb$::AsView::as_view(self).serialize() + } + } + + impl $pb$::Clear for $Msg$Mut<'_> { + fn clear(&mut self) { + $MsgMut::clear$ + } + } + #[allow(dead_code)] impl<'msg> $Msg$Mut<'msg> { #[doc(hidden)] @@ -1087,18 +1169,12 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.inner } - pub fn serialize(&self) -> Result, $pb$::SerializeError> { - $pb$::ViewProxy::as_view(self).serialize() - } - pub fn to_owned(&self) -> $Msg$ { - $pb$::ViewProxy::as_view(self).to_owned() + $pb$::AsView::as_view(self).to_owned() } $msg_merge_from$ - $get_upb_minitable$ - $raw_arena_getter_for_msgmut$ $accessor_fns_for_muts$ @@ -1110,23 +1186,39 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { // splitting, synchronous access of an arena is impossible. unsafe impl Sync for $Msg$Mut<'_> {} - impl<'msg> $pb$::MutProxy<'msg> for $Msg$Mut<'msg> { - fn as_mut(&mut self) -> $pb$::Mut<'_, $Msg$> { - $Msg$Mut { inner: self.inner } - } - fn into_mut<'shorter>(self) -> $pb$::Mut<'shorter, $Msg$> where 'msg : 'shorter { self } - } + impl<'msg> $pb$::Proxy<'msg> for $Msg$Mut<'msg> {} + impl<'msg> $pb$::MutProxy<'msg> for $Msg$Mut<'msg> {} - impl<'msg> $pb$::ViewProxy<'msg> for $Msg$Mut<'msg> { + impl<'msg> $pb$::AsView for $Msg$Mut<'msg> { type Proxied = $Msg$; fn as_view(&self) -> $pb$::View<'_, $Msg$> { $Msg$View { msg: self.raw_msg(), _phantom: $std$::marker::PhantomData } } - fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> where 'msg: 'shorter { + } + + impl<'msg> $pb$::IntoView<'msg> for $Msg$Mut<'msg> { + fn into_view<'shorter>(self) -> $pb$::View<'shorter, $Msg$> + where + 'msg: 'shorter { $Msg$View { msg: self.raw_msg(), _phantom: $std$::marker::PhantomData } } } + impl<'msg> $pb$::AsMut for $Msg$Mut<'msg> { + type MutProxied = $Msg$; + fn as_mut(&mut self) -> $Msg$Mut<'msg> { + $Msg$Mut { inner: self.inner } + } + } + + impl<'msg> $pb$::IntoMut<'msg> for $Msg$Mut<'msg> { + fn into_mut<'shorter>(self) -> $Msg$Mut<'shorter> + where + 'msg: 'shorter { + self + } + } + #[allow(dead_code)] impl $Msg$ { pub fn new() -> Self { @@ -1143,13 +1235,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $raw_arena_getter_for_message$ - pub fn serialize(&self) -> Result, $pb$::SerializeError> { - self.as_view().serialize() - } - #[deprecated = "Prefer Msg::parse(), or use the new name 'clear_and_parse' to parse into a pre-existing message."] - pub fn deserialize(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { - self.clear_and_parse(data) - } pub fn clear_and_parse(&mut self, data: &[u8]) -> Result<(), $pb$::ParseError> { $Msg::clear_and_parse$ } @@ -1166,12 +1251,10 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $Msg$Mut::new($pbi$::Private, &mut self.inner) } - pub fn merge_from<'src>(&mut self, src: impl $pb$::ViewProxy<'src, Proxied = $Msg$>) { + pub fn merge_from<'src>(&mut self, src: impl $pb$::Proxy<'src, Proxied = $Msg$>) { self.as_mut().merge_from(src); } - $get_upb_minitable$ - $accessor_fns$ } // impl $Msg$ @@ -1189,6 +1272,22 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { } } + impl $pb$::AsView for $Msg$ { + type Proxied = Self; + fn as_view(&self) -> $Msg$View { + self.as_view() + } + } + + impl $pb$::AsMut for $Msg$ { + type MutProxied = Self; + fn as_mut(&mut self) -> $Msg$Mut { + self.as_mut() + } + } + + $upb_generated_message_trait_impls$ + extern "C" { $Msg_externs$ @@ -1200,8 +1299,8 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $nested_in_msg$ )rs"); + ctx.printer().PrintRaw("\n"); if (ctx.is_cpp()) { - ctx.printer().PrintRaw("\n"); ctx.Emit({{"Msg", RsSafeName(msg.name())}}, R"rs( impl $Msg$ { pub fn __unstable_wrap_cpp_grant_permission_to_break(msg: $pbr$::RawMessage) -> Self { @@ -1212,7 +1311,6 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { s.raw_msg() } } - impl<'a> $Msg$Mut<'a> { //~ msg is a &mut so that the borrow checker enforces exclusivity to //~ prevent constructing multiple Muts/Views from the same RawMessage. @@ -1238,9 +1336,58 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.msg } } + + impl $pb$::OwnedMessageInterop for $Msg$ { + unsafe fn __unstable_take_ownership_of_raw_message(msg: $pbr$::RawMessage) -> Self { + Self { inner: $pbr$::MessageInner { msg } } + } + + fn __unstable_leak_raw_message(self) -> $pbr$::RawMessage { + let s = std::mem::ManuallyDrop::new(self); + s.raw_msg() + } + } + + impl<'a> $pb$::MessageMutInterop<'a> for $Msg$Mut<'a> { + unsafe fn __unstable_wrap_raw_message_mut( + msg: &'a mut $pbr$::RawMessage) -> Self { + Self { + inner: $pbr$::MutatorMessageRef::from_raw_msg($pbi$::Private, msg) + } + } + fn __unstable_as_raw_message_mut(&mut self) -> $pbr$::RawMessage { + self.raw_msg() + } + } + + impl<'a> $pb$::MessageViewInterop<'a> for $Msg$View<'a> { + unsafe fn __unstable_wrap_raw_message( + msg: &'a $pbr$::RawMessage) -> Self { + Self::new($pbi$::Private, *msg) + } + fn __unstable_as_raw_message(&self) -> $pbr$::RawMessage { + self.msg + } + } + )rs"); + } else { + ctx.Emit({{"Msg", RsSafeName(msg.name())}}, R"rs( + // upb kernel doesn't support any owned message or message mut interop. + impl $pb$::OwnedMessageInterop for $Msg$ {} + impl<'a> $pb$::MessageMutInterop<'a> for $Msg$Mut<'a> {} + + impl<'a> $pb$::MessageViewInterop<'a> for $Msg$View<'a> { + unsafe fn __unstable_wrap_raw_message( + msg: &'a $pbr$::RawMessage) -> Self { + Self::new($pbi$::Private, *msg) + } + fn __unstable_as_raw_message(&self) -> $pbr$::RawMessage { + self.msg + } + } )rs"); } -} +} // NOLINT(readability/fn_size) // Generates code for a particular message in `.pb.thunk.cc`. void GenerateThunksCc(Context& ctx, const Descriptor& msg) { @@ -1255,7 +1402,9 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { {"Msg", RsSafeName(msg.name())}, {"QualifiedMsg", cpp::QualifiedClassName(&msg)}, {"new_thunk", ThunkName(ctx, msg, "new")}, + {"placement_new_thunk", ThunkName(ctx, msg, "placement_new")}, {"delete_thunk", ThunkName(ctx, msg, "delete")}, + {"clear_thunk", ThunkName(ctx, msg, "clear")}, {"serialize_thunk", ThunkName(ctx, msg, "serialize")}, {"parse_thunk", ThunkName(ctx, msg, "parse")}, {"copy_from_thunk", ThunkName(ctx, msg, "copy_from")}, @@ -1269,6 +1418,7 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { {"repeated_clear_thunk", ThunkName(ctx, msg, "repeated_clear")}, {"repeated_copy_from_thunk", ThunkName(ctx, msg, "repeated_copy_from")}, {"repeated_reserve_thunk", ThunkName(ctx, msg, "repeated_reserve")}, + {"map_size_info_thunk", ThunkName(ctx, msg, "size_info")}, {"nested_msg_thunks", [&] { for (int i = 0; i < msg.nested_type_count(); ++i) { @@ -1291,13 +1441,19 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { } }}}, R"cc( - //~ $abi$ is a workaround for a syntax highlight bug in VSCode. However, - //~ that confuses clang-format (it refuses to keep the newline after - //~ `$abi${`). Disabling clang-format for the block. + //~ $abi$ is a workaround for a syntax highlight bug in VSCode. + // However, ~ that confuses clang-format (it refuses to keep the + // newline after ~ `$abi${`). Disabling clang-format for the block. // clang-format off extern $abi$ { void* $new_thunk$() { return new $QualifiedMsg$(); } + void $placement_new_thunk$(void* ptr, $QualifiedMsg$& m) { + new (ptr) $QualifiedMsg$(std::move(m)); + } void $delete_thunk$(void* ptr) { delete static_cast<$QualifiedMsg$*>(ptr); } + void $clear_thunk$(void* ptr) { + static_cast<$QualifiedMsg$*>(ptr)->Clear(); + } bool $serialize_thunk$($QualifiedMsg$* msg, google::protobuf::rust::SerializedData* out) { return google::protobuf::rust::SerializeMsg(msg, out); } @@ -1351,89 +1507,32 @@ void GenerateThunksCc(Context& ctx, const Descriptor& msg) { size_t additional) { field->Reserve(field->size() + additional); } + google::protobuf::internal::MapNodeSizeInfoT $map_size_info_thunk$(int32_t i) { + static constexpr google::protobuf::internal::MapNodeSizeInfoT size_infos[] = {)cc" + // LINT.IfChange(size_info_mapping) + R"cc( + google::protobuf::internal::RustMapHelper::SizeInfo(), + google::protobuf::internal::RustMapHelper::SizeInfo(), + google::protobuf::internal::RustMapHelper::SizeInfo(), + google::protobuf::internal::RustMapHelper::SizeInfo() + )cc" + // LINT.ThenChange(//depot/google3/third_party/protobuf/rust/cpp.rs:size_info_mapping) + R"cc( + } + ; + return size_infos[i]; + } $accessor_thunks$ - $oneof_thunks$ + $oneof_thunks$ } // extern $abi$ // clang-format on $nested_msg_thunks$ )cc"); - for (const auto& t : kMapKeyTypes) { - ctx.Emit( - { - {"map_new_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "new")}, - {"map_free_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "free")}, - {"map_clear_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "clear")}, - {"map_size_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "size")}, - {"map_insert_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "insert")}, - {"map_get_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "get")}, - {"map_remove_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "remove")}, - {"map_iter_thunk", RawMapThunk(ctx, msg, t.thunk_ident, "iter")}, - {"map_iter_get_thunk", - RawMapThunk(ctx, msg, t.thunk_ident, "iter_get")}, - {"key_t", t.cc_key_t}, - {"ffi_key_t", t.cc_ffi_key_t}, - {"key_expr", t.cc_from_ffi_key_expr}, - {"to_ffi_key_expr", t.cc_to_ffi_key_expr}, - {"pkg::Msg", cpp::QualifiedClassName(&msg)}, - {"abi", "\"C\""}, // Workaround for syntax highlight bug in VSCode. - }, - R"cc( - extern $abi$ { - const google::protobuf::Map<$key_t$, $pkg::Msg$>* $map_new_thunk$() { - return new google::protobuf::Map<$key_t$, $pkg::Msg$>(); - } - void $map_free_thunk$(const google::protobuf::Map<$key_t$, $pkg::Msg$>* m) { delete m; } - void $map_clear_thunk$(google::protobuf::Map<$key_t$, $pkg::Msg$> * m) { m->clear(); } - size_t $map_size_thunk$(const google::protobuf::Map<$key_t$, $pkg::Msg$>* m) { - return m->size(); - } - bool $map_insert_thunk$(google::protobuf::Map<$key_t$, $pkg::Msg$> * m, - $ffi_key_t$ key, $pkg::Msg$ value) { - auto k = $key_expr$; - auto it = m->find(k); - if (it != m->end()) { - return false; - } - (*m)[k] = value; - return true; - } - bool $map_get_thunk$(const google::protobuf::Map<$key_t$, $pkg::Msg$>* m, - $ffi_key_t$ key, const $pkg::Msg$** value) { - auto it = m->find($key_expr$); - if (it == m->end()) { - return false; - } - const $pkg::Msg$* cpp_value = &it->second; - *value = cpp_value; - return true; - } - bool $map_remove_thunk$(google::protobuf::Map<$key_t$, $pkg::Msg$> * m, - $ffi_key_t$ key, $pkg::Msg$ * value) { - auto num_removed = m->erase($key_expr$); - return num_removed > 0; - } - google::protobuf::internal::UntypedMapIterator $map_iter_thunk$( - const google::protobuf::Map<$key_t$, $pkg::Msg$>* m) { - return google::protobuf::internal::UntypedMapIterator::FromTyped(m->cbegin()); - } - void $map_iter_get_thunk$( - const google::protobuf::internal::UntypedMapIterator* iter, - $ffi_key_t$* key, const $pkg::Msg$** value) { - auto typed_iter = iter->ToTyped< - google::protobuf::Map<$key_t$, $pkg::Msg$>::const_iterator>(); - const auto& cpp_key = typed_iter->first; - const auto& cpp_value = typed_iter->second; - *key = $to_ffi_key_expr$; - *value = &cpp_value; - } - } - )cc"); - } } } // namespace rust diff --git a/src/google/protobuf/cpp_edition_defaults.h b/src/google/protobuf/cpp_edition_defaults.h index 33cdbab8a3..52e5fd65ae 100644 --- a/src/google/protobuf/cpp_edition_defaults.h +++ b/src/google/protobuf/cpp_edition_defaults.h @@ -5,7 +5,7 @@ // the C++ runtime. This is used for feature resolution under Editions. // NOLINTBEGIN // clang-format off -#define PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS "\n\035\030\204\007\"\003\302>\000*\023\010\001\020\002\030\002 \003(\0010\002\302>\004\010\001\020\003\n\035\030\347\007\"\003\302>\000*\023\010\002\020\001\030\001 \002(\0010\001\302>\004\010\000\020\003\n\035\030\350\007\"\023\010\001\020\001\030\001 \002(\0010\001\302>\004\010\000\020\003*\003\302>\000\n\035\030\351\007\"\023\010\001\020\001\030\001 \002(\0010\001\302>\004\010\000\020\001*\003\302>\000 \346\007(\351\007" +#define PROTOBUF_INTERNAL_CPP_EDITION_DEFAULTS "\n\037\030\204\007\"\003\302>\000*\025\010\001\020\002\030\002 \003(\0010\002\302>\006\010\001\020\003\030\000\n\037\030\347\007\"\003\302>\000*\025\010\002\020\001\030\001 \002(\0010\001\302>\006\010\000\020\003\030\000\n\037\030\350\007\"\023\010\001\020\001\030\001 \002(\0010\001\302>\004\010\000\020\003*\005\302>\002\030\000\n\037\030\351\007\"\025\010\001\020\001\030\001 \002(\0010\001\302>\006\010\000\020\001\030\001*\003\302>\000 \346\007(\351\007" // clang-format on // NOLINTEND diff --git a/src/google/protobuf/cpp_features.pb.cc b/src/google/protobuf/cpp_features.pb.cc index e97ea4d6fc..4dea8f4771 100644 --- a/src/google/protobuf/cpp_features.pb.cc +++ b/src/google/protobuf/cpp_features.pb.cc @@ -28,8 +28,9 @@ namespace pb { inline constexpr CppFeatures::Impl_::Impl_( ::_pbi::ConstantInitialized) noexcept : _cached_size_{0}, + string_type_{static_cast< ::pb::CppFeatures_StringType >(0)}, legacy_closed_enum_{false}, - string_type_{static_cast< ::pb::CppFeatures_StringType >(0)} {} + enum_name_uses_string_view_{false} {} template PROTOBUF_CONSTEXPR CppFeatures::CppFeatures(::_pbi::ConstantInitialized) @@ -67,13 +68,15 @@ const ::uint32_t ~0u, // no sizeof(Split) PROTOBUF_FIELD_OFFSET(::pb::CppFeatures, _impl_.legacy_closed_enum_), PROTOBUF_FIELD_OFFSET(::pb::CppFeatures, _impl_.string_type_), - 0, + PROTOBUF_FIELD_OFFSET(::pb::CppFeatures, _impl_.enum_name_uses_string_view_), 1, + 0, + 2, }; static const ::_pbi::MigrationSchema schemas[] ABSL_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - {0, 10, -1, sizeof(::pb::CppFeatures)}, + {0, 11, -1, sizeof(::pb::CppFeatures)}, }; static const ::_pb::Message* const file_default_instances[] = { &::pb::_CppFeatures_default_instance_._instance, @@ -81,7 +84,7 @@ static const ::_pb::Message* const file_default_instances[] = { const char descriptor_table_protodef_google_2fprotobuf_2fcpp_5ffeatures_2eproto[] ABSL_ATTRIBUTE_SECTION_VARIABLE( protodesc_cold) = { "\n\"google/protobuf/cpp_features.proto\022\002pb" - "\032 google/protobuf/descriptor.proto\"\256\003\n\013C" + "\032 google/protobuf/descriptor.proto\"\374\003\n\013C" "ppFeatures\022\373\001\n\022legacy_closed_enum\030\001 \001(\010B" "\336\001\210\001\001\230\001\004\230\001\001\242\001\t\022\004true\030\204\007\242\001\n\022\005false\030\347\007\262\001\270\001" "\010\350\007\020\350\007\032\257\001The legacy closed enum treatmen" @@ -90,11 +93,13 @@ const char descriptor_table_protodef_google_2fprotobuf_2fcpp_5ffeatures_2eproto[ "m type on the enum definitions themselve" "s rather than on fields.\022Z\n\013string_type\030" "\002 \001(\0162\032.pb.CppFeatures.StringTypeB)\210\001\001\230\001" - "\004\230\001\001\242\001\013\022\006STRING\030\204\007\242\001\t\022\004VIEW\030\351\007\262\001\003\010\350\007\"E\n\n" - "StringType\022\027\n\023STRING_TYPE_UNKNOWN\020\000\022\010\n\004V" - "IEW\020\001\022\010\n\004CORD\020\002\022\n\n\006STRING\020\003::\n\003cpp\022\033.goo" - "gle.protobuf.FeatureSet\030\350\007 \001(\0132\017.pb.CppF" - "eatures" + "\004\230\001\001\242\001\013\022\006STRING\030\204\007\242\001\t\022\004VIEW\030\351\007\262\001\003\010\350\007\022L\n\032" + "enum_name_uses_string_view\030\003 \001(\010B(\210\001\002\230\001\006" + "\230\001\001\242\001\n\022\005false\030\204\007\242\001\t\022\004true\030\351\007\262\001\003\010\351\007\"E\n\nSt" + "ringType\022\027\n\023STRING_TYPE_UNKNOWN\020\000\022\010\n\004VIE" + "W\020\001\022\010\n\004CORD\020\002\022\n\n\006STRING\020\003::\n\003cpp\022\033.googl" + "e.protobuf.FeatureSet\030\350\007 \001(\0132\017.pb.CppFea" + "tures" }; static const ::_pbi::DescriptorTable* const descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2eproto_deps[1] = { @@ -104,7 +109,7 @@ static ::absl::once_flag descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2ep PROTOBUF_CONSTINIT const ::_pbi::DescriptorTable descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2eproto = { false, false, - 567, + 645, descriptor_table_protodef_google_2fprotobuf_2fcpp_5ffeatures_2eproto, "google/protobuf/cpp_features.proto", &descriptor_table_google_2fprotobuf_2fcpp_5ffeatures_2eproto_once, @@ -172,11 +177,11 @@ inline PROTOBUF_NDEBUG_INLINE CppFeatures::Impl_::Impl_( inline void CppFeatures::SharedCtor(::_pb::Arena* arena) { new (&_impl_) Impl_(internal_visibility(), arena); ::memset(reinterpret_cast(&_impl_) + - offsetof(Impl_, legacy_closed_enum_), + offsetof(Impl_, string_type_), 0, - offsetof(Impl_, string_type_) - - offsetof(Impl_, legacy_closed_enum_) + - sizeof(Impl_::string_type_)); + offsetof(Impl_, enum_name_uses_string_view_) - + offsetof(Impl_, string_type_) + + sizeof(Impl_::enum_name_uses_string_view_)); } CppFeatures::~CppFeatures() { // @@protoc_insertion_point(destructor:pb.CppFeatures) @@ -217,15 +222,15 @@ const ::google::protobuf::MessageLite::ClassData* CppFeatures::GetClassData() co return _class_data_.base(); } PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 -const ::_pbi::TcParseTable<1, 2, 1, 0, 2> CppFeatures::_table_ = { +const ::_pbi::TcParseTable<2, 3, 1, 0, 2> CppFeatures::_table_ = { { PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_._has_bits_), 0, // no _extensions_ - 2, 8, // max_field_number, fast_idx_mask + 3, 24, // max_field_number, fast_idx_mask offsetof(decltype(_table_), field_lookup_table), - 4294967292, // skipmap + 4294967288, // skipmap offsetof(decltype(_table_), field_entries), - 2, // num_field_entries + 3, // num_field_entries 1, // num_aux_entries offsetof(decltype(_table_), aux_entries), _class_data_.base(), @@ -235,21 +240,28 @@ const ::_pbi::TcParseTable<1, 2, 1, 0, 2> CppFeatures::_table_ = { ::_pbi::TcParser::GetTable<::pb::CppFeatures>(), // to_prefetch #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ + {::_pbi::TcParser::MiniParse, {}}, + // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { + {::_pbi::TcParser::SingularVarintNoZag1(), + {8, 1, 0, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_)}}, // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { {::_pbi::TcParser::FastEr0S1, - {16, 1, 3, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_)}}, - // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - {::_pbi::TcParser::SingularVarintNoZag1(), - {8, 0, 0, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_)}}, + {16, 0, 3, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_)}}, + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + {::_pbi::TcParser::SingularVarintNoZag1(), + {24, 2, 0, PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.enum_name_uses_string_view_)}}, }}, {{ 65535, 65535 }}, {{ // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_), _Internal::kHasBitsOffset + 0, 0, + {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_), _Internal::kHasBitsOffset + 1, 0, (0 | ::_fl::kFcOptional | ::_fl::kBool)}, // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_), _Internal::kHasBitsOffset + 1, 0, + {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_), _Internal::kHasBitsOffset + 0, 0, (0 | ::_fl::kFcOptional | ::_fl::kEnumRange)}, + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + {PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.enum_name_uses_string_view_), _Internal::kHasBitsOffset + 2, 0, + (0 | ::_fl::kFcOptional | ::_fl::kBool)}, }}, {{ {0, 4}, }}, {{ @@ -264,10 +276,10 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { (void) cached_has_bits; cached_has_bits = _impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - ::memset(&_impl_.legacy_closed_enum_, 0, static_cast<::size_t>( - reinterpret_cast(&_impl_.string_type_) - - reinterpret_cast(&_impl_.legacy_closed_enum_)) + sizeof(_impl_.string_type_)); + if (cached_has_bits & 0x00000007u) { + ::memset(&_impl_.string_type_, 0, static_cast<::size_t>( + reinterpret_cast(&_impl_.enum_name_uses_string_view_) - + reinterpret_cast(&_impl_.string_type_)) + sizeof(_impl_.enum_name_uses_string_view_)); } _impl_._has_bits_.Clear(); _internal_metadata_.Clear<::google::protobuf::UnknownFieldSet>(); @@ -290,19 +302,26 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { cached_has_bits = this_._impl_._has_bits_[0]; // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000001u) { + if (cached_has_bits & 0x00000002u) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteBoolToArray( 1, this_._internal_legacy_closed_enum(), target); } // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000002u) { + if (cached_has_bits & 0x00000001u) { target = stream->EnsureSpace(target); target = ::_pbi::WireFormatLite::WriteEnumToArray( 2, this_._internal_string_type(), target); } + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + if (cached_has_bits & 0x00000004u) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteBoolToArray( + 3, this_._internal_enum_name_uses_string_view(), target); + } + if (PROTOBUF_PREDICT_FALSE(this_._internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( @@ -328,16 +347,20 @@ PROTOBUF_NOINLINE void CppFeatures::Clear() { ::_pbi::Prefetch5LinesFrom7Lines(&this_); cached_has_bits = this_._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { - // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000001u) { - total_size += 2; - } + if (cached_has_bits & 0x00000007u) { // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - if (cached_has_bits & 0x00000002u) { + if (cached_has_bits & 0x00000001u) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this_._internal_string_type()); } + // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { + if (cached_has_bits & 0x00000002u) { + total_size += 2; + } + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + if (cached_has_bits & 0x00000004u) { + total_size += 2; + } } return this_.MaybeComputeUnknownFieldsSize(total_size, &this_._impl_._cached_size_); @@ -352,12 +375,15 @@ void CppFeatures::MergeImpl(::google::protobuf::MessageLite& to_msg, const ::goo (void) cached_has_bits; cached_has_bits = from._impl_._has_bits_[0]; - if (cached_has_bits & 0x00000003u) { + if (cached_has_bits & 0x00000007u) { if (cached_has_bits & 0x00000001u) { - _this->_impl_.legacy_closed_enum_ = from._impl_.legacy_closed_enum_; + _this->_impl_.string_type_ = from._impl_.string_type_; } if (cached_has_bits & 0x00000002u) { - _this->_impl_.string_type_ = from._impl_.string_type_; + _this->_impl_.legacy_closed_enum_ = from._impl_.legacy_closed_enum_; + } + if (cached_has_bits & 0x00000004u) { + _this->_impl_.enum_name_uses_string_view_ = from._impl_.enum_name_uses_string_view_; } } _this->_impl_._has_bits_[0] |= cached_has_bits; @@ -377,11 +403,11 @@ void CppFeatures::InternalSwap(CppFeatures* PROTOBUF_RESTRICT other) { _internal_metadata_.InternalSwap(&other->_internal_metadata_); swap(_impl_._has_bits_[0], other->_impl_._has_bits_[0]); ::google::protobuf::internal::memswap< - PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_) - + sizeof(CppFeatures::_impl_.string_type_) - - PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.legacy_closed_enum_)>( - reinterpret_cast(&_impl_.legacy_closed_enum_), - reinterpret_cast(&other->_impl_.legacy_closed_enum_)); + PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.enum_name_uses_string_view_) + + sizeof(CppFeatures::_impl_.enum_name_uses_string_view_) + - PROTOBUF_FIELD_OFFSET(CppFeatures, _impl_.string_type_)>( + reinterpret_cast(&_impl_.string_type_), + reinterpret_cast(&other->_impl_.string_type_)); } ::google::protobuf::Metadata CppFeatures::GetMetadata() const { diff --git a/src/google/protobuf/cpp_features.pb.h b/src/google/protobuf/cpp_features.pb.h index 8bfb1478cb..3c09b51fea 100644 --- a/src/google/protobuf/cpp_features.pb.h +++ b/src/google/protobuf/cpp_features.pb.h @@ -261,9 +261,21 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message // accessors ------------------------------------------------------- enum : int { - kLegacyClosedEnumFieldNumber = 1, kStringTypeFieldNumber = 2, + kLegacyClosedEnumFieldNumber = 1, + kEnumNameUsesStringViewFieldNumber = 3, }; + // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { + bool has_string_type() const; + void clear_string_type() ; + ::pb::CppFeatures_StringType string_type() const; + void set_string_type(::pb::CppFeatures_StringType value); + + private: + ::pb::CppFeatures_StringType _internal_string_type() const; + void _internal_set_string_type(::pb::CppFeatures_StringType value); + + public: // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { bool has_legacy_closed_enum() const; void clear_legacy_closed_enum() ; @@ -275,15 +287,15 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message void _internal_set_legacy_closed_enum(bool value); public: - // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { - bool has_string_type() const; - void clear_string_type() ; - ::pb::CppFeatures_StringType string_type() const; - void set_string_type(::pb::CppFeatures_StringType value); + // optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { + bool has_enum_name_uses_string_view() const; + void clear_enum_name_uses_string_view() ; + bool enum_name_uses_string_view() const; + void set_enum_name_uses_string_view(bool value); private: - ::pb::CppFeatures_StringType _internal_string_type() const; - void _internal_set_string_type(::pb::CppFeatures_StringType value); + bool _internal_enum_name_uses_string_view() const; + void _internal_set_enum_name_uses_string_view(bool value); public: // @@protoc_insertion_point(class_scope:pb.CppFeatures) @@ -291,7 +303,7 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message class _Internal; friend class ::google::protobuf::internal::TcParser; static const ::google::protobuf::internal::TcParseTable< - 1, 2, 1, + 2, 3, 1, 0, 2> _table_; @@ -311,9 +323,10 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const CppFeatures& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; - bool legacy_closed_enum_; + ::google::protobuf::internal::CachedSize _cached_size_; int string_type_; + bool legacy_closed_enum_; + bool enum_name_uses_string_view_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -343,13 +356,13 @@ PROTOBUF_EXPORT extern ::google::protobuf::internal::ExtensionIdentifier< // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { inline bool CppFeatures::has_legacy_closed_enum() const { - bool value = (_impl_._has_bits_[0] & 0x00000001u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; return value; } inline void CppFeatures::clear_legacy_closed_enum() { ::google::protobuf::internal::TSanWrite(&_impl_); _impl_.legacy_closed_enum_ = false; - _impl_._has_bits_[0] &= ~0x00000001u; + _impl_._has_bits_[0] &= ~0x00000002u; } inline bool CppFeatures::legacy_closed_enum() const { // @@protoc_insertion_point(field_get:pb.CppFeatures.legacy_closed_enum) @@ -357,7 +370,7 @@ inline bool CppFeatures::legacy_closed_enum() const { } inline void CppFeatures::set_legacy_closed_enum(bool value) { _internal_set_legacy_closed_enum(value); - _impl_._has_bits_[0] |= 0x00000001u; + _impl_._has_bits_[0] |= 0x00000002u; // @@protoc_insertion_point(field_set:pb.CppFeatures.legacy_closed_enum) } inline bool CppFeatures::_internal_legacy_closed_enum() const { @@ -371,13 +384,13 @@ inline void CppFeatures::_internal_set_legacy_closed_enum(bool value) { // optional .pb.CppFeatures.StringType string_type = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { inline bool CppFeatures::has_string_type() const { - bool value = (_impl_._has_bits_[0] & 0x00000002u) != 0; + bool value = (_impl_._has_bits_[0] & 0x00000001u) != 0; return value; } inline void CppFeatures::clear_string_type() { ::google::protobuf::internal::TSanWrite(&_impl_); _impl_.string_type_ = 0; - _impl_._has_bits_[0] &= ~0x00000002u; + _impl_._has_bits_[0] &= ~0x00000001u; } inline ::pb::CppFeatures_StringType CppFeatures::string_type() const { // @@protoc_insertion_point(field_get:pb.CppFeatures.string_type) @@ -385,7 +398,7 @@ inline ::pb::CppFeatures_StringType CppFeatures::string_type() const { } inline void CppFeatures::set_string_type(::pb::CppFeatures_StringType value) { _internal_set_string_type(value); - _impl_._has_bits_[0] |= 0x00000002u; + _impl_._has_bits_[0] |= 0x00000001u; // @@protoc_insertion_point(field_set:pb.CppFeatures.string_type) } inline ::pb::CppFeatures_StringType CppFeatures::_internal_string_type() const { @@ -398,6 +411,34 @@ inline void CppFeatures::_internal_set_string_type(::pb::CppFeatures_StringType _impl_.string_type_ = value; } +// optional bool enum_name_uses_string_view = 3 [retention = RETENTION_SOURCE, targets = TARGET_TYPE_ENUM, targets = TARGET_TYPE_FILE, edition_defaults = { +inline bool CppFeatures::has_enum_name_uses_string_view() const { + bool value = (_impl_._has_bits_[0] & 0x00000004u) != 0; + return value; +} +inline void CppFeatures::clear_enum_name_uses_string_view() { + ::google::protobuf::internal::TSanWrite(&_impl_); + _impl_.enum_name_uses_string_view_ = false; + _impl_._has_bits_[0] &= ~0x00000004u; +} +inline bool CppFeatures::enum_name_uses_string_view() const { + // @@protoc_insertion_point(field_get:pb.CppFeatures.enum_name_uses_string_view) + return _internal_enum_name_uses_string_view(); +} +inline void CppFeatures::set_enum_name_uses_string_view(bool value) { + _internal_set_enum_name_uses_string_view(value); + _impl_._has_bits_[0] |= 0x00000004u; + // @@protoc_insertion_point(field_set:pb.CppFeatures.enum_name_uses_string_view) +} +inline bool CppFeatures::_internal_enum_name_uses_string_view() const { + ::google::protobuf::internal::TSanRead(&_impl_); + return _impl_.enum_name_uses_string_view_; +} +inline void CppFeatures::_internal_set_enum_name_uses_string_view(bool value) { + ::google::protobuf::internal::TSanWrite(&_impl_); + _impl_.enum_name_uses_string_view_ = value; +} + #ifdef __GNUC__ #pragma GCC diagnostic pop #endif // __GNUC__ diff --git a/src/google/protobuf/cpp_features.proto b/src/google/protobuf/cpp_features.proto index f3d670428b..99f1661303 100644 --- a/src/google/protobuf/cpp_features.proto +++ b/src/google/protobuf/cpp_features.proto @@ -53,4 +53,15 @@ message CppFeatures { edition_defaults = { edition: EDITION_LEGACY, value: "STRING" }, edition_defaults = { edition: EDITION_2024, value: "VIEW" } ]; + + optional bool enum_name_uses_string_view = 3 [ + retention = RETENTION_SOURCE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2024, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "false" }, + edition_defaults = { edition: EDITION_2024, value: "true" } + ]; } diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index ed5f6af920..62a2dd58cc 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3939,6 +3939,29 @@ bool FieldDescriptor::has_optional_keyword() const { is_optional() && !containing_oneof()); } +FieldDescriptor::CppStringType FieldDescriptor::cpp_string_type() const { + ABSL_DCHECK(cpp_type() == FieldDescriptor::CPPTYPE_STRING); + switch (features().GetExtension(pb::cpp).string_type()) { + case pb::CppFeatures::VIEW: + return CppStringType::kView; + case pb::CppFeatures::CORD: + // In open-source, protobuf CORD is only supported for singular bytes + // fields. + if (type() != FieldDescriptor::TYPE_BYTES || is_repeated() || + is_extension()) { + return CppStringType::kString; + } + return CppStringType::kCord; + case pb::CppFeatures::STRING: + return CppStringType::kString; + default: + // If features haven't been resolved, this is a dynamic build not for C++ + // codegen. Just use string type. + ABSL_DCHECK(!features().GetExtension(pb::cpp).has_string_type()); + return CppStringType::kString; + } +} + // Location methods =============================================== bool FileDescriptor::GetSourceLocation(const std::vector& path, @@ -5476,8 +5499,7 @@ static void EnforceCTypeStringTypeConsistency( Edition edition, FieldDescriptor::CppType type, const pb::CppFeatures& cpp_features, FieldOptions& options) { if (&options == &FieldOptions::default_instance()) return; - if (edition < Edition::EDITION_2024 && - type == FieldDescriptor::CPPTYPE_STRING) { + if (type == FieldDescriptor::CPPTYPE_STRING) { switch (cpp_features.string_type()) { case pb::CppFeatures::CORD: options.set_ctype(FieldOptions::CORD); @@ -6150,6 +6172,14 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( }); internal::VisitDescriptors(*result, [&](const FieldDescriptor& field) { + if (result->edition() >= Edition::EDITION_2024 && + field.options().has_ctype()) { + // "ctype" is no longer supported in edition 2024 and beyond. + AddError( + field.full_name(), proto, DescriptorPool::ErrorCollector::NAME, + "ctype option is not allowed under edition 2024 and beyond. Use " + "the feature string_type = VIEW|CORD|STRING|... instead."); + } EnforceCTypeStringTypeConsistency( field.file()->edition(), field.cpp_type(), field.merged_features_->GetExtension(pb::cpp), @@ -7904,12 +7934,7 @@ void DescriptorBuilder::ValidateOptions(const FieldDescriptor* field, auto& field_options = field->options(); if (field_options.has_ctype()) { - if (edition >= Edition::EDITION_2024) { - // "ctype" is no longer supported in edition 2024 and beyond. - AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, - "ctype option is not allowed under edition 2024 and beyond. Use " - "the feature string_type = VIEW|CORD|STRING|... instead."); - } else if (edition == Edition::EDITION_2023) { + if (edition == Edition::EDITION_2023) { if (field->cpp_type() != FieldDescriptor::CPPTYPE_STRING) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 28db34d95e..08a3d5c510 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -875,6 +875,10 @@ class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase, const char* cpp_type_name() const; // Name of the C++ type. Label label() const; // optional/required/repeated +#ifndef SWIG + CppStringType cpp_string_type() const; // The C++ string type of this field. +#endif + bool is_required() const; // shorthand for label() == LABEL_REQUIRED bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL bool is_repeated() const; // shorthand for label() == LABEL_REPEATED @@ -2932,22 +2936,21 @@ PROTOBUF_EXPORT bool HasPreservingUnknownEnumSemantics( PROTOBUF_EXPORT bool HasHasbit(const FieldDescriptor* field); +#ifndef SWIG // For a string field, returns the effective ctype. If the actual ctype is // not supported, returns the default of STRING. template typename FieldOpts::CType EffectiveStringCType(const FieldDesc* field) { - ABSL_DCHECK(field->cpp_type() == FieldDescriptor::CPPTYPE_STRING); - // Open-source protobuf release only supports STRING ctype and CORD for - // sinuglar bytes. - if (field->type() == FieldDescriptor::TYPE_BYTES && !field->is_repeated() && - field->options().ctype() == FieldOpts::CORD && !field->is_extension()) { - return FieldOpts::CORD; + // TODO Replace this function with FieldDescriptor::string_type; + switch (field->cpp_string_type()) { + case FieldDescriptor::CppStringType::kCord: + return FieldOpts::CORD; + default: + return FieldOpts::STRING; } - return FieldOpts::STRING; } -#ifndef SWIG enum class Utf8CheckMode : uint8_t { kStrict = 0, // Parsing will fail if non UTF-8 data is in string fields. kVerify = 1, // Only log an error but parsing will succeed. diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index bd9612cbea..63caf08785 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -3304,21 +3304,17 @@ PROTOBUF_NOINLINE void FileDescriptorProto::Clear() { // repeated int32 public_dependency = 10; { std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_public_dependency()) - ; + this_._internal_public_dependency()); std::size_t tag_size = std::size_t{1} * ::_pbi::FromIntSize(this_._internal_public_dependency_size()); - ; total_size += tag_size + data_size; } // repeated int32 weak_dependency = 11; { std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_weak_dependency()) - ; + this_._internal_weak_dependency()); std::size_t tag_size = std::size_t{1} * ::_pbi::FromIntSize(this_._internal_weak_dependency_size()); - ; total_size += tag_size + data_size; } } @@ -10112,15 +10108,11 @@ PROTOBUF_NOINLINE void FieldOptions::Clear() { { // repeated .google.protobuf.FieldOptions.OptionTargetType targets = 19; { - std::size_t data_size = 0; - auto count = static_cast(this_._internal_targets_size()); - - for (std::size_t i = 0; i < count; ++i) { - data_size += ::_pbi::WireFormatLite::EnumSize( - this_._internal_targets().Get(static_cast(i))); - } - total_size += data_size; - total_size += std::size_t{2} * count; + std::size_t data_size = + ::_pbi::WireFormatLite::EnumSize(this_._internal_targets()); + std::size_t tag_size = std::size_t{2} * + ::_pbi::FromIntSize(this_._internal_targets_size()); + total_size += data_size + tag_size; } // repeated .google.protobuf.FieldOptions.EditionDefault edition_defaults = 20; { @@ -10538,8 +10530,7 @@ PROTOBUF_NOINLINE void OneofOptions::Clear() { } { // optional .google.protobuf.FeatureSet features = 1; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.features_); @@ -14098,31 +14089,17 @@ PROTOBUF_NOINLINE void SourceCodeInfo_Location::Clear() { { // repeated int32 path = 1 [packed = true]; { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_path()) - ; - this_._impl_._path_cached_byte_size_.Set( - ::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + total_size += + ::_pbi::WireFormatLite::Int32SizeWithPackedTagSize( + this_._internal_path(), 1, + this_._impl_._path_cached_byte_size_); } // repeated int32 span = 2 [packed = true]; { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_span()) - ; - this_._impl_._span_cached_byte_size_.Set( - ::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + total_size += + ::_pbi::WireFormatLite::Int32SizeWithPackedTagSize( + this_._internal_span(), 1, + this_._impl_._span_cached_byte_size_); } // repeated string leading_detached_comments = 6; { @@ -14692,17 +14669,10 @@ PROTOBUF_NOINLINE void GeneratedCodeInfo_Annotation::Clear() { { // repeated int32 path = 1 [packed = true]; { - std::size_t data_size = ::_pbi::WireFormatLite::Int32Size( - this_._internal_path()) - ; - this_._impl_._path_cached_byte_size_.Set( - ::_pbi::ToCachedSize(data_size)); - std::size_t tag_size = data_size == 0 - ? 0 - : 1 + ::_pbi::WireFormatLite::Int32Size( - static_cast(data_size)) - ; - total_size += tag_size + data_size; + total_size += + ::_pbi::WireFormatLite::Int32SizeWithPackedTagSize( + this_._internal_path(), 1, + this_._impl_._path_cached_byte_size_); } } cached_has_bits = this_._impl_._has_bits_[0]; diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index a389d1d95a..6443a80622 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -903,7 +903,7 @@ class PROTOBUF_EXPORT UninterpretedOption_NamePart final : public ::google::prot ::google::protobuf::Arena* arena, const Impl_& from, const UninterpretedOption_NamePart& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_part_; bool is_extension_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -1177,11 +1177,11 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location final : public ::google::protobuf: ::google::protobuf::Arena* arena, const Impl_& from, const SourceCodeInfo_Location& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedField<::int32_t> path_; - mutable ::google::protobuf::internal::CachedSize _path_cached_byte_size_; + ::google::protobuf::internal::CachedSize _path_cached_byte_size_; ::google::protobuf::RepeatedField<::int32_t> span_; - mutable ::google::protobuf::internal::CachedSize _span_cached_byte_size_; + ::google::protobuf::internal::CachedSize _span_cached_byte_size_; ::google::protobuf::RepeatedPtrField leading_detached_comments_; ::google::protobuf::internal::ArenaStringPtr leading_comments_; ::google::protobuf::internal::ArenaStringPtr trailing_comments_; @@ -1452,9 +1452,9 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation final : public ::google::prot ::google::protobuf::Arena* arena, const Impl_& from, const GeneratedCodeInfo_Annotation& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedField<::int32_t> path_; - mutable ::google::protobuf::internal::CachedSize _path_cached_byte_size_; + ::google::protobuf::internal::CachedSize _path_cached_byte_size_; ::google::protobuf::internal::ArenaStringPtr source_file_; ::int32_t begin_; ::int32_t end_; @@ -1687,7 +1687,7 @@ class PROTOBUF_EXPORT FieldOptions_FeatureSupport final : public ::google::proto ::google::protobuf::Arena* arena, const Impl_& from, const FieldOptions_FeatureSupport& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr deprecation_warning_; int edition_introduced_; int edition_deprecated_; @@ -1896,7 +1896,7 @@ class PROTOBUF_EXPORT FieldOptions_EditionDefault final : public ::google::proto ::google::protobuf::Arena* arena, const Impl_& from, const FieldOptions_EditionDefault& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr value_; int edition_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -2451,7 +2451,7 @@ class PROTOBUF_EXPORT FeatureSet final : public ::google::protobuf::Message const FeatureSet& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; int field_presence_; int enum_type_; int repeated_field_encoding_; @@ -2704,7 +2704,7 @@ class PROTOBUF_EXPORT ExtensionRangeOptions_Declaration final : public ::google: ::google::protobuf::Arena* arena, const Impl_& from, const ExtensionRangeOptions_Declaration& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr full_name_; ::google::protobuf::internal::ArenaStringPtr type_; ::int32_t number_; @@ -2908,7 +2908,7 @@ class PROTOBUF_EXPORT EnumDescriptorProto_EnumReservedRange final : public ::goo ::google::protobuf::Arena* arena, const Impl_& from, const EnumDescriptorProto_EnumReservedRange& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::int32_t start_; ::int32_t end_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -3109,7 +3109,7 @@ class PROTOBUF_EXPORT DescriptorProto_ReservedRange final : public ::google::pro ::google::protobuf::Arena* arena, const Impl_& from, const DescriptorProto_ReservedRange& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::int32_t start_; ::int32_t end_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -3400,7 +3400,7 @@ class PROTOBUF_EXPORT UninterpretedOption final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const UninterpretedOption& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption_NamePart > name_; ::google::protobuf::internal::ArenaStringPtr identifier_value_; ::google::protobuf::internal::ArenaStringPtr string_value_; @@ -3601,7 +3601,7 @@ class PROTOBUF_EXPORT SourceCodeInfo final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const SourceCodeInfo& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::SourceCodeInfo_Location > location_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -3795,7 +3795,7 @@ class PROTOBUF_EXPORT GeneratedCodeInfo final : public ::google::protobuf::Messa ::google::protobuf::Arena* arena, const Impl_& from, const GeneratedCodeInfo& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::GeneratedCodeInfo_Annotation > annotation_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -4019,7 +4019,7 @@ class PROTOBUF_EXPORT FeatureSetDefaults_FeatureSetEditionDefault final : public ::google::protobuf::Arena* arena, const Impl_& from, const FeatureSetDefaults_FeatureSetEditionDefault& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::FeatureSet* overridable_features_; ::google::protobuf::FeatureSet* fixed_features_; int edition_; @@ -4428,7 +4428,7 @@ class PROTOBUF_EXPORT ServiceOptions final : public ::google::protobuf::Message const ServiceOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool deprecated_; @@ -4825,7 +4825,7 @@ class PROTOBUF_EXPORT OneofOptions final : public ::google::protobuf::Message const OneofOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -5265,7 +5265,7 @@ class PROTOBUF_EXPORT MethodOptions final : public ::google::protobuf::Message const MethodOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool deprecated_; @@ -5723,7 +5723,7 @@ class PROTOBUF_EXPORT MessageOptions final : public ::google::protobuf::Message const MessageOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool message_set_wire_format_; @@ -6432,7 +6432,7 @@ class PROTOBUF_EXPORT FileOptions final : public ::google::protobuf::Message const FileOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::internal::ArenaStringPtr java_package_; ::google::protobuf::internal::ArenaStringPtr java_outer_classname_; @@ -7098,7 +7098,7 @@ class PROTOBUF_EXPORT FieldOptions final : public ::google::protobuf::Message const FieldOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedField targets_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FieldOptions_EditionDefault > edition_defaults_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; @@ -7335,7 +7335,7 @@ class PROTOBUF_EXPORT FeatureSetDefaults final : public ::google::protobuf::Mess ::google::protobuf::Arena* arena, const Impl_& from, const FeatureSetDefaults& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FeatureSetDefaults_FeatureSetEditionDefault > defaults_; int minimum_edition_; int maximum_edition_; @@ -7782,7 +7782,7 @@ class PROTOBUF_EXPORT ExtensionRangeOptions final : public ::google::protobuf::M const ExtensionRangeOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::ExtensionRangeOptions_Declaration > declaration_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; @@ -8220,7 +8220,7 @@ class PROTOBUF_EXPORT EnumValueOptions final : public ::google::protobuf::Messag const EnumValueOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; ::google::protobuf::FieldOptions_FeatureSupport* feature_support_; @@ -8655,7 +8655,7 @@ class PROTOBUF_EXPORT EnumOptions final : public ::google::protobuf::Message const EnumOptions& from_msg); ::google::protobuf::internal::ExtensionSet _extensions_; ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::UninterpretedOption > uninterpreted_option_; ::google::protobuf::FeatureSet* features_; bool allow_alias_; @@ -8874,7 +8874,7 @@ class PROTOBUF_EXPORT OneofDescriptorProto final : public ::google::protobuf::Me ::google::protobuf::Arena* arena, const Impl_& from, const OneofDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::OneofOptions* options_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -9150,7 +9150,7 @@ class PROTOBUF_EXPORT MethodDescriptorProto final : public ::google::protobuf::M ::google::protobuf::Arena* arena, const Impl_& from, const MethodDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr input_type_; ::google::protobuf::internal::ArenaStringPtr output_type_; @@ -9557,7 +9557,7 @@ class PROTOBUF_EXPORT FieldDescriptorProto final : public ::google::protobuf::Me ::google::protobuf::Arena* arena, const Impl_& from, const FieldDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::internal::ArenaStringPtr extendee_; ::google::protobuf::internal::ArenaStringPtr type_name_; @@ -9794,7 +9794,7 @@ class PROTOBUF_EXPORT EnumValueDescriptorProto final : public ::google::protobuf ::google::protobuf::Arena* arena, const Impl_& from, const EnumValueDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::EnumValueOptions* options_; ::int32_t number_; @@ -10017,7 +10017,7 @@ class PROTOBUF_EXPORT DescriptorProto_ExtensionRange final : public ::google::pr ::google::protobuf::Arena* arena, const Impl_& from, const DescriptorProto_ExtensionRange& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::ExtensionRangeOptions* options_; ::int32_t start_; ::int32_t end_; @@ -10252,7 +10252,7 @@ class PROTOBUF_EXPORT ServiceDescriptorProto final : public ::google::protobuf:: ::google::protobuf::Arena* arena, const Impl_& from, const ServiceDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::MethodDescriptorProto > method_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::ServiceOptions* options_; @@ -10529,7 +10529,7 @@ class PROTOBUF_EXPORT EnumDescriptorProto final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const EnumDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumValueDescriptorProto > value_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumDescriptorProto_EnumReservedRange > reserved_range_; ::google::protobuf::RepeatedPtrField reserved_name_; @@ -10899,7 +10899,7 @@ class PROTOBUF_EXPORT DescriptorProto final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const DescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::FieldDescriptorProto > field_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::DescriptorProto > nested_type_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumDescriptorProto > enum_type_; @@ -11320,7 +11320,7 @@ class PROTOBUF_EXPORT FileDescriptorProto final : public ::google::protobuf::Mes ::google::protobuf::Arena* arena, const Impl_& from, const FileDescriptorProto& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField dependency_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::DescriptorProto > message_type_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumDescriptorProto > enum_type_; @@ -11531,7 +11531,7 @@ class PROTOBUF_EXPORT FileDescriptorSet final : public ::google::protobuf::Messa ::google::protobuf::Arena* arena, const Impl_& from, const FileDescriptorSet& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::FileDescriptorProto > file_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/descriptor_lite.h b/src/google/protobuf/descriptor_lite.h index db5805affe..e1a90b7fa6 100644 --- a/src/google/protobuf/descriptor_lite.h +++ b/src/google/protobuf/descriptor_lite.h @@ -78,6 +78,17 @@ class FieldDescriptorLite { MAX_LABEL = 3, // Constant useful for defining lookup tables // indexed by Label. }; + + // Identifies the storage type of a C++ string field. This corresponds to + // pb.CppFeatures.StringType, but is compatible with ctype prior to Edition + // 2024. 0 is reserved for errors. +#ifndef SWIG + enum class CppStringType { + kView = 1, + kCord = 2, + kString = 3, + }; +#endif }; } // namespace internal diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 0c13b3a2b4..e3360e171d 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -7546,33 +7546,42 @@ TEST_F(FeaturesTest, Proto2Features) { EXPECT_THAT(file->options(), EqualsProto("")); EXPECT_EQ(GetFeatures(file).GetExtension(pb::test).file_feature(), pb::VALUE1); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: CLOSED - repeated_field_encoding: EXPANDED - utf8_validation: NONE - message_encoding: LENGTH_PREFIXED - json_format: LEGACY_BEST_EFFORT - [pb.cpp] { legacy_closed_enum: true string_type: STRING })pb")); - EXPECT_THAT( - GetCoreFeatures(field), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: CLOSED - repeated_field_encoding: EXPANDED - utf8_validation: NONE - message_encoding: LENGTH_PREFIXED - json_format: LEGACY_BEST_EFFORT - [pb.cpp] { legacy_closed_enum: true string_type: STRING })pb")); - EXPECT_THAT( - GetCoreFeatures(group), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: CLOSED - repeated_field_encoding: EXPANDED - utf8_validation: NONE - message_encoding: DELIMITED - json_format: LEGACY_BEST_EFFORT - [pb.cpp] { legacy_closed_enum: true string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: CLOSED + repeated_field_encoding: EXPANDED + utf8_validation: NONE + message_encoding: LENGTH_PREFIXED + json_format: LEGACY_BEST_EFFORT + [pb.cpp] { + legacy_closed_enum: true + string_type: STRING + enum_name_uses_string_view: false + })pb")); + EXPECT_THAT(GetCoreFeatures(field), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: CLOSED + repeated_field_encoding: EXPANDED + utf8_validation: NONE + message_encoding: LENGTH_PREFIXED + json_format: LEGACY_BEST_EFFORT + [pb.cpp] { + legacy_closed_enum: true + string_type: STRING + enum_name_uses_string_view: false + })pb")); + EXPECT_THAT(GetCoreFeatures(group), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: CLOSED + repeated_field_encoding: EXPANDED + utf8_validation: NONE + message_encoding: DELIMITED + json_format: LEGACY_BEST_EFFORT + [pb.cpp] { + legacy_closed_enum: true + string_type: STRING + enum_name_uses_string_view: false + })pb")); EXPECT_TRUE(field->has_presence()); EXPECT_FALSE(field->requires_utf8_validation()); EXPECT_EQ( @@ -7594,6 +7603,9 @@ TEST_F(FeaturesTest, Proto2Features) { EXPECT_TRUE(message->FindFieldByName("req")->is_required()); EXPECT_TRUE(file->enum_type(0)->is_closed()); + EXPECT_EQ(message->FindFieldByName("str")->cpp_string_type(), + FieldDescriptor::CppStringType::kString); + // Check round-trip consistency. FileDescriptorProto proto; file->CopyTo(&proto); @@ -7633,24 +7645,30 @@ TEST_F(FeaturesTest, Proto3Features) { EXPECT_THAT(file->options(), EqualsProto("")); EXPECT_EQ(GetFeatures(file).GetExtension(pb::test).file_feature(), pb::VALUE2); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); - EXPECT_THAT( - GetCoreFeatures(field), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); + EXPECT_THAT(GetCoreFeatures(field), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); EXPECT_FALSE(field->has_presence()); EXPECT_FALSE(field->requires_utf8_validation()); EXPECT_EQ( @@ -7826,7 +7844,11 @@ TEST_F(FeaturesTest, Edition2023Defaults) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING } + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + } )pb")); // Since pb::test is registered in the pool, it should end up with defaults in @@ -7905,7 +7927,11 @@ TEST_F(FeaturesTest, Edition2024Defaults) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: VIEW } + [pb.cpp] { + legacy_closed_enum: false + string_type: VIEW + enum_name_uses_string_view: true + } )pb")); // Since pb::test is registered in the pool, it should end up with defaults in @@ -7934,7 +7960,11 @@ TEST_F(FeaturesBaseTest, DefaultEdition2023Defaults) { utf8_validation: VERIFY message_encoding: LENGTH_PREFIXED json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING } + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + } )pb")); EXPECT_FALSE(GetFeatures(file).HasExtension(pb::test)); } @@ -7951,15 +7981,18 @@ TEST_F(FeaturesTest, ClearsOptions) { } )pb"); EXPECT_THAT(file->options(), EqualsProto("java_package: 'bar'")); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, RestoresOptionsRoundTrip) { @@ -8316,15 +8349,18 @@ TEST_F(FeaturesTest, NoOptions) { name: "foo.proto" syntax: "editions" edition: EDITION_2023 )pb"); EXPECT_EQ(&file->options(), &FileOptions::default_instance()); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, InvalidEdition) { @@ -8346,15 +8382,18 @@ TEST_F(FeaturesTest, FileFeatures) { options { features { field_presence: IMPLICIT } } )pb"); EXPECT_THAT(file->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, FileFeaturesExtension) { @@ -8424,15 +8463,18 @@ TEST_F(FeaturesTest, MessageFeaturesDefault) { )pb"); const Descriptor* message = file->message_type(0); EXPECT_THAT(message->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(message), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(message), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, MessageFeaturesInherit) { @@ -8532,15 +8574,18 @@ TEST_F(FeaturesTest, FieldFeaturesDefault) { )pb"); const FieldDescriptor* field = file->message_type(0)->field(0); EXPECT_THAT(field->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(field), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(field), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, FieldFeaturesInherit) { @@ -9003,15 +9048,18 @@ TEST_F(FeaturesTest, EnumFeaturesDefault) { )pb"); const EnumDescriptor* enm = file->enum_type(0); EXPECT_THAT(enm->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(enm), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(enm), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, EnumFeaturesInherit) { @@ -9115,15 +9163,18 @@ TEST_F(FeaturesTest, EnumValueFeaturesDefault) { )pb"); const EnumValueDescriptor* value = file->enum_type(0)->value(0); EXPECT_THAT(value->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(value), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(value), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, EnumValueFeaturesInherit) { @@ -9211,15 +9262,18 @@ TEST_F(FeaturesTest, OneofFeaturesDefault) { )pb"); const OneofDescriptor* oneof = file->message_type(0)->oneof_decl(0); EXPECT_THAT(oneof->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(oneof), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(oneof), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, OneofFeaturesInherit) { @@ -9316,15 +9370,18 @@ TEST_F(FeaturesTest, ExtensionRangeFeaturesDefault) { const Descriptor::ExtensionRange* range = file->message_type(0)->extension_range(0); EXPECT_THAT(range->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(range), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(range), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, ExtensionRangeFeaturesInherit) { @@ -9406,15 +9463,18 @@ TEST_F(FeaturesTest, ServiceFeaturesDefault) { )pb"); const ServiceDescriptor* service = file->service(0); EXPECT_THAT(service->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(service), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(service), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, ServiceFeaturesInherit) { @@ -9473,15 +9533,18 @@ TEST_F(FeaturesTest, MethodFeaturesDefault) { )pb"); const MethodDescriptor* method = file->service(0)->method(0); EXPECT_THAT(method->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(method), EqualsProto(R"pb( - field_presence: EXPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(method), EqualsProto(R"pb( + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, MethodFeaturesInherit) { @@ -9710,6 +9773,73 @@ TEST_F(FeaturesTest, EnumFeatureHelpers) { EXPECT_FALSE(HasPreservingUnknownEnumSemantics(field_legacy_closed)); } +TEST_F(FeaturesTest, FieldCppStringType) { + BuildDescriptorMessagesInTestPool(); + const std::string file_contents = absl::Substitute( + R"pb( + name: "foo.proto" + syntax: "editions" + edition: EDITION_2024 + message_type { + name: "Foo" + field { + name: "view" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + } + field { + name: "str" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_STRING + options { + features { + [pb.cpp] { string_type: STRING } + } + } + } + field { + name: "cord" + number: 3 + label: LABEL_OPTIONAL + type: TYPE_STRING + options { + features { + [pb.cpp] { string_type: CORD } + } + } + } + field { + name: "cord_bytes" + number: 4 + label: LABEL_OPTIONAL + type: TYPE_BYTES + options { + features { + [pb.cpp] { string_type: CORD } + } + } + } $0 + } + )pb", + "" + ); + const FileDescriptor* file = BuildFile(file_contents); + const Descriptor* message = file->message_type(0); + const FieldDescriptor* view = message->field(0); + const FieldDescriptor* str = message->field(1); + const FieldDescriptor* cord = message->field(2); + const FieldDescriptor* cord_bytes = message->field(3); + + EXPECT_EQ(view->cpp_string_type(), FieldDescriptor::CppStringType::kView); + EXPECT_EQ(str->cpp_string_type(), FieldDescriptor::CppStringType::kString); + EXPECT_EQ(cord_bytes->cpp_string_type(), + FieldDescriptor::CppStringType::kCord); + EXPECT_EQ(cord->cpp_string_type(), FieldDescriptor::CppStringType::kString); + +} + TEST_F(FeaturesTest, MergeFeatureValidationFailed) { BuildDescriptorMessagesInTestPool(); BuildFileInTestPool(pb::TestFeatures::descriptor()->file()); @@ -10484,15 +10614,18 @@ TEST_F(FeaturesTest, UninterpretedOptions) { } )pb"); EXPECT_THAT(file->options(), EqualsProto("")); - EXPECT_THAT( - GetCoreFeatures(file), EqualsProto(R"pb( - field_presence: IMPLICIT - enum_type: OPEN - repeated_field_encoding: PACKED - utf8_validation: VERIFY - message_encoding: LENGTH_PREFIXED - json_format: ALLOW - [pb.cpp] { legacy_closed_enum: false string_type: STRING })pb")); + EXPECT_THAT(GetCoreFeatures(file), EqualsProto(R"pb( + field_presence: IMPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + utf8_validation: VERIFY + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + [pb.cpp] { + legacy_closed_enum: false + string_type: STRING + enum_name_uses_string_view: false + })pb")); } TEST_F(FeaturesTest, UninterpretedOptionsMerge) { diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 92e931a371..b1414b3f64 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -255,7 +255,7 @@ class PROTOBUF_EXPORT Duration final : public ::google::protobuf::Message const Duration& from_msg); ::int64_t seconds_; ::int32_t nanos_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 00b55f6031..138c941282 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -246,7 +246,7 @@ class DynamicMessage final : public Message { void* MutableOneofFieldRaw(const FieldDescriptor* f); const DynamicMessageFactory::TypeInfo* type_info_; - mutable internal::CachedSize cached_byte_size_; + internal::CachedSize cached_byte_size_; }; struct DynamicMessageFactory::TypeInfo { diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index 21f24037a9..dee8e7795d 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -1358,7 +1358,7 @@ size_t ExtensionSet::Extension::ByteSize(int number) const { break; } - cached_size = ToCachedSize(result); + cached_size.set(ToCachedSize(result)); if (result > 0) { result += io::CodedOutputStream::VarintSize32(result); result += io::CodedOutputStream::VarintSize32(WireFormatLite::MakeTag( @@ -1678,12 +1678,12 @@ uint8_t* ExtensionSet::Extension::InternalSerializeFieldWithCachedSizesToArray( uint8_t* target, io::EpsCopyOutputStream* stream) const { if (is_repeated) { if (is_packed) { - if (cached_size == 0) return target; + if (cached_size() == 0) return target; target = stream->EnsureSpace(target); target = WireFormatLite::WriteTagToArray( number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, target); - target = WireFormatLite::WriteInt32NoTagToArray(cached_size, target); + target = WireFormatLite::WriteInt32NoTagToArray(cached_size(), target); switch (real_type(type)) { #define HANDLE_TYPE(UPPERCASE, CAMELCASE, LOWERCASE) \ diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 8b95b72249..2d8e753029 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -632,7 +632,41 @@ class PROTOBUF_EXPORT ExtensionSet { } static std::atomic maybe_create_lazy_extension_; + + // We can't directly use std::atomic for Extension::cached_size because + // Extension needs to be trivially copyable. + class TrivialAtomicInt { + public: + int operator()() const { + return reinterpret_cast(int_)->load( + std::memory_order_relaxed); + } + void set(int v) { + reinterpret_cast(int_)->store(v, std::memory_order_relaxed); + } + + private: + using AtomicT = std::atomic; + alignas(AtomicT) char int_[sizeof(AtomicT)]; + }; + struct Extension { + // Some helper methods for operations on a single Extension. + uint8_t* InternalSerializeFieldWithCachedSizesToArray( + const MessageLite* extendee, const ExtensionSet* extension_set, + int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; + uint8_t* InternalSerializeMessageSetItemWithCachedSizesToArray( + const MessageLite* extendee, const ExtensionSet* extension_set, + int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; + size_t ByteSize(int number) const; + size_t MessageSetItemByteSize(int number) const; + void Clear(); + int GetSize() const; + void Free(); + size_t SpaceUsedExcludingSelfLong() const; + bool IsInitialized(const ExtensionSet* ext_set, const MessageLite* extendee, + int number, Arena* arena) const; + // The order of these fields packs Extension into 24 bytes when using 8 // byte alignment. Consider this when adding or removing fields here. union { @@ -683,29 +717,12 @@ class PROTOBUF_EXPORT ExtensionSet { // For packed fields, the size of the packed data is recorded here when // ByteSize() is called then used during serialization. - // TODO: Use atomic when C++ supports it. - mutable int cached_size; + mutable TrivialAtomicInt cached_size; // The descriptor for this extension, if one exists and is known. May be // nullptr. Must not be nullptr if the descriptor for the extension does // not live in the same pool as the descriptor for the containing type. const FieldDescriptor* descriptor; - - // Some helper methods for operations on a single Extension. - uint8_t* InternalSerializeFieldWithCachedSizesToArray( - const MessageLite* extendee, const ExtensionSet* extension_set, - int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; - uint8_t* InternalSerializeMessageSetItemWithCachedSizesToArray( - const MessageLite* extendee, const ExtensionSet* extension_set, - int number, uint8_t* target, io::EpsCopyOutputStream* stream) const; - size_t ByteSize(int number) const; - size_t MessageSetItemByteSize(int number) const; - void Clear(); - int GetSize() const; - void Free(); - size_t SpaceUsedExcludingSelfLong() const; - bool IsInitialized(const ExtensionSet* ext_set, const MessageLite* extendee, - int number, Arena* arena) const; }; // The Extension struct is small enough to be passed by value, so we use it diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 3b7114551f..b6f7e493c4 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -255,7 +255,7 @@ class PROTOBUF_EXPORT FieldMask final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const FieldMask& from_msg); ::google::protobuf::RepeatedPtrField paths_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/generated_message_bases.h b/src/google/protobuf/generated_message_bases.h index 3dc5991d0c..48d94db6c4 100644 --- a/src/google/protobuf/generated_message_bases.h +++ b/src/google/protobuf/generated_message_bases.h @@ -51,7 +51,7 @@ class PROTOBUF_EXPORT ZeroFieldsBase : public Message { io::EpsCopyOutputStream* stream); struct { - mutable internal::CachedSize _cached_size_; + internal::CachedSize _cached_size_; } _impl_; }; diff --git a/src/google/protobuf/implicit_weak_message.h b/src/google/protobuf/implicit_weak_message.h index 457d34db1f..095028a6be 100644 --- a/src/google/protobuf/implicit_weak_message.h +++ b/src/google/protobuf/implicit_weak_message.h @@ -110,7 +110,7 @@ class PROTOBUF_EXPORT ImplicitWeakMessage final : public MessageLite { // the default instance can be constant-initialized. In the const methods, we // have to handle the possibility of data_ being null. std::string* data_; - mutable google::protobuf::internal::CachedSize cached_size_{}; + google::protobuf::internal::CachedSize cached_size_{}; }; struct ImplicitWeakMessageDefaultType; diff --git a/src/google/protobuf/inlined_string_field.h b/src/google/protobuf/inlined_string_field.h index d894acb7ec..25c7a7a573 100644 --- a/src/google/protobuf/inlined_string_field.h +++ b/src/google/protobuf/inlined_string_field.h @@ -85,16 +85,28 @@ namespace internal { // For more details of the donating states transitions, go/pd-inlined-string. class PROTOBUF_EXPORT InlinedStringField { public: - InlinedStringField() { Init(); } + InlinedStringField() : str_() {} InlinedStringField(const InlinedStringField&) = delete; InlinedStringField& operator=(const InlinedStringField&) = delete; - inline void Init() { new (get_mutable()) std::string(); } +#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L + // No need to do dynamic initialization here. + constexpr void Init() {} // Add the dummy parameter just to make InlinedStringField(nullptr) // unambiguous. constexpr InlinedStringField( const ExplicitlyConstructed* /*default_value*/, bool /*dummy*/) - : value_{} {} + : str_{} {} +#else + inline void Init() { ::new (static_cast(&str_)) std::string(); } + // Add the dummy parameter just to make InlinedStringField(nullptr) + // unambiguous. + constexpr InlinedStringField( + const ExplicitlyConstructed* /*default_value*/, + bool /*dummy*/) + : dummy_{} {} +#endif + explicit InlinedStringField(const std::string& default_value); explicit InlinedStringField(Arena* arena); InlinedStringField(Arena* arena, const InlinedStringField& rhs); @@ -346,7 +358,10 @@ class PROTOBUF_EXPORT InlinedStringField { PROTOBUF_NDEBUG_INLINE std::string* get_mutable(); PROTOBUF_NDEBUG_INLINE const std::string* get_const() const; - alignas(std::string) char value_[sizeof(std::string)]; + union { + std::string str_; + char dummy_; + }; std::string* MutableSlow(::google::protobuf::Arena* arena, bool donated, uint32_t* donating_states, uint32_t mask, @@ -359,12 +374,10 @@ class PROTOBUF_EXPORT InlinedStringField { typedef void DestructorSkippable_; }; -inline std::string* InlinedStringField::get_mutable() { - return reinterpret_cast(&value_); -} +inline std::string* InlinedStringField::get_mutable() { return &str_; } inline const std::string* InlinedStringField::get_const() const { - return reinterpret_cast(&value_); + return &str_; } inline InlinedStringField::InlinedStringField( @@ -386,12 +399,12 @@ inline void InternalRegisterArenaDtor(Arena* arena, void* object, } #endif // GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL_INLINE -inline InlinedStringField::InlinedStringField(Arena* /*arena*/) { Init(); } +inline InlinedStringField::InlinedStringField(Arena* /*arena*/) : str_() {} inline InlinedStringField::InlinedStringField(Arena* arena, const InlinedStringField& rhs) { const std::string& src = *rhs.get_const(); - new (value_) std::string(src); + ::new (static_cast(&str_)) std::string(src); } inline const std::string& InlinedStringField::GetNoArena() const { diff --git a/src/google/protobuf/internal_visibility.h b/src/google/protobuf/internal_visibility.h index fed45bd944..fd9bfc4115 100644 --- a/src/google/protobuf/internal_visibility.h +++ b/src/google/protobuf/internal_visibility.h @@ -17,6 +17,7 @@ class MessageLite; namespace internal { class InternalVisibilityForTesting; +class InternalMetadata; // Empty class to use as a mandatory 'internal token' for functions that have to // be public, such as arena constructors, but that are for internal use only. @@ -29,6 +30,7 @@ class InternalVisibility { friend class ::google::protobuf::Arena; friend class ::google::protobuf::Message; friend class ::google::protobuf::MessageLite; + friend class ::google::protobuf::internal::InternalMetadata; friend class InternalVisibilityForTesting; }; diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel index 096ebf71b8..b22c677358 100644 --- a/src/google/protobuf/io/BUILD.bazel +++ b/src/google/protobuf/io/BUILD.bazel @@ -27,6 +27,7 @@ cc_library( deps = [ ":io_win32", "//src/google/protobuf:arena", + "//src/google/protobuf:endian", "//src/google/protobuf:port", "//src/google/protobuf/stubs:lite", "@com_google_absl//absl/base", @@ -208,6 +209,7 @@ cc_test( "//src/google/protobuf/testing", "//src/google/protobuf/testing:file", "@com_google_absl//absl/base", + "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:log_severity", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc index a9297e116f..23a692a0bc 100644 --- a/src/google/protobuf/io/coded_stream.cc +++ b/src/google/protobuf/io/coded_stream.cc @@ -20,9 +20,13 @@ #include #include +#include #include #include #include +#include +#include +#include #include #include "absl/log/absl_check.h" @@ -33,7 +37,6 @@ #include "google/protobuf/arena.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" -#include "google/protobuf/port.h" // Must be included last. diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 4a5ca80cf8..3a40dedd80 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -111,9 +111,9 @@ #include "absl/numeric/bits.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "google/protobuf/endian.h" #include "google/protobuf/port.h" - // Must be included last. #include "google/protobuf/port_def.inc" @@ -1324,37 +1324,16 @@ inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) { // static inline const uint8_t* CodedInputStream::ReadLittleEndian32FromArray( const uint8_t* buffer, uint32_t* value) { -#if defined(ABSL_IS_LITTLE_ENDIAN) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(value, buffer, sizeof(*value)); + *value = google::protobuf::internal::little_endian::ToHost(*value); return buffer + sizeof(*value); -#else - *value = (static_cast(buffer[0])) | - (static_cast(buffer[1]) << 8) | - (static_cast(buffer[2]) << 16) | - (static_cast(buffer[3]) << 24); - return buffer + sizeof(*value); -#endif } // static inline const uint8_t* CodedInputStream::ReadLittleEndian64FromArray( const uint8_t* buffer, uint64_t* value) { -#if defined(ABSL_IS_LITTLE_ENDIAN) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(value, buffer, sizeof(*value)); + *value = google::protobuf::internal::little_endian::ToHost(*value); return buffer + sizeof(*value); -#else - uint32_t part0 = (static_cast(buffer[0])) | - (static_cast(buffer[1]) << 8) | - (static_cast(buffer[2]) << 16) | - (static_cast(buffer[3]) << 24); - uint32_t part1 = (static_cast(buffer[4])) | - (static_cast(buffer[5]) << 8) | - (static_cast(buffer[6]) << 16) | - (static_cast(buffer[7]) << 24); - *value = static_cast(part0) | (static_cast(part1) << 32); - return buffer + sizeof(*value); -#endif } inline bool CodedInputStream::ReadLittleEndian32(uint32_t* value) { diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc index 32c6594efa..295bcbf1b9 100644 --- a/src/google/protobuf/io/coded_stream_unittest.cc +++ b/src/google/protobuf/io/coded_stream_unittest.cc @@ -16,20 +16,29 @@ #include #include +#include +#include +#include +#include +#include #include +#include #include +#include +#include #include -#include "google/protobuf/stubs/common.h" +#include #include #include "absl/base/casts.h" #include "absl/base/log_severity.h" -#include "absl/log/absl_check.h" +#include "absl/base/macros.h" #include "absl/log/absl_log.h" #include "absl/log/scoped_mock_log.h" #include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/testing/googletest.h" @@ -42,72 +51,6 @@ namespace protobuf { namespace io { namespace { -// =================================================================== -// Data-Driven Test Infrastructure - -// TEST_1D and TEST_2D are macros I'd eventually like to see added to -// gTest. These macros can be used to declare tests which should be -// run multiple times, once for each item in some input array. TEST_1D -// tests all cases in a single input array. TEST_2D tests all -// combinations of cases from two arrays. The arrays must be statically -// defined such that the ABSL_ARRAYSIZE() macro works on them. Example: -// -// int kCases[] = {1, 2, 3, 4} -// TEST_1D(MyFixture, MyTest, kCases) { -// EXPECT_GT(kCases_case, 0); -// } -// -// This test iterates through the numbers 1, 2, 3, and 4 and tests that -// they are all grater than zero. In case of failure, the exact case -// which failed will be printed. The case type must be printable using -// ostream::operator<<. - -// TODO: gTest now supports "parameterized tests" which would be -// a better way to accomplish this. Rewrite when time permits. - -#define TEST_1D(FIXTURE, NAME, CASES) \ - class FIXTURE##_##NAME##_DD : public FIXTURE { \ - protected: \ - template \ - void DoSingleCase(const CaseType& CASES##_case); \ - }; \ - \ - TEST_F(FIXTURE##_##NAME##_DD, NAME) { \ - for (size_t i = 0; i < ABSL_ARRAYSIZE(CASES); i++) { \ - SCOPED_TRACE(testing::Message() \ - << #CASES " case #" << i << ": " << CASES[i]); \ - DoSingleCase(CASES[i]); \ - } \ - } \ - \ - template \ - void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType& CASES##_case) - -#define TEST_2D(FIXTURE, NAME, CASES1, CASES2) \ - class FIXTURE##_##NAME##_DD : public FIXTURE { \ - protected: \ - template \ - void DoSingleCase(const CaseType1& CASES1##_case, \ - const CaseType2& CASES2##_case); \ - }; \ - \ - TEST_F(FIXTURE##_##NAME##_DD, NAME) { \ - for (size_t i = 0; i < ABSL_ARRAYSIZE(CASES1); i++) { \ - for (size_t j = 0; j < ABSL_ARRAYSIZE(CASES2); j++) { \ - SCOPED_TRACE(testing::Message() \ - << #CASES1 " case #" << i << ": " << CASES1[i] << ", " \ - << #CASES2 " case #" << j << ": " << CASES2[j]); \ - DoSingleCase(CASES1[i], CASES2[j]); \ - } \ - } \ - } \ - \ - template \ - void FIXTURE##_##NAME##_DD::DoSingleCase(const CaseType1& CASES1##_case, \ - const CaseType2& CASES2##_case) - -// =================================================================== - class CodedStreamTest : public testing::Test { protected: // Buffer used during most of the tests. This assumes tests run sequentially. @@ -171,7 +114,13 @@ VarintCase kVarintCases[] = { (uint64_t{0x26u} << 56) | (uint64_t{0x01u} << 63)}, }; -TEST_2D(CodedStreamTest, ReadVarint32, kVarintCases, kBlockSizes) { +class VarintCasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +TEST_P(VarintCasesWithSizes, ReadVarint32) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -186,7 +135,9 @@ TEST_2D(CodedStreamTest, ReadVarint32, kVarintCases, kBlockSizes) { EXPECT_EQ(kVarintCases_case.size, input.ByteCount()); } -TEST_2D(CodedStreamTest, ReadTag, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, ReadTag) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -231,7 +182,11 @@ TEST_F(CodedStreamTest, EmptyInputBeforeEos) { EXPECT_TRUE(input.ConsumedEntireMessage()); } -TEST_1D(CodedStreamTest, ExpectTag, kVarintCases) { +class VarintCases : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(VarintCases, ExpectTag) { + const VarintCase& kVarintCases_case = GetParam(); // Leave one byte at the beginning of the buffer so we can read it // to force the first buffer to be loaded. buffer_[0] = '\0'; @@ -265,7 +220,8 @@ TEST_1D(CodedStreamTest, ExpectTag, kVarintCases) { } } -TEST_1D(CodedStreamTest, ExpectTagFromArray, kVarintCases) { +TEST_P(VarintCases, ExpectTagFromArray) { + const VarintCase& kVarintCases_case = GetParam(); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); const uint32_t expected_value = @@ -283,7 +239,9 @@ TEST_1D(CodedStreamTest, ExpectTagFromArray, kVarintCases) { } } -TEST_2D(CodedStreamTest, ReadVarint64, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, ReadVarint64) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintCases_case.bytes, kVarintCases_case.size); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -298,7 +256,9 @@ TEST_2D(CodedStreamTest, ReadVarint64, kVarintCases, kBlockSizes) { EXPECT_EQ(kVarintCases_case.size, input.ByteCount()); } -TEST_2D(CodedStreamTest, WriteVarint32, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, WriteVarint32) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); if (kVarintCases_case.value > uint64_t{0x00000000FFFFFFFFu}) { // Skip this test for the 64-bit values. return; @@ -320,7 +280,9 @@ TEST_2D(CodedStreamTest, WriteVarint32, kVarintCases, kBlockSizes) { memcmp(buffer_, kVarintCases_case.bytes, kVarintCases_case.size)); } -TEST_2D(CodedStreamTest, WriteVarint64, kVarintCases, kBlockSizes) { +TEST_P(VarintCasesWithSizes, WriteVarint64) { + const VarintCase& kVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -337,14 +299,15 @@ TEST_2D(CodedStreamTest, WriteVarint64, kVarintCases, kBlockSizes) { memcmp(buffer_, kVarintCases_case.bytes, kVarintCases_case.size)); } -// This test causes gcc 3.3.5 (and earlier?) to give the cryptic error: -// "sorry, unimplemented: `method_call_expr' not supported by dump_expr" -#if !defined(__GNUC__) || __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3) +class SignedVarintCasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; int32_t kSignExtendedVarintCases[] = {0, 1, -1, 1237894, -37895138}; -TEST_2D(CodedStreamTest, WriteVarint32SignExtended, kSignExtendedVarintCases, - kBlockSizes) { +TEST_P(SignedVarintCasesWithSizes, WriteVarint32SignExtended) { + const int32_t& kSignExtendedVarintCases_case = std::get<0>(GetParam()); + const int& kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -381,8 +344,6 @@ TEST_2D(CodedStreamTest, WriteVarint32SignExtended, kSignExtendedVarintCases, EXPECT_EQ(output.ByteCount(), input.ByteCount()); } -#endif - // ------------------------------------------------------------------- // Varint failure test. @@ -391,6 +352,7 @@ struct VarintErrorCase { uint8_t bytes[12]; size_t size; bool can_parse; + const char* name; }; inline std::ostream& operator<<(std::ostream& os, const VarintErrorCase& c) { @@ -400,24 +362,34 @@ inline std::ostream& operator<<(std::ostream& os, const VarintErrorCase& c) { const VarintErrorCase kVarintErrorCases[] = { // Control case. (Insures that there isn't something else wrong that // makes parsing always fail.) - {{0x00}, 1, true}, + {{0x00}, 1, true, "Control"}, // No input data. - {{}, 0, false}, + {{}, 0, false, "No_Input"}, // Input ends unexpectedly. - {{0xf0, 0xab}, 2, false}, + {{0xf0, 0xab}, 2, false, "Input_Ends_Unexpectedly"}, // Input ends unexpectedly after 32 bits. - {{0xf0, 0xab, 0xc9, 0x9a, 0xf8, 0xb2}, 6, false}, + {{0xf0, 0xab, 0xc9, 0x9a, 0xf8, 0xb2}, + 6, + false, + "Input_Ends_Unexpectedly_After_32_Bits"}, // Longer than 10 bytes. {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}, 11, - false}, + false, + "Longer_Than_10_Bytes"}, }; -TEST_2D(CodedStreamTest, ReadVarint32Error, kVarintErrorCases, kBlockSizes) { +class VarintErrorCasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +TEST_P(VarintErrorCasesWithSizes, ReadVarint32Error) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -427,8 +399,10 @@ TEST_2D(CodedStreamTest, ReadVarint32Error, kVarintErrorCases, kBlockSizes) { EXPECT_EQ(kVarintErrorCases_case.can_parse, coded_input.ReadVarint32(&value)); } -TEST_2D(CodedStreamTest, ReadVarint32Error_LeavesValueInInitializedState, - kVarintErrorCases, kBlockSizes) { +TEST_P(VarintErrorCasesWithSizes, + ReadVarint32Error_LeavesValueInInitializedState) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -442,7 +416,9 @@ TEST_2D(CodedStreamTest, ReadVarint32Error_LeavesValueInInitializedState, EXPECT_EQ(value, value); } -TEST_2D(CodedStreamTest, ReadVarint64Error, kVarintErrorCases, kBlockSizes) { +TEST_P(VarintErrorCasesWithSizes, ReadVarint64Error) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -452,8 +428,10 @@ TEST_2D(CodedStreamTest, ReadVarint64Error, kVarintErrorCases, kBlockSizes) { EXPECT_EQ(kVarintErrorCases_case.can_parse, coded_input.ReadVarint64(&value)); } -TEST_2D(CodedStreamTest, ReadVarint64Error_LeavesValueInInitializedState, - kVarintErrorCases, kBlockSizes) { +TEST_P(VarintErrorCasesWithSizes, + ReadVarint64Error_LeavesValueInInitializedState) { + VarintErrorCase kVarintErrorCases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kVarintErrorCases_case.bytes, kVarintErrorCases_case.size); ArrayInputStream input(buffer_, static_cast(kVarintErrorCases_case.size), kBlockSizes_case); @@ -490,7 +468,11 @@ VarintSizeCase kVarintSizeCases[] = { {uint64_t{11964378330978735131u}, 10}, }; -TEST_1D(CodedStreamTest, VarintSize32, kVarintSizeCases) { +class VarintSizeCases : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(VarintSizeCases, VarintSize32) { + VarintSizeCase kVarintSizeCases_case = GetParam(); if (kVarintSizeCases_case.value > 0xffffffffu) { // Skip 64-bit values. return; @@ -501,7 +483,8 @@ TEST_1D(CodedStreamTest, VarintSize32, kVarintSizeCases) { static_cast(kVarintSizeCases_case.value))); } -TEST_1D(CodedStreamTest, VarintSize64, kVarintSizeCases) { +TEST_P(VarintSizeCases, VarintSize64) { + VarintSizeCase kVarintSizeCases_case = GetParam(); EXPECT_EQ(kVarintSizeCases_case.size, CodedOutputStream::VarintSize64(kVarintSizeCases_case.value)); } @@ -540,6 +523,20 @@ struct Fixed64Case { uint64_t value; // Parsed value. }; +class Fixed32Cases : public CodedStreamTest, + public testing::WithParamInterface {}; + +class Fixed32CasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +class Fixed64Cases : public CodedStreamTest, + public testing::WithParamInterface {}; + +class Fixed64CasesWithSizes + : public CodedStreamTest, + public testing::WithParamInterface> {}; + inline std::ostream& operator<<(std::ostream& os, const Fixed32Case& c) { return os << "0x" << std::hex << c.value << std::dec; } @@ -560,7 +557,9 @@ Fixed64Case kFixed64Cases[] = { uint64_t{0x8877665544332211u}}, }; -TEST_2D(CodedStreamTest, ReadLittleEndian32, kFixed32Cases, kBlockSizes) { +TEST_P(Fixed32CasesWithSizes, ReadLittleEndian32) { + Fixed32Case kFixed32Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kFixed32Cases_case.bytes, sizeof(kFixed32Cases_case.bytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -575,7 +574,9 @@ TEST_2D(CodedStreamTest, ReadLittleEndian32, kFixed32Cases, kBlockSizes) { EXPECT_EQ(sizeof(uint32_t), input.ByteCount()); } -TEST_2D(CodedStreamTest, ReadLittleEndian64, kFixed64Cases, kBlockSizes) { +TEST_P(Fixed64CasesWithSizes, ReadLittleEndian64) { + Fixed64Case kFixed64Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); memcpy(buffer_, kFixed64Cases_case.bytes, sizeof(kFixed64Cases_case.bytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -590,7 +591,9 @@ TEST_2D(CodedStreamTest, ReadLittleEndian64, kFixed64Cases, kBlockSizes) { EXPECT_EQ(sizeof(uint64_t), input.ByteCount()); } -TEST_2D(CodedStreamTest, WriteLittleEndian32, kFixed32Cases, kBlockSizes) { +TEST_P(Fixed32CasesWithSizes, WriteLittleEndian32) { + Fixed32Case kFixed32Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -606,7 +609,9 @@ TEST_2D(CodedStreamTest, WriteLittleEndian32, kFixed32Cases, kBlockSizes) { EXPECT_EQ(0, memcmp(buffer_, kFixed32Cases_case.bytes, sizeof(uint32_t))); } -TEST_2D(CodedStreamTest, WriteLittleEndian64, kFixed64Cases, kBlockSizes) { +TEST_P(Fixed64CasesWithSizes, WriteLittleEndian64) { + Fixed64Case kFixed64Cases_case = std::get<0>(GetParam()); + int kBlockSizes_case = std::get<1>(GetParam()); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -624,7 +629,8 @@ TEST_2D(CodedStreamTest, WriteLittleEndian64, kFixed64Cases, kBlockSizes) { // Tests using the static methods to read fixed-size values from raw arrays. -TEST_1D(CodedStreamTest, ReadLittleEndian32FromArray, kFixed32Cases) { +TEST_P(Fixed32Cases, ReadLittleEndian32FromArray) { + Fixed32Case kFixed32Cases_case = GetParam(); memcpy(buffer_, kFixed32Cases_case.bytes, sizeof(kFixed32Cases_case.bytes)); uint32_t value; @@ -634,7 +640,8 @@ TEST_1D(CodedStreamTest, ReadLittleEndian32FromArray, kFixed32Cases) { EXPECT_TRUE(end == buffer_ + sizeof(value)); } -TEST_1D(CodedStreamTest, ReadLittleEndian64FromArray, kFixed64Cases) { +TEST_P(Fixed64Cases, ReadLittleEndian64FromArray) { + Fixed64Case kFixed64Cases_case = GetParam(); memcpy(buffer_, kFixed64Cases_case.bytes, sizeof(kFixed64Cases_case.bytes)); uint64_t value; @@ -649,7 +656,11 @@ TEST_1D(CodedStreamTest, ReadLittleEndian64FromArray, kFixed64Cases) { const char kRawBytes[] = "Some bytes which will be written and read raw."; -TEST_1D(CodedStreamTest, ReadRaw, kBlockSizes) { +class BlockSizes : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(BlockSizes, ReadRaw) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); char read_buffer[sizeof(kRawBytes)]; @@ -664,7 +675,8 @@ TEST_1D(CodedStreamTest, ReadRaw, kBlockSizes) { EXPECT_EQ(sizeof(kRawBytes), input.ByteCount()); } -TEST_1D(CodedStreamTest, WriteRaw, kBlockSizes) { +TEST_P(BlockSizes, WriteRaw) { + int kBlockSizes_case = GetParam(); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -680,7 +692,8 @@ TEST_1D(CodedStreamTest, WriteRaw, kBlockSizes) { EXPECT_EQ(0, memcmp(buffer_, kRawBytes, sizeof(kRawBytes))); } -TEST_1D(CodedStreamTest, ReadString, kBlockSizes) { +TEST_P(BlockSizes, ReadString) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -696,7 +709,8 @@ TEST_1D(CodedStreamTest, ReadString, kBlockSizes) { } // Check to make sure ReadString doesn't crash on impossibly large strings. -TEST_1D(CodedStreamTest, ReadStringImpossiblyLarge, kBlockSizes) { +TEST_P(BlockSizes, ReadStringImpossiblyLarge) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -724,7 +738,8 @@ TEST_F(CodedStreamTest, ReadStringImpossiblyLargeFromStringOnHeap) { EXPECT_FALSE(coded_input.ReadString(&str, 1 << 30)); } -TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnTotalLimit, kBlockSizes) { +TEST_P(BlockSizes, ReadStringReservesMemoryOnTotalLimit) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -745,7 +760,8 @@ TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnTotalLimit, kBlockSizes) { EXPECT_EQ(strlen(kRawBytes), input.ByteCount()); } -TEST_1D(CodedStreamTest, ReadStringReservesMemoryOnPushedLimit, kBlockSizes) { +TEST_P(BlockSizes, ReadStringReservesMemoryOnPushedLimit) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -913,7 +929,15 @@ TEST_F(CodedStreamTest, // ------------------------------------------------------------------- // Cord reads and writes -TEST_1D(CodedStreamTest, ReadCord, kBlockSizes) { +class BlockSizesWithResetCords + : public CodedStreamTest, + public testing::WithParamInterface> {}; + +class ResetCords : public CodedStreamTest, + public testing::WithParamInterface {}; + +TEST_P(BlockSizes, ReadCord) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kRawBytes, sizeof(kRawBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -928,8 +952,9 @@ TEST_1D(CodedStreamTest, ReadCord, kBlockSizes) { EXPECT_EQ(strlen(kRawBytes), input.ByteCount()); } -TEST_1D(CodedStreamTest, ReadCordReuseCord, kBlockSizes) { +TEST_P(BlockSizes, ReadCordReuseCord) { ASSERT_GT(sizeof(buffer_), 1362 * sizeof(kRawBytes)); + int kBlockSizes_case = GetParam(); for (size_t i = 0; i < 1362; i++) { memcpy(buffer_ + i * sizeof(kRawBytes), kRawBytes, sizeof(kRawBytes)); } @@ -967,7 +992,9 @@ TEST_1D(CodedStreamTest, ReadCordReuseCord, kBlockSizes) { EXPECT_EQ(total_read, input.ByteCount()); } -TEST_2D(CodedStreamTest, ReadCordWithLimit, kBlockSizes, kResetCords) { +TEST_P(BlockSizesWithResetCords, ReadCordWithLimit) { + int kBlockSizes_case = std::get<0>(GetParam()); + bool kResetCords_case = std::get<1>(GetParam()); memcpy(buffer_, kRawBytes, strlen(kRawBytes)); ArrayInputStream input(buffer_, strlen(kRawBytes), kBlockSizes_case); CodedInputStream coded_input(&input); @@ -991,7 +1018,8 @@ TEST_2D(CodedStreamTest, ReadCordWithLimit, kBlockSizes, kResetCords) { EXPECT_EQ(std::string(kRawBytes + 10), std::string(cord)); } -TEST_1D(CodedStreamTest, ReadLargeCord, kResetCords) { +TEST_P(ResetCords, ReadLargeCord) { + bool kResetCords_case = GetParam(); absl::Cord large_cord; for (int i = 0; i < 1024; i++) { large_cord.Append(kRawBytes); @@ -1012,7 +1040,9 @@ TEST_1D(CodedStreamTest, ReadLargeCord, kResetCords) { } // Check to make sure ReadString doesn't crash on impossibly large strings. -TEST_2D(CodedStreamTest, ReadCordImpossiblyLarge, kBlockSizes, kResetCords) { +TEST_P(BlockSizesWithResetCords, ReadCordImpossiblyLarge) { + int kBlockSizes_case = std::get<0>(GetParam()); + bool kResetCords_case = std::get<1>(GetParam()); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1026,7 +1056,8 @@ TEST_2D(CodedStreamTest, ReadCordImpossiblyLarge, kBlockSizes, kResetCords) { } } -TEST_1D(CodedStreamTest, WriteCord, kBlockSizes) { +TEST_P(BlockSizes, WriteCord) { + int kBlockSizes_case = GetParam(); ArrayOutputStream output(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1096,7 +1127,8 @@ TEST_F(CodedStreamTest, Trim) { const char kSkipTestBytes[] = ""; -TEST_1D(CodedStreamTest, SkipInput, kBlockSizes) { +TEST_P(BlockSizes, SkipInput) { + int kBlockSizes_case = GetParam(); memcpy(buffer_, kSkipTestBytes, sizeof(kSkipTestBytes)); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); @@ -1179,7 +1211,8 @@ TEST_F(CodedStreamTest, GetDirectBufferPointerInlineInput) { // ------------------------------------------------------------------- // Limits -TEST_1D(CodedStreamTest, BasicLimit, kBlockSizes) { +TEST_P(BlockSizes, BasicLimit) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1209,7 +1242,8 @@ TEST_1D(CodedStreamTest, BasicLimit, kBlockSizes) { // Test what happens when we push two limits where the second (top) one is // shorter. -TEST_1D(CodedStreamTest, SmallLimitOnTopOfBigLimit, kBlockSizes) { +TEST_P(BlockSizes, SmallLimitOnTopOfBigLimit) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1251,7 +1285,8 @@ TEST_1D(CodedStreamTest, SmallLimitOnTopOfBigLimit, kBlockSizes) { // Test what happens when we push two limits where the second (top) one is // longer. In this case, the top limit is shortened to match the previous // limit. -TEST_1D(CodedStreamTest, BigLimitOnTopOfSmallLimit, kBlockSizes) { +TEST_P(BlockSizes, BigLimitOnTopOfSmallLimit) { + int kBlockSizes_case = GetParam(); ArrayInputStream input(buffer_, sizeof(buffer_), kBlockSizes_case); { @@ -1503,6 +1538,98 @@ TEST_F(CodedStreamTest, InputOver2G) { EXPECT_EQ(INT_MAX - 512, input.backup_amount_); } +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, ResetCords, testing::ValuesIn(kResetCords), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("ResetCords_", param_info.param ? "true" : "false"); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, BlockSizesWithResetCords, + testing::Combine(testing::ValuesIn(kBlockSizes), + testing::ValuesIn(kResetCords)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("BlockSize_", std::get<0>(param_info.param), + "_ResetCords_", + std::get<1>(param_info.param) ? "true" : "false"); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintErrorCasesWithSizes, + testing::Combine(testing::ValuesIn(kVarintErrorCases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("VarintErrorCase_", + std::get<0>(param_info.param).name, "_BlockSize_", + std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintSizeCases, testing::ValuesIn(kVarintSizeCases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("VarintSizeCase_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintCases, testing::ValuesIn(kVarintCases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("VarintCase_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, VarintCasesWithSizes, + testing::Combine(testing::ValuesIn(kVarintCases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("VarintCase_Value_", + std::get<0>(param_info.param).value, "_BlockSize_", + std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, BlockSizes, testing::ValuesIn(kBlockSizes), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("BlockSize_", param_info.param); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, SignedVarintCasesWithSizes, + testing::Combine(testing::ValuesIn(kSignExtendedVarintCases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("SignedVarintCase_Value_", + std::get<0>(param_info.param) < 0 ? "_Negative_" : "", + std::abs(std::get<0>(param_info.param)), + "_BlockSize_", std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed32Cases, testing::ValuesIn(kFixed32Cases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("Fixed32Case_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed64Cases, testing::ValuesIn(kFixed64Cases), + [](const testing::TestParamInfo& param_info) { + return absl::StrCat("Fixed64Case_Value_", param_info.param.value); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed32CasesWithSizes, + testing::Combine(testing::ValuesIn(kFixed32Cases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("Fixed32Case_Value_", + std::get<0>(param_info.param).value, "_BlockSize_", + std::get<1>(param_info.param)); + }); +INSTANTIATE_TEST_SUITE_P( + CodedStreamUnitTest, Fixed64CasesWithSizes, + testing::Combine(testing::ValuesIn(kFixed64Cases), + testing::ValuesIn(kBlockSizes)), + [](const testing::TestParamInfo& + param_info) { + return absl::StrCat("Fixed64Case_Value_", + std::get<0>(param_info.param).value, "_BlockSize_", + std::get<1>(param_info.param)); + }); + } // namespace } // namespace io } // namespace protobuf diff --git a/src/google/protobuf/lazy_repeated_field.cc b/src/google/protobuf/lazy_repeated_field.cc new file mode 100644 index 0000000000..e3b83ad04d --- /dev/null +++ b/src/google/protobuf/lazy_repeated_field.cc @@ -0,0 +1,344 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "google/protobuf/lazy_repeated_field.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/log/log.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/generated_message_util.h" +#include "google/protobuf/io/coded_stream.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/message_lite.h" +#include "google/protobuf/parse_context.h" + +// Must be included last. +// clang-format off +#include "google/protobuf/port_def.inc" +#include "google/protobuf/repeated_ptr_field.h" +// clang-format on + +namespace google { +namespace protobuf { +namespace internal { +namespace {} // namespace + +namespace { + +inline const char* InternalParseRepeated(const char* ptr, + ParseContext* local_ctx, + RepeatedPtrFieldBase* value, + const MessageLite* prototype) { + uint32_t expected_tag; + ptr = ReadTag(ptr, &expected_tag); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + // TODO: Try to optimize this. The tags and lengths are read again + // which is a bit wasteful. + return LazyRepeatedPtrField::ParseToRepeatedMessage( + ptr, local_ctx, prototype, expected_tag, value); +} + +template +inline bool ParseWithNullOuterContextImpl(const T& input, + RepeatedPtrFieldBase* value, + const MessageLite* prototype, + bool set_missing_required) { + // Null outer context means it's either already verified or unverified. + // + // If the payload is eagerly verified, the recursion limit was also verified + // and we don't need to repeat that. Also, users might have used a custom + // limit which is not known at this access. + // + // Unverified lazy fields may suffer from stack overflow with deeply nested + // data. We argue that it should be better than silent data corruption. + constexpr int kUnlimitedDepth = std::numeric_limits::max(); + const char* ptr; + ParseContext local_ctx(kUnlimitedDepth, false, &ptr, input); + + if (set_missing_required) { + local_ctx.SetParentMissingRequiredFields(); + } + // Unparsed data is already verified at parsing. Disable eager-verification. + (void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy); + + ptr = InternalParseRepeated(ptr, &local_ctx, value, prototype); + return ptr != nullptr && + (local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit()); +} + +template +inline bool ParseWithOuterContextImpl(const T& input, ParseContext* ctx, + RepeatedPtrFieldBase* value, + const MessageLite* prototype, + bool set_missing_required) { + if (ctx == nullptr) { + return ParseWithNullOuterContextImpl(input, value, prototype, + set_missing_required); + } + + ABSL_DCHECK(!ctx->AliasingEnabled()); + // set_missing_required => ctx == nullptr + ABSL_DCHECK(!set_missing_required); + + // Create local context with depth. + const char* ptr; + ParseContext local_ctx(ParseContext::kSpawn, *ctx, &ptr, input); + + if (set_missing_required) { + local_ctx.SetParentMissingRequiredFields(); + } + if (ctx->lazy_parse_mode() == ParseContext::LazyParseMode::kEagerVerify) { + // Unparsed data is already verified at parsing. Disable eager-verification. + (void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy); + } + + ptr = InternalParseRepeated(ptr, &local_ctx, value, prototype); + + if (local_ctx.missing_required_fields()) { + ctx->SetMissingRequiredFields(); + } + + return ptr != nullptr && + (local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit()); +} + +class ByPrototype { + public: + explicit ByPrototype(const MessageLite* prototype) : prototype_(prototype) {} + + MessageLite* New(Arena* arena) const { return prototype_->New(arena); } + + const MessageLite& Default() const { return *prototype_; } + + private: + const MessageLite* prototype_; +}; +} // namespace + +const RepeatedPtrFieldBase* LazyRepeatedPtrField::GetByPrototype( + const MessageLite* prototype, Arena* arena, ParseContext* ctx) const { + return GetGeneric(ByPrototype(prototype), arena, ctx); +} + +RepeatedPtrFieldBase* LazyRepeatedPtrField::MutableByPrototype( + const MessageLite* prototype, Arena* arena, ParseContext* ctx) { + return MutableGeneric(ByPrototype(prototype), arena, ctx); +} + +void LazyRepeatedPtrField::Clear() { + PerformTransition([](ExclusiveTxn& txn) { + auto* value = txn.mutable_value(); + if (value != nullptr) value->Clear>(); + return RawState::kCleared; + }); +} + +bool LazyRepeatedPtrField::IsEagerSerializeSafe(const MessageLite* prototype, + int32_t number, + Arena* arena) const { + // "prototype" may be null if it is for dynamic messages. This is ok as + // dynamic extensions won't be lazy as they lack verify functions any way. + if (prototype == nullptr) return false; + + for (;;) { + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + case LogicalState::kDirty: + return true; + case LogicalState::kNoParseRequired: { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + size_t tag_size = WireFormatLite::TagSize( + number, WireFormatLite::FieldType::TYPE_MESSAGE); + size_t total_size = tag_size * value->size(); + for (int i = 0; i < value->size(); i++) { + total_size += WireFormatLite::LengthDelimitedSize( + value->Get>(i).ByteSizeLong()); + } + return total_size == unparsed_.Size(); + } + case LogicalState::kParseRequired: { + GetByPrototype(prototype, arena); + break; // reswitch + } + } + } + // Required for certain compiler configurations. + ABSL_LOG(FATAL) << "Not reachable"; + return false; +} + +void LazyRepeatedPtrField::swap_atomics(std::atomic& lhs, + std::atomic& rhs) { + auto l = lhs.exchange(rhs.load(std::memory_order_relaxed), + std::memory_order_relaxed); + rhs.store(l, std::memory_order_relaxed); +} + +void LazyRepeatedPtrField::Swap(LazyRepeatedPtrField* lhs, Arena* lhs_arena, + LazyRepeatedPtrField* rhs, Arena* rhs_arena) { + static auto reallocate = [](LazyRepeatedPtrField* f, Arena* arena, + bool cleanup_old) { + auto raw = f->raw_.load(std::memory_order_relaxed); + if (raw.value() != nullptr) { + auto* new_value = Arena::Create(arena); + if (!raw.value()->empty()) { + new_value->MergeFrom(*raw.value()); + } + if (cleanup_old) { + delete reinterpret_cast*>( + raw.value()); + }; + raw.set_value(new_value); + f->raw_.store(raw, std::memory_order_relaxed); + } + auto old_unparsed = f->unparsed_; + f->unparsed_.Visit( + [] {}, + [&](auto& cord) { f->unparsed_.InitAsCord(arena, std::move(cord)); }, + [&](auto view) { + if (arena == nullptr) { + f->unparsed_.InitAsCord(arena, view); + } else { + f->unparsed_.InitAndSetArray(arena, view); + } + }); + if (cleanup_old) old_unparsed.Destroy(); + }; + static auto take_ownership = [](LazyRepeatedPtrField* f, Arena* arena) { +#ifdef PROTOBUF_FORCE_COPY_IN_SWAP + reallocate(f, arena, true); +#else + arena->Own(reinterpret_cast*>( + f->raw_.load(std::memory_order_relaxed).mutable_value())); + f->unparsed_.TransferHeapOwnershipToArena(arena); +#endif + }; + + using std::swap; // Enable ADL with fallback + swap_atomics(lhs->raw_, rhs->raw_); + swap(lhs->unparsed_, rhs->unparsed_); + // At this point we are in a weird state. The messages have been swapped into + // their destination, but we have completely ignored the arenas, so the owning + // arena is actually on the opposite message. Now we straighten out our + // ownership by forcing reallocations/ownership changes as needed. + if (lhs_arena == rhs_arena) { +#ifdef PROTOBUF_FORCE_COPY_IN_SWAP + if (lhs_arena == nullptr) { + reallocate(lhs, lhs_arena, true); + reallocate(rhs, rhs_arena, true); + } +#endif + } else { + if (lhs_arena == nullptr) { + take_ownership(rhs, rhs_arena); + reallocate(lhs, lhs_arena, false); + } else if (rhs_arena == nullptr) { + take_ownership(lhs, lhs_arena); + reallocate(rhs, rhs_arena, false); + } else { + reallocate(lhs, lhs_arena, false); + reallocate(rhs, rhs_arena, false); + } + } +} + +void LazyRepeatedPtrField::InternalSwap( + LazyRepeatedPtrField* PROTOBUF_RESTRICT lhs, + LazyRepeatedPtrField* PROTOBUF_RESTRICT rhs) { + using std::swap; // Enable ADL with fallback + swap_atomics(lhs->raw_, rhs->raw_); + swap(lhs->unparsed_, rhs->unparsed_); +} + +bool LazyRepeatedPtrField::ParseWithOuterContext(RepeatedPtrFieldBase* value, + const absl::Cord& input, + ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required) { + absl::optional flat = input.TryFlat(); + if (flat.has_value()) { + return ParseWithOuterContextImpl(*flat, ctx, value, prototype, + set_missing_required); + } + + io::CordInputStream cis(&input); + return ParseWithOuterContextImpl(&cis, ctx, value, prototype, + set_missing_required); +} + +bool LazyRepeatedPtrField::ParseWithOuterContext(RepeatedPtrFieldBase* value, + absl::string_view input, + ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required) { + return ParseWithOuterContextImpl(input, ctx, value, prototype, + set_missing_required); +} + +size_t LazyRepeatedPtrField::ByteSizeLong(size_t tag_size) const { + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: + return unparsed_.Size(); + + case LogicalState::kDirty: + const auto* value = raw_.load(std::memory_order_relaxed).value(); + size_t total_size = tag_size * value->size(); + for (int i = 0; i < value->size(); i++) { + total_size += WireFormatLite::LengthDelimitedSize( + value->Get>(i).ByteSizeLong()); + } + return total_size; + } + // Required for certain compiler configurations. + ABSL_LOG(FATAL) << "Not reachable"; + return -1; +} + +void LazyRepeatedPtrField::LogParseError(const RepeatedPtrFieldBase* value) { + const MessageLite* message = + &value->at>(0); + auto get_error_string = [&value]() { + std::string str; + for (int i = 0; i < value->size(); i++) { + absl::StrAppend(&str, "[", i, "]: ", + value->at>(i) + .InitializationErrorString(), + "\n"); + } + return str; + }; +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) + // In fuzzing mode, we log less to speed up fuzzing. + ABSL_LOG_EVERY_N(INFO, 100000) +#else + ABSL_LOG_EVERY_N_SEC(INFO, 1) +#endif + << "Lazy parsing failed for RepeatedPtrField<" << message->GetTypeName() + << "> error=" << get_error_string() << " (N = " << COUNTER << ")"; +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" diff --git a/src/google/protobuf/lazy_repeated_field.h b/src/google/protobuf/lazy_repeated_field.h new file mode 100644 index 0000000000..ab573e58c0 --- /dev/null +++ b/src/google/protobuf/lazy_repeated_field.h @@ -0,0 +1,1123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_LAZY_REPEATED_FIELD_H__ +#define GOOGLE_PROTOBUF_LAZY_REPEATED_FIELD_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/log/absl_check.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/generated_message_util.h" +#include "google/protobuf/internal_visibility.h" +#include "google/protobuf/io/coded_stream.h" +#include "google/protobuf/message_lite.h" +#include "google/protobuf/parse_context.h" +#include "google/protobuf/port.h" +#include "google/protobuf/raw_ptr.h" +#include "google/protobuf/repeated_ptr_field.h" +#include "google/protobuf/wire_format_verify.h" + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif + +// must be last +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { + +class Descriptor; +namespace io { +class CodedInputStream; +class CodedOutputStream; +} // namespace io +} // namespace protobuf +} // namespace google + +namespace google { +namespace protobuf { +namespace internal { + +inline const char* ReadTagInternal(const char* ptr, uint8_t* tag) { + *tag = UnalignedLoad(ptr); + return ptr + sizeof(uint8_t); +} + +inline const char* ReadTagInternal(const char* ptr, uint16_t* tag) { + *tag = UnalignedLoad(ptr); + return ptr + sizeof(uint16_t); +} + +inline const char* ReadTagInternal(const char* ptr, uint32_t* tag) { + return ReadTag(ptr, tag); +} + +template +inline size_t TagSizeInternal(TagType tag); +template <> +inline size_t TagSizeInternal(uint8_t tag) { + return sizeof(uint8_t); +} +template <> +inline size_t TagSizeInternal(uint16_t tag) { + return sizeof(uint16_t); +} +template <> +inline size_t TagSizeInternal(uint32_t tag) { + return io::CodedOutputStream::VarintSize32(tag); +} + +// This class is used to represent lazily-loaded repeated message fields. +// It stores the field in a raw buffer or a Cord initially, and then parses that +// on-demand if a caller asks for the RepeatedPtrField object. +// +// As with most protobuf classes, const methods of this class are safe to call +// from multiple threads at once, but non-const methods may only be called when +// the thread has guaranteed that it has exclusive access to the field. +class LazyRepeatedPtrField { + public: + constexpr LazyRepeatedPtrField() : raw_(MessageState(RawState::kCleared)) {} + LazyRepeatedPtrField(const LazyRepeatedPtrField& rhs) + : LazyRepeatedPtrField(nullptr, rhs, nullptr) {} + + // Arena enabled constructors. + LazyRepeatedPtrField(internal::InternalVisibility, Arena* arena) + : LazyRepeatedPtrField(arena) {} + LazyRepeatedPtrField(internal::InternalVisibility, Arena* arena, + const LazyRepeatedPtrField& rhs, Arena* rhs_arena) + : LazyRepeatedPtrField(arena, rhs, rhs_arena) {} + + // TODO: make this constructor private + explicit constexpr LazyRepeatedPtrField(Arena*) + : raw_(MessageState(RawState::kCleared)) {} + + LazyRepeatedPtrField& operator=(const LazyRepeatedPtrField&) = delete; + + ~LazyRepeatedPtrField(); + + bool IsClear() const { + auto state = GetLogicalState(); + return state == LogicalState::kClear || + state == LogicalState::kClearExposed; + } + + // Get and Mutable trigger parsing. + template + const RepeatedPtrField& Get(const Element* default_instance, + Arena* arena) const { + return *reinterpret_cast*>( + GetGeneric(ByTemplate(default_instance), arena, nullptr)); + } + + template + RepeatedPtrField* Mutable(const Element* default_instance, + Arena* arena) { + return reinterpret_cast*>( + MutableGeneric(ByTemplate(default_instance), arena, nullptr)); + } + + bool IsInitialized(const MessageLite* prototype, Arena* arena) const { + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: { + return true; + } + case LogicalState::kParseRequired: + case LogicalState::kNoParseRequired: { + // Returns true if "unparsed" is not verified to be (maybe) + // uninitialized. Otherwise, falls through to next cases to eagerly + // parse message and call IsInitialized(). + if (!MaybeUninitialized()) return true; + } + ABSL_FALLTHROUGH_INTENDED; + case LogicalState::kDirty: { + const auto& value = *GetByPrototype(prototype, arena); + for (int i = 0; i < value.size(); ++i) { + if (!value.Get>(i).IsInitialized()) + return false; + } + return true; + } + default: + __builtin_unreachable(); + } + } + + // Dynamic versions of basic accessors. + const RepeatedPtrFieldBase* GetDynamic(const Descriptor* type, + MessageFactory* factory, + Arena* arena) const; + RepeatedPtrFieldBase* MutableDynamic(const Descriptor* type, + MessageFactory* factory, Arena* arena); + + // Basic accessors that use a default instance to create the message. + const RepeatedPtrFieldBase* GetByPrototype(const MessageLite* prototype, + Arena* arena, + ParseContext* ctx = nullptr) const; + RepeatedPtrFieldBase* MutableByPrototype(const MessageLite* prototype, + Arena* arena, + ParseContext* ctx = nullptr); + + void Clear(); + + // Updates state such that state set in other overwrites this. + // + // Internal Lazy state transitions are updated as such: + // + // src\dest | UNINIT | INIT | DIRTY | CLEAR | ERROR + // :------- | :----: | :---: | :---: | :-----------: | :---: + // UNINIT | DIRTY | DIRTY | DIRTY | UNINIT/DIRTY* | DIRTY + // INIT | DIRTY | DIRTY | DIRTY | UNINIT/DIRTY* | UNDEF + // DIRTY | DIRTY | DIRTY | DIRTY | UNINIT/DIRTY* | UNDEF + // CLEAR | UNINIT | INIT | DIRTY | CLEAR | UNDEF + // ERROR | DIRTY | DIRTY | DIRTY | DIRTY | DIRTY + // * Depends on if clear was initialized before. + // TODO: The state after ERROR should be DIRTY. Also need to make the + // change for LazyField. + void MergeFrom(const MessageLite* prototype, + const LazyRepeatedPtrField& other, Arena* arena, + Arena* other_arena); + + static void Swap(LazyRepeatedPtrField* lhs, Arena* lhs_arena, + LazyRepeatedPtrField* rhs, Arena* rhs_arena); + static void InternalSwap(LazyRepeatedPtrField* lhs, + LazyRepeatedPtrField* rhs); + + const RepeatedPtrFieldBase* TryGetRepeated() const; + + // Returns true when the lazy field has data that have not yet parsed. + // (i.e. parsing has been deferred) Once parsing has been attempted, this + // returns false. Note that the LazyField object may still contain + // the raw unparsed data with parsing errors. + bool HasUnparsed() const { + return GetLogicalState() == LogicalState::kParseRequired; + } + + // Returns true if parsing has been attempted and it failed. + bool HasParsingError() const { + auto raw = raw_.load(std::memory_order_relaxed); + return raw.status() == RawState::kParseError; + } + + // APIs that will be used by table-driven parsing. + // + // `TagType` is passed from table-driven parser. On fast path it's uint8 or + // uint16; on slow path it's uint32. + template + const char* _InternalParse(const MessageLite* prototype, Arena* arena, + const char* ptr, ParseContext* ctx, + TagType expected_tag) { + // If this message is eagerly-verified lazy, kEager mode likely suggests + // that previous verification has failed and we fall back to eager-parsing + // (either to initialize the message to match eager field or to fix false + // errors. + // + // Lazy parsing does not support aliasing and may result in data copying. + // It seems prudent to honor aliasing to avoid any observable gaps between + // lazy and eager parsing. + if (ctx->lazy_parse_mode() == ParseContext::kEager || + ctx->AliasingEnabled()) { + auto* value = MutableByPrototype(prototype, arena, ctx); + ptr = ParseToRepeatedMessage(ptr, ctx, prototype, expected_tag, + value); + return ptr; + } + + switch (GetLogicalState()) { + case LogicalState::kParseRequired: { + return ParseToCord(ptr, ctx, prototype, arena, expected_tag); + } break; + + case LogicalState::kClear: { + // Clear/Fresh have empty unparsed data; so this is the equivalent + // of setting it to the passed in bytes. + return ParseToCord(ptr, ctx, prototype, arena, expected_tag); + } break; + + // Pointers exposed. + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: { + PerformTransition([&](ExclusiveTxn& txn) { + auto* value = txn.mutable_value(); + ptr = ParseToRepeatedMessage(ptr, ctx, prototype, + expected_tag, value); + return RawState::kIsParsed; + }); + return ptr; + } + } + // Required for certain compiler configurations. + internal::Unreachable(); + return nullptr; + } + + template + const char* _InternalParseVerify(const MessageLite* prototype, Arena* arena, + const char* ptr, ParseContext* ctx, + TagType expected_tag, + absl::string_view data) { + ABSL_DCHECK(ptr != nullptr); + if (ctx->lazy_parse_mode() == ParseContext::kLazy || + ctx->lazy_eager_verify_func() == nullptr) { + return ptr; + } + VerifyResult res = WireFormatVerifyView(data, ctx); + if (res.verified) { + if (res.missing_required_fields) { + // Unparsed data may be uninitialized and need to be parsed to be sure. + SetNeedsParseMaybeUninitialized(); + } + return ptr; + } + + // Try eager parsing on potentially malformed wire in case the eager parsing + // fixes the issue. For example, a negative int32 encoded as 5B varint can + // be parsed correctly. + // + // Should preserve the old parsing mode because we don't want to + // unnecessarily eager-parse other parts of message tree. This can be + // especially inefficient if the eager verification results in false + // positive errors. + ParseContext::LazyParseMode old = + ctx->set_lazy_parse_mode(ParseContext::kEager); + (void)GetByPrototype(prototype, arena, ctx); + + // If eager parsing still fails, don't bother restoring the parse mode. + if (HasParsingError()) return nullptr; + + // Unverified lazy fields may miss parsing errors on eager parsing. If it's + // certain, just mark error and return. + if (!ctx->treat_eager_parsing_errors_as_errors()) { + auto raw = raw_.load(std::memory_order_relaxed); + raw.set_status(RawState::kParseError); + raw_.store(raw, std::memory_order_relaxed); + ABSL_DCHECK(HasParsingError()); + return nullptr; + } + + // We need to transition to dirty to prefer eager serialization as the + // unparsed_ has non-canonical wire format. + (void)MutableByPrototype(prototype, arena); + + (void)ctx->set_lazy_parse_mode(old); + return ptr; + } + + template + static const char* ParseToRepeatedMessage(const char* ptr, ParseContext* ctx, + const MessageLite* prototype, + TagType expected_tag, + RepeatedPtrFieldBase* value) { + const char* ptr2 = ptr; + TagType next_tag; + do { + MessageLite* submsg = value->AddMessage(prototype); + // ptr2 points to the start of the element's encoded length. + ptr = ctx->ParseMessage(submsg, ptr2); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + if (ctx->Done(&ptr)) { + break; + } + } + ptr2 = ReadTagInternal(ptr, &next_tag); + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) return nullptr; + } while (next_tag == expected_tag); + return ptr; + } + + template + const char* ParseToCord(const char* ptr, ParseContext* ctx, + const MessageLite* prototype, Arena* arena, + TagType expected_tag) { + // ptr2 points to the start of the encoded length. + const char* ptr2 = ptr; + TagType next_tag; + // Move ptr back to the start of the tag. + size_t tag_size = TagSizeInternal(expected_tag); + ptr -= tag_size; + if (ctx->parent_missing_required_fields()) { + SetNeedsParseMaybeUninitialized(); + } else { + SetNeedsParse(); + } + do { + std::string tmp; + // Append the tag. + tmp.append(absl::string_view(ptr, ptr2 - ptr)); + size_t taglen_size; + ptr = ctx->ParseLengthDelimitedInlined( + ptr2, [&tmp, &taglen_size, ctx, ptr2](const char* p) { + // At this moment length is read and p points to the start of + // the payload. + ABSL_DCHECK(p - ptr2 > 0 && p - ptr2 <= 5) << p - ptr2; + // Append the length. + tmp.append(absl::string_view(ptr2, p - ptr2)); + taglen_size = tmp.size(); + return ctx->AppendString(p, &tmp); + }); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + const auto tmp_size = tmp.size(); + ABSL_DCHECK_GE(tmp_size, taglen_size); + if (unparsed_.IsCord()) { + unparsed_.AsCord().Append(tmp); + } else if (arena != nullptr && + unparsed_.Size() + tmp_size <= kMaxArraySize) { + if (unparsed_.IsEmpty()) { + unparsed_.InitAsArray(arena, 0); + } + unparsed_.AppendToArray(tmp); + } else { + unparsed_.UpgradeToCord(arena).Append(tmp); + } + if (tmp_size > taglen_size) { + ptr = _InternalParseVerify( + prototype, arena, ptr, ctx, expected_tag, + absl::string_view(tmp.data() + taglen_size, + tmp_size - taglen_size)); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; + } + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + // `Done` advances the stream to the next buffer chunk. + if (ctx->Done(&ptr)) { + break; + } + } + // ptr points to the start of the next tag. + ptr2 = ReadTagInternal(ptr, &next_tag); + // ptr2 points to the start of the next element's encoded length. + + // TODO: Try to remove the following condition for 8 and 16 bits + // TagType. + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) return nullptr; + } while (next_tag == expected_tag); + if (unparsed_.IsArray()) { + unparsed_.ZeroOutTailingBytes(); + } + return ptr; + } + + uint8_t* InternalWrite(const MessageLite* prototype, int32_t number, + uint8_t* target, + io::EpsCopyOutputStream* stream) const; + + // ByteSize of the repeated ptr field (including the varints of tags and + // lengths). + size_t ByteSizeLong(size_t tag_size) const; + size_t SpaceUsedExcludingSelfLong() const; + + // LogicalState combines the `raw_` and `unparsed_` fields to produce the + // current state. + // + // This separation allows more easily adding fine-grained states w/o touching + // std::atomics; most state transitions are in a write context and do not + // require subtle atomicity. + // TODO: Deduplicate with LazyField. + enum class LogicalState { + // The serialized data is available and unparsed. + // (kParseRequired, !unparsed.empty(), message = undefined). + kParseRequired, + // The message has been parsed from the serialized data. + // (kIsParsed, !unparsed.empty(), message != nullptr). + kNoParseRequired, + // The field is clear (freshly constructed or cleared): + // - (kCleared, unparsed.empty(), message = nullptr) + kClear, + // The field is clear but previously exposed a pointer. + // - (kCleared, unparsed.empty(), message = !nullptr) + kClearExposed, + // A write operation was done after a parse. + // (kIsParsed, unparsed.empty(), message != nullptr) + kDirty, + }; + LogicalState GetLogicalState() const { + auto raw = raw_.load(std::memory_order_acquire); + switch (raw.status()) { + case RawState::kParseError: + ABSL_DCHECK_NE(raw.value(), nullptr); + return LogicalState::kDirty; + case RawState::kCleared: + ABSL_DCHECK(unparsed_.IsEmpty()); + ABSL_DCHECK(raw.value() == nullptr || raw.value()->empty()) + << (raw.value() == nullptr + ? "nullptr" + : absl::StrCat("non-empty:", raw.value()->size())); + return raw.value() == nullptr ? LogicalState::kClear + : LogicalState::kClearExposed; + case RawState::kNeedsParse: + case RawState::kNeedsParseMaybeUninitialized: + // There is no SetEncoded, so unparsed_ is always from _InternalParse, + // which can't be empty. + ABSL_DCHECK(!unparsed_.IsEmpty()); + ABSL_DCHECK(raw.value() == nullptr || raw.value()->empty()); + return LogicalState::kParseRequired; + default: + ABSL_DCHECK(raw.status() == RawState::kIsParsed || + raw.status() == RawState::kIsParsedMaybeUninitialized); + ABSL_DCHECK(raw.value() != nullptr); + // Only other Initialized state was kParseError which is handled above. + if (unparsed_.IsEmpty()) { + return LogicalState::kDirty; + } + // Non-null message, unparsed exists. + return LogicalState::kNoParseRequired; + } + } + + private: + // Values that can be kept in `MessageState`'s status bits. + // TODO: Deduplicate with LazyField. + enum class RawState { + // `unparsed_` is empty. + // `message_` is either nullptr or an empty container. + kCleared, + + // `unparsed_` contains the canonical field data. + // `message_` points to the result of parsing that data. + // + // NOTE: serializing `message_` may produce different bytes than + // `unparsed_`, so care must be taken around issues of canonical or + // deterministic serialization. Generally, `unparsed_` should be preferred + // if it is not empty, as that is lower overhead. + kIsParsed, + + // IsParsed and may be uninitialized. See + // kNeedsParseMaybeUninitialized for details. + kIsParsedMaybeUninitialized, + + // TODO: add kIsParsedIgnoreUnparsed and + // kIsParsedIgnoreUnparsedMaybeUninitialized. + + // `message_` points to the result of parsing that data, but there was an + // error when parsing. Partially parsed `message_` is considered canonical + // to match eager fields. + kParseError, + + // `unparsed_` contains the field data. + // `message_` is either nullptr or an empty container. + kNeedsParse, + + // kNeedsParse and may be uninitialized. + // + // MaybeUninitialized is flagged in the verification and recorded to trigger + // eager parsing on IsInitialized() to be certain. + // + // Note that unverified data is assumed to be initialized (to support legacy + // cases) and treated as if it's verified to be initialized. Therefore, we + // need "MaybeUninitialized" rather than "Initialized". + kNeedsParseMaybeUninitialized, + + kMaxState = kNeedsParseMaybeUninitialized + }; + + class MessageState { + public: + constexpr explicit MessageState(RawState state) : raw_(ToUint32(state)) {} + MessageState(const RepeatedPtrFieldBase* message, RawState state) + : raw_(reinterpret_cast(message) | ToUint32(state)) { + ABSL_DCHECK_EQ(reinterpret_cast(message) & ToUint32(state), + 0u); + } + + const RepeatedPtrFieldBase* value() const { + return reinterpret_cast(raw_ & ~0b111); + } + + RepeatedPtrFieldBase* mutable_value() const { + return reinterpret_cast(raw_ & ~0b111); + } + + RawState status() const { return ToRawState(raw_ & 0b111); } + + void set_status(RawState status) { + raw_ &= ~0b111; + raw_ |= ToUint32(status); + } + + void set_value(const RepeatedPtrFieldBase* message) { + raw_ &= 0b111; + raw_ |= reinterpret_cast(message); + } + + static inline constexpr uint32_t ToUint32(RawState status) { + return static_cast(status); + } + static inline RawState ToRawState(uint32_t status) { + ABSL_DCHECK_LE(status, ToUint32(RawState::kMaxState)); + return static_cast(status); + } + + bool NeedsParse() const { + // kNeedsParse and kNeedsParseMaybeUninitialized must be 0 and 1 to make + // NeedsParse() check cheap. + static_assert( + RawState::kNeedsParseMaybeUninitialized == RawState::kMaxState, ""); + static_assert(ToUint32(RawState::kNeedsParseMaybeUninitialized) == + ToUint32(RawState::kNeedsParse) + 1, + ""); + return status() >= RawState::kNeedsParse; + } + + private: + uintptr_t raw_; + }; + + // TODO: Deduplicate. + template + class ByTemplate { + public: + // Only `Get()` needs access to the default element, but we don't want to + // force instantiation of `MessageType::default_instance()` because it + // doesn't exist in all configurations. + explicit ByTemplate() : ByTemplate(nullptr) {} + explicit ByTemplate(const MessageType* default_instance) + : default_instance_(default_instance) {} + + MessageLite* New(Arena* arena) const { + return reinterpret_cast( + Arena::DefaultConstruct(arena)); + } + + const MessageLite& Default() const { + ABSL_DCHECK(default_instance_ != nullptr); + return *reinterpret_cast(default_instance_); + } + + private: + const MessageType* default_instance_; + }; + + // Copy constructor on arena. + LazyRepeatedPtrField(Arena* arena, const LazyRepeatedPtrField& rhs, + Arena* rhs_arena); + + // Serialization methods. Note that WriteToCord may override/clear the + // given cord. + template + bool MergeFrom(const MessageLite* prototype, const Input& data, Arena* arena); + + private: + template + MessageState SharedInit(Strategy strategy, Arena* arena, + ParseContext* ctx) const { + auto old_raw = raw_.load(std::memory_order_acquire); + if (!old_raw.NeedsParse()) return old_raw; + MessageState new_raw = + // Transfer MaybeUninitialized state after a state transition. + DoParse(nullptr, strategy.Default(), arena, ctx, + old_raw.status() == RawState::kNeedsParseMaybeUninitialized); + if (raw_.compare_exchange_strong(old_raw, new_raw, + std::memory_order_release, + std::memory_order_acquire)) { + // We won the race. Dispose of the old message (if there was one). + if (arena == nullptr) { + delete reinterpret_cast*>( + old_raw.value()); + } + return new_raw; + } else { + // We lost the race, but someone else will have installed the new + // value. Dispose of the our attempt at installing. + if (arena == nullptr) { + delete reinterpret_cast*>( + new_raw.value()); + } + ABSL_DCHECK(!old_raw.NeedsParse()); + return old_raw; + } + } + + template + MessageState ExclusiveInitWithoutStore(Strategy strategy, Arena* arena, + ParseContext* ctx) { + auto old_raw = raw_.load(std::memory_order_relaxed); + if (!old_raw.NeedsParse() && old_raw.value() != nullptr) return old_raw; + if (old_raw.NeedsParse()) { + // Mutable messages need not transfer MaybeUninitialized. + return DoParse(old_raw.mutable_value(), strategy.Default(), arena, ctx, + false); + } + ABSL_DCHECK(old_raw.value() == nullptr); + return MessageState(Arena::Create(arena), + RawState::kIsParsed); + } + + template + const RepeatedPtrFieldBase* GetGeneric(Strategy strategy, Arena* arena, + ParseContext* ctx) const { + const auto* value = SharedInit(strategy, arena, ctx).value(); + if (value == nullptr) { + return reinterpret_cast(DefaultRawPtr()); + } + return value; + } + + template + RepeatedPtrFieldBase* MutableGeneric(Strategy strategy, Arena* arena, + ParseContext* ctx) { + auto raw = ExclusiveInitWithoutStore(strategy, arena, ctx); + unparsed_.Clear(); + ABSL_DCHECK(raw.value() != nullptr); + raw.set_status(RawState::kIsParsed); + raw_.store(raw, std::memory_order_relaxed); + return raw.mutable_value(); + } + + void SetNeedsParse() { + auto raw = raw_.load(std::memory_order_relaxed); + raw.set_status(RawState::kNeedsParse); + raw_.store(raw, std::memory_order_relaxed); + } + + void SetNeedsParseMaybeUninitialized() { + auto raw = raw_.load(std::memory_order_relaxed); + ABSL_DCHECK(raw.status() == RawState::kNeedsParse || + raw.status() == RawState::kNeedsParseMaybeUninitialized); + raw.set_status(RawState::kNeedsParseMaybeUninitialized); + raw_.store(raw, std::memory_order_relaxed); + } + + void SetParseNotRequiredMaybeUninitialized() { + auto raw = raw_.load(std::memory_order_relaxed); + ABSL_DCHECK(raw.status() == RawState::kIsParsed || + raw.status() == RawState::kIsParsedMaybeUninitialized); + raw.set_status(RawState::kIsParsedMaybeUninitialized); + raw_.store(raw, std::memory_order_relaxed); + } + + bool MaybeUninitialized() const { + auto raw = raw_.load(std::memory_order_relaxed); + if (raw.status() == RawState::kNeedsParseMaybeUninitialized) return true; + + // Make sure the logical state matches as well. + return raw.status() == RawState::kIsParsedMaybeUninitialized && + GetLogicalState() == LogicalState::kNoParseRequired; + } + + // Adds MaybeUninitialized state if "other" may be uninitialized. + void MergeMaybeUninitializedState(const LazyRepeatedPtrField& other); + + bool IsEagerSerializeSafe(const MessageLite* prototype, int32_t number, + Arena* arena) const; + + static void swap_atomics(std::atomic& lhs, + std::atomic& rhs); + + // Helper to enforce invariants when exclusive R/M/W access is required. + class ExclusiveTxn { + public: + explicit ExclusiveTxn(LazyRepeatedPtrField& lazy) + : lazy_(lazy), state_(lazy_.raw_.load(std::memory_order_relaxed)) {} + + RepeatedPtrFieldBase* mutable_value() { + // Any write to the message at this point should nuke unparsed_. + lazy_.unparsed_.Clear(); + return state_.mutable_value(); + } + + void Commit(RawState new_status) { + if (state_.status() != new_status) { + state_.set_status(new_status); + lazy_.raw_.store(state_, std::memory_order_relaxed); + } + } + + private: + LazyRepeatedPtrField& lazy_; + MessageState state_; + }; + + template + RawState PerformTransition(Transition fn) { + ExclusiveTxn txn(*this); + RawState new_state = fn(txn); + txn.Commit(new_state); + return new_state; + } + + public: + // Payload abstraction that can hold a raw char array or a Cord depending on + // how much data it needs to hold. + // The caller is responsible for managing the lifetime of the payload. + // TODO: Deduplicate with the LazyField::UnparsedPayload. + class UnparsedPayload { + enum Tag : uintptr_t { + kTagEmpty = 0, + kTagArray = 1, + kTagCord = 2, + + kTagBits = 3, + kRemoveMask = ~kTagBits, + }; + + public: + using ArraySizeType = uint16_t; + + // Visit the payload and calls the respective callback. The signatures are: + // - () for kUnset + // - (Cord&) for kCord + // - (absl::string_view) for kArray + // Returns the value returned by the callback. + template + auto Visit(UnsetF unset_f, CordF cord_f, ViewF view_f) const { + Tag t = tag(); + // Using ternary to allow for common-type implicit conversions. + return t == kTagEmpty ? unset_f() + : t == kTagArray ? view_f(AsStringView()) + : cord_f(AsCord()); + } + + Tag tag() const { return static_cast(value_ & kTagBits); } + + bool IsCord() const { + ABSL_DCHECK_EQ(static_cast(value_ & kTagCord), + static_cast(tag() == kTagCord)); + return (value_ & kTagCord) != 0u; + } + + bool IsArray() const { + ABSL_DCHECK_EQ(static_cast(value_ & kTagArray), + static_cast(tag() == kTagArray)); + return (value_ & kTagArray) != 0u; + } + + // Requires: IsCord() + absl::Cord& AsCord() const { + ABSL_DCHECK(IsCord()); + return *reinterpret_cast(value_ & kRemoveMask); + } + + // Return the payload as Cord regardless of the existing storage. + absl::Cord ForceAsCord() const { + return Visit([] { return absl::Cord(); }, // + [](const auto& c) { return c; }, + [](auto view) { return absl::Cord(view); }); + } + + // Similar to AsCord(), but if the payload is not already a Cord it will + // convert it first, maintaining existing bytes. + absl::Cord& UpgradeToCord(Arena* arena) { + if (IsCord()) return AsCord(); + absl::Cord new_cord(AsStringView()); + return InitAsCord(arena, std::move(new_cord)); + } + + // Requires: input array is the untagged value. + ArraySizeType GetArraySize(const char* array) const { + ABSL_DCHECK_EQ(array, reinterpret_cast(value_ - kTagArray)); + ArraySizeType size; + memcpy(&size, array, sizeof(size)); + return size; + } + + void SetArraySize(void* array, ArraySizeType size) const { + ABSL_DCHECK_EQ(array, reinterpret_cast(value_ - kTagArray)); + memcpy(array, &size, sizeof(ArraySizeType)); + } + + void SetArraySize(ArraySizeType size) const { + void* array = reinterpret_cast(value_ - kTagArray); + memcpy(array, &size, sizeof(ArraySizeType)); + } + + // Requires: !IsCord() + absl::string_view AsStringView() const { + switch (tag()) { + case kTagEmpty: + return {}; + + case kTagArray: { + const char* array = reinterpret_cast(value_ - kTagArray); + auto size = GetArraySize(array); + return absl::string_view(array + sizeof(size), size); + } + + default: + Unreachable(); + } + } + + // Clear the payload. After this call `Size()==0` and `IsEmpty()==true`, but + // it is not necessarily true that `tag()==kTagEmpty`. + // In particular, it keeps the Cord around in case it needs to be reused. + void Clear() { + switch (tag()) { + case kTagEmpty: + case kTagArray: + value_ = 0; + break; + default: + AsCord().Clear(); + break; + } + } + + // Destroys allocated memory if necessary. Does not reset the object. + void Destroy() { + if (IsCord()) delete &AsCord(); + } + + bool IsEmpty() const { + return Visit([] { return true; }, + [](const auto& cord) { return cord.empty(); }, + [](auto view) { + ABSL_DCHECK(!view.empty()); + return false; + }); + } + + size_t Size() const { + return Visit([] { return 0; }, + [](const auto& cord) { return cord.size(); }, + [](auto view) { return view.size(); }); + } + + // Sets the currently value as a Cord constructed from `args...`. + // It will clean up the existing value if necessary. + template + void SetCord(Arena* arena, Arg&& arg) { + if (IsCord()) { + // Reuse the existing cord. + AsCord() = std::forward(arg); + } else { + absl::Cord* cord = + Arena::Create(arena, std::forward(arg)); + value_ = reinterpret_cast(cord) | kTagCord; + } + } + + // Initialize the value as a Cord constructed from `args...` + // Ignores existing value. + template + absl::Cord& InitAsCord(Arena* arena, Args&&... args) { + auto* cord = + Arena::Create(arena, std::forward(args)...); + value_ = reinterpret_cast(cord) | kTagCord; + return *cord; + } + + // Initialize the value as an array copied from `view`. The tailing bytes + // are set to 0 to avoid UB. + // Ignores existing value. + void InitAndSetArray(Arena* arena, absl::string_view view) { + char* array = InitAsArray(arena, view.size()); + memcpy(array, view.data(), view.size()); + if (view.size() < kMaxArraySize) { + // Memset uninit data to avoid UB later. + memset(array + view.size(), '\0', kMaxArraySize - view.size()); + } + ABSL_DCHECK_EQ(view, AsStringView()); + } + + // Initialize the value as an array copied from `cord`. The tailing bytes + // are set to 0 to avoid UB. + // Ignores existing value. + void InitAndSetArray(Arena* arena, const absl::Cord& cord) { + auto size = cord.size(); + char* array = InitAsArray(arena, size); + cord.CopyToArray(array); + if (size < kMaxArraySize) { + // Memset uninit data to avoid UB later. + memset(array + size, '\0', kMaxArraySize - size); + } + } + + // Initialize the value as an array of size `size`. The payload bytes are + // uninitialized. + // Ignores existing value. + char* InitAsArray(Arena* arena, ArraySizeType size) { + ABSL_DCHECK(arena != nullptr); + // Allocate max allowed capacity. + // TODO: improve this to reduce waste when the size is small. + void* c = arena->AllocateAligned(kMaxArraySize + sizeof(ArraySizeType)); + ABSL_DCHECK_EQ(reinterpret_cast(c) & kTagBits, uintptr_t{0}); + value_ = reinterpret_cast(c) | kTagArray; + SetArraySize(c, size); + return static_cast(c) + sizeof(ArraySizeType); + } + + void AppendToArray(absl::string_view view) { + char* array = reinterpret_cast(value_ - kTagArray); + ArraySizeType size = GetArraySize(array); + char* c = array + sizeof(size) + size; + size += view.size(); + SetArraySize(array, size); + memcpy(c, view.data(), view.size()); + } + + void ZeroOutTailingBytes() { + char* array = reinterpret_cast(value_ - kTagArray); + auto size = GetArraySize(array); + if (size < kMaxArraySize) { + memset(array + sizeof(ArraySizeType) + size, '\0', + kMaxArraySize - size); + } + } + + size_t SpaceUsedExcludingSelf() const { + return Visit( + [] { return 0; }, + [](const auto& cord) { return cord.EstimatedMemoryUsage(); }, + [](auto view) { return kMaxArraySize + sizeof(ArraySizeType); }); + } + + void TransferHeapOwnershipToArena(Arena* arena) { + ABSL_DCHECK(tag() == kTagCord || tag() == kTagEmpty); + if (IsCord()) arena->Own(&AsCord()); + } + + private: + uintptr_t value_ = 0; + }; + + public: + static bool ParseWithOuterContext(RepeatedPtrFieldBase* value, + const absl::Cord& input, ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required); + static bool ParseWithOuterContext(RepeatedPtrFieldBase* value, + absl::string_view input, ParseContext* ctx, + const MessageLite* prototype, + bool set_missing_required); + + private: + // This method has to be below the definition of class UnparsedPayload due to + // the call to `unparsed_.Visit`. + // TODO: Deduplicate with LazyField. + MessageState DoParse(RepeatedPtrFieldBase* old, const MessageLite& prototype, + Arena* arena, ParseContext* ctx, + bool maybe_uninitialized) const { + auto* value = + (old == nullptr) ? Arena::Create(arena) : old; + if (!unparsed_.Visit( + [] { return true; }, + [&](const auto& cord) { + return ParseWithOuterContext(value, cord, ctx, &prototype, + maybe_uninitialized); + }, + [&](auto view) { + return ParseWithOuterContext(value, view, ctx, &prototype, + maybe_uninitialized); + })) { + // If this is called by eager verficiation, ctx != nullptr and logging + // parsing error in that case is likely redundant because the parsing will + // fail anyway. Users who care about parsing errors would have already + // checked the return value and others may find the error log unexpected. + // + // `ctx == nullptr` means it's not eagerly verified (e.g. unverified lazy) + // and logging in that case makes sense. + if (ctx == nullptr) { + LogParseError(value); + } + return MessageState(value, RawState::kParseError); + } + return MessageState(value, maybe_uninitialized + ? RawState::kIsParsedMaybeUninitialized + : RawState::kIsParsed); + } + + // Mutable because it is initialized lazily. + // A MessageState is a tagged RepeatedPtrFieldBase* + mutable std::atomic raw_; + + // NOT mutable because we keep the payload around until the message changes in + // some way. + UnparsedPayload unparsed_; + // absl::Cord will make copies on anything under this limit, so we might as + // well do the copies into our own buffer instead. + static constexpr size_t kMaxArraySize = 512; + static_assert(kMaxArraySize <= + std::numeric_limits::max()); + friend class ::google::protobuf::Arena; + friend class ::google::protobuf::Reflection; + friend class ExtensionSet; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + + // Logs a parsing error. + static void LogParseError(const RepeatedPtrFieldBase* value); + + bool IsAllocated() const { + return raw_.load(std::memory_order_acquire).value() != nullptr; + } + + // For testing purposes. + friend class LazyRepeatedPtrFieldTest; + friend class LazyRepeatedInMessageTest; + template + void OverwriteForTest(RawState status, const absl::Cord& unparsed, + RepeatedPtrField* value, Arena* arena); +}; + +inline LazyRepeatedPtrField::~LazyRepeatedPtrField() { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + delete reinterpret_cast*>(value); + unparsed_.Destroy(); +} + +// TODO: Deduplicate with LazyField. +inline const RepeatedPtrFieldBase* LazyRepeatedPtrField::TryGetRepeated() + const { + switch (GetLogicalState()) { + case LogicalState::kDirty: + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: + return raw_.load(std::memory_order_relaxed).value(); + case LogicalState::kClear: + case LogicalState::kClearExposed: + return nullptr; + } + internal::Unreachable(); + return nullptr; +} + +// ------------------------------------------------------------------- +// Testing stuff. + +// It's in the header due to the template. +// TODO: Deduplicate with LazyField. +template +void LazyRepeatedPtrField::OverwriteForTest(RawState status, + const absl::Cord& unparsed, + RepeatedPtrField* value, + Arena* arena) { + auto raw = raw_.load(std::memory_order_relaxed); + if (arena == nullptr) { + delete reinterpret_cast*>(raw.value()); + } + raw.set_value(reinterpret_cast(value)); + raw.set_status(status); + if (!unparsed.empty()) { + if (arena != nullptr && unparsed.size() <= kMaxArraySize) { + unparsed_.InitAndSetArray(arena, unparsed); + } else { + unparsed_.SetCord(arena, unparsed); + } + } + raw_.store(raw, std::memory_order_relaxed); +} + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" + +#endif // GOOGLE_PROTOBUF_LAZY_REPEATED_FIELD_H__ diff --git a/src/google/protobuf/lazy_repeated_field_heavy.cc b/src/google/protobuf/lazy_repeated_field_heavy.cc new file mode 100644 index 0000000000..1c4e1bf686 --- /dev/null +++ b/src/google/protobuf/lazy_repeated_field_heavy.cc @@ -0,0 +1,401 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include +#include +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/strings/cord.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/arena.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "google/protobuf/lazy_repeated_field.h" +#include "google/protobuf/message.h" +#include "google/protobuf/message_lite.h" +#include "google/protobuf/repeated_ptr_field.h" +#include "google/protobuf/wire_format_lite.h" + +namespace google { +namespace protobuf { +namespace internal { +namespace { + +class ByFactory { + public: + explicit ByFactory(const Descriptor* type, MessageFactory* factory) + : type_(type), factory_(factory) {} + + Message* New(Arena* arena) const { + return factory_->GetPrototype(type_)->New(arena); + } + + const Message& Default() const { return *factory_->GetPrototype(type_); } + + private: + const Descriptor* type_; + MessageFactory* factory_; +}; + +// Escape C++ trigraphs by escaping question marks to \? +std::string EscapeTrigraphs(absl::string_view to_escape) { + return absl::StrReplaceAll(to_escape, {{"?", "\\?"}}); +} + +std::string EscapeEncoded(absl::string_view encoded) { + std::string out; + out.reserve(encoded.size() * 2); + constexpr size_t kBytesPerLine = 25; + for (size_t i = 0; i < encoded.size(); i += kBytesPerLine) { + absl::StrAppend( + &out, "\"", + EscapeTrigraphs(absl::CEscape(encoded.substr(i, kBytesPerLine))), + "\"\n"); + } + return out; +} + +// Deterministic serialization is required to minimize false positives: e.g. +// ordering, redundant wire format data, etc. Such discrepancies are +// expected and tolerated. To prevent this serialization starts yet another +// consistency check, we should skip consistency-check. +std::string DeterministicSerialization(const google::protobuf::MessageLite& m) { + std::string result; + { + google::protobuf::io::StringOutputStream sink(&result); + google::protobuf::io::CodedOutputStream out(&sink); + out.SetSerializationDeterministic(true); + out.SkipCheckConsistency(); + m.SerializePartialToCodedStream(&out); + } + return result; +} + +// If LazyField is initialized, unparsed and message should be consistent. If +// a LazyField is mutated via const_cast, that may break. We should rather fail +// than silently propagate such discrepancy. Note that this aims to detect +// missing/added data. +void VerifyConsistency(LazyRepeatedPtrField::LogicalState state, + const RepeatedPtrFieldBase* value, + const MessageLite* prototype, const absl::Cord& unparsed, + io::EpsCopyOutputStream* stream) { +#ifndef NDEBUG + if (stream != nullptr && !stream->ShouldCheckConsistency()) return; + if (state != LazyRepeatedPtrField::LogicalState::kNoParseRequired) return; + + RepeatedPtrField unparsed_msg; + if (!LazyRepeatedPtrField::ParseWithOuterContext( + reinterpret_cast(&unparsed_msg), unparsed, + nullptr, prototype, /*set_missing_required=*/false)) { + // Bail out on parse failure as it can result in false positive + // inconsistency and ABSL_CHECK failure. Warn instead. + ABSL_LOG(WARNING) + << "Verify skipped due to parse falure: RepeatedPtrField of " + << prototype->GetTypeName(); + return; + } + + const auto* msgs = reinterpret_cast*>(value); + // Eagerly parse all lazy fields to eliminate non-canonical wireformat data. + for (int i = 0; i < msgs->size(); i++) { + // Clone a new one from the original to eagerly parse all lazy + // fields. + const auto& msg = msgs->Get(i); + std::unique_ptr clone(msg.New()); + clone->CopyFrom(msg); + EagerParseLazyFieldIgnoreUnparsed(*clone); + EagerParseLazyFieldIgnoreUnparsed(*unparsed_msg.Mutable(i)); + ABSL_DCHECK_EQ(DeterministicSerialization(*clone), + DeterministicSerialization(unparsed_msg.Get(i))) + << "RepeatedPtrField<" << msg.GetTypeName() << ">(" << i << ")" + << ": likely mutated via getters + const_cast\n" + << "unparsed:\n" + << EscapeEncoded(DeterministicSerialization(unparsed_msg.Get(i))) + << "\nmessage:\n" + << EscapeEncoded(DeterministicSerialization(*clone)); + } +#endif // !NDEBUG +} + +} // namespace + +LazyRepeatedPtrField::LazyRepeatedPtrField(Arena* arena, + const LazyRepeatedPtrField& rhs, + Arena* rhs_arena) + : raw_(MessageState(RawState::kCleared)) { + switch (rhs.GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + return; // Leave uninitialized / empty + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: { + rhs.unparsed_.Visit( + [] {}, // + [&](const auto& cord) { unparsed_.InitAsCord(arena, cord); }, + [&](auto view) { + if (arena == nullptr) { + unparsed_.InitAsCord(nullptr, view); + } else { + unparsed_.InitAndSetArray(arena, view); + } + }); + raw_.store( + MessageState(nullptr, rhs.MaybeUninitialized() + ? RawState::kNeedsParseMaybeUninitialized + : RawState::kNeedsParse), + std::memory_order_relaxed); + return; + } + case LogicalState::kDirty: { + MessageState state = rhs.raw_.load(std::memory_order_relaxed); + const auto* src = state.value(); + if (src->empty()) { + return; // Leave uninitialized / empty + } + // Retain the existing IsParsed or IsParsedMaybeUninitialized status. + // TODO: use copy construction. + auto new_state = state.status(); + auto* value = Arena::Create(arena); + // MergeFrom calls reserve. + value->MergeFrom(*src); + raw_.store(MessageState(value, new_state), std::memory_order_relaxed); + return; + } + } +} + +const RepeatedPtrFieldBase* LazyRepeatedPtrField::GetDynamic( + const Descriptor* type, MessageFactory* factory, Arena* arena) const { + return GetGeneric(ByFactory(type, factory), arena, nullptr); +} + +RepeatedPtrFieldBase* LazyRepeatedPtrField::MutableDynamic( + const Descriptor* type, MessageFactory* factory, Arena* arena) { + return MutableGeneric(ByFactory(type, factory), arena, nullptr); +} + +size_t LazyRepeatedPtrField::SpaceUsedExcludingSelfLong() const { + // absl::Cord::EstimatedMemoryUsage counts itself that should be excluded + // because sizeof(Cord) is already counted in self. + size_t total_size = unparsed_.SpaceUsedExcludingSelf(); + switch (GetLogicalState()) { + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + total_size += + value->SpaceUsedExcludingSelfLong>(); + } break; + case LogicalState::kClear: + case LogicalState::kParseRequired: + // We may have a `Message*` here, but we cannot safely access it + // because, a racing SharedInit could delete it out from under us. + // Other states in this structure are already passed kSharedInit and are + // thus safe. + break; // Nothing to add. + } + return total_size; +} + +template +bool LazyRepeatedPtrField::MergeFrom(const MessageLite* prototype, + const Input& data, Arena* arena) { + switch (GetLogicalState()) { + case LogicalState::kParseRequired: { + unparsed_.UpgradeToCord(arena).Append(data); + break; + } + case LogicalState::kClear: { + size_t num_bytes = data.size(); + ABSL_DCHECK(num_bytes > 0); + if (arena == nullptr || num_bytes > kMaxArraySize || unparsed_.IsCord()) { + unparsed_.SetCord(arena, data); + } else { + unparsed_.InitAndSetArray(arena, data); + } + SetNeedsParse(); + break; + } + + // Pointer was previously exposed merge into that object. + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kDirty: { + auto new_state = PerformTransition([&](ExclusiveTxn& txn) { + auto* value = txn.mutable_value(); + bool res = + ParseWithOuterContext(value, data, /*ctx=*/nullptr, prototype, + /*set_missing_required=*/false); + if (!res) { + LogParseError(value); + return RawState::kParseError; + } else { + return RawState::kIsParsed; + } + }); + return new_state == RawState::kIsParsed; + } + } + return true; +} + +void LazyRepeatedPtrField::MergeMaybeUninitializedState( + const LazyRepeatedPtrField& other) { + if (MaybeUninitialized() || !other.MaybeUninitialized()) return; + + switch (GetLogicalState()) { + case LogicalState::kParseRequired: + SetNeedsParseMaybeUninitialized(); + break; + case LogicalState::kNoParseRequired: + SetParseNotRequiredMaybeUninitialized(); + break; + default: + break; + } +} + +void LazyRepeatedPtrField::MergeFrom(const MessageLite* prototype, + const LazyRepeatedPtrField& other, + Arena* arena, Arena* other_arena) { +#ifndef NDEBUG + VerifyConsistency(other.GetLogicalState(), + other.raw_.load(std::memory_order_relaxed).value(), + prototype, other.unparsed_.ForceAsCord(), nullptr); +#endif // !NDEBUG + switch (other.GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + return; // Nothing to do. + + case LogicalState::kParseRequired: + case LogicalState::kNoParseRequired: + if (other.unparsed_.IsCord()) { + MergeFrom(prototype, other.unparsed_.AsCord(), arena); + } else { + MergeFrom(prototype, other.unparsed_.AsStringView(), arena); + } + MergeMaybeUninitializedState(other); + return; + + case LogicalState::kDirty: { + const auto* other_value = + other.raw_.load(std::memory_order_relaxed).value(); + if (other_value->empty()) { + return; + } + auto* value = MutableByPrototype(prototype, arena); + value->MergeFrom(*other_value); + // No need to merge uninitialized state. + ABSL_DCHECK(GetLogicalState() == LogicalState::kDirty); + return; + } + } +} + +uint8_t* LazyRepeatedPtrField::InternalWrite( + const MessageLite* prototype, int32_t number, uint8_t* target, + io::EpsCopyOutputStream* stream) const { +#ifndef NDEBUG + VerifyConsistency(GetLogicalState(), + raw_.load(std::memory_order_relaxed).value(), prototype, + unparsed_.ForceAsCord(), stream); +#endif // !NDEBUG + switch (GetLogicalState()) { + case LogicalState::kClear: + case LogicalState::kClearExposed: + case LogicalState::kNoParseRequired: + case LogicalState::kParseRequired: + // If deterministic is enabled then attempt to parse to a message which + // can then be serialized deterministically. (The serialized bytes may + // have been created undeterministically). + if (stream->IsSerializationDeterministic() && prototype != nullptr) { + RepeatedPtrField value; + // TODO: Test this path. + bool success = unparsed_.Visit( + [] { return true; }, + [&](const auto& cord) { + // `set_missing_required = false` to avoid checking require fields + // (simialr to Message::ParsePartial*). + return ParseWithOuterContext( + reinterpret_cast(&value), cord, + /*ctx=*/nullptr, prototype, /*set_missing_required=*/false); + }, + [&](auto view) { + return ParseWithOuterContext( + reinterpret_cast(&value), view, + /*ctx=*/nullptr, prototype, false); + }); + if (success) { + size_t tag_size = WireFormatLite::TagSize( + number, WireFormatLite::FieldType::TYPE_MESSAGE); + auto count = tag_size * value.size(); + for (int i = 0; i < value.size(); i++) { + count += WireFormatLite::LengthDelimitedSize( + value.Get(i).ByteSizeLong()); + } + + // Serialization takes place in two phases: + // 1) Figure out the expected number of bytes (e.g. ByteSizeLong() on + // the container message) 2) InternalWrite the bytes. + // + // There is a golden contract that the # of bytes written matches + // the returned value from the first step. + // + // In this case unparsed_ was used as the source of truth for the + // number of bytes. There are some known cases where the number of + // serialized bytes is different than the number of bytes written + // by a message parsed from the serialized bytes. For example if the + // serialized representation contained multiple entries for the same + // non-repeated field the duplicates are removed upon parsing. + // + // If this (relatively rare) case is hit then there is no choice + // but to serialize the original unparsed bytes; otherwise the + // golden contract is broken. + // It's possible for the size to change if the unparsed_ was not + // canonical, for example it can have repeated entries for the same + // tag (this is more common then you would think). + if (count == unparsed_.Size()) { + for (int i = 0, n = value.size(); i < n; i++) { + const auto& repfield = value.Get(i); + target = WireFormatLite::InternalWriteMessage( + number, repfield, repfield.GetCachedSize(), target, stream); + } + return target; + } + } + } + return unparsed_.Visit( + [&] { return target; }, + [&](const auto& cord) { return stream->WriteCord(cord, target); }, + [&](auto view) { + return stream->WriteRaw(view.data(), view.size(), target); + }); + case LogicalState::kDirty: { + const auto* value = raw_.load(std::memory_order_relaxed).value(); + for (int i = 0, n = value->size(); i < n; i++) { + const auto& repfield = value->Get>(i); + target = WireFormatLite::InternalWriteMessage( + number, repfield, repfield.GetCachedSize(), target, stream); + } + return target; + } + } + // Required for certain compiler configurations. + ABSL_LOG(FATAL) << "Not reachable"; + return nullptr; +} + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/lite_unittest.cc b/src/google/protobuf/lite_unittest.cc index bd5f5e7926..be4c428a59 100644 --- a/src/google/protobuf/lite_unittest.cc +++ b/src/google/protobuf/lite_unittest.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,7 @@ #include "google/protobuf/unittest_lite.pb.h" #include "google/protobuf/wire_format_lite.h" + // Must be included last #include "google/protobuf/port_def.inc" @@ -1164,6 +1166,7 @@ TYPED_TEST(LiteTest, EnumValueToName) { EXPECT_EQ("", protobuf_unittest::ForeignEnumLite_Name(999)); } + TYPED_TEST(LiteTest, NestedEnumValueToName) { EXPECT_EQ("FOO", protobuf_unittest::TestAllTypesLite::NestedEnum_Name( protobuf_unittest::TestAllTypesLite::FOO)); diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 0953b12d00..74cfa9d439 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -62,6 +62,10 @@ class MapIterator; template struct is_proto_enum; +namespace rust { +struct PtrAndLen; +} // namespace rust + namespace internal { template class MapFieldLite; @@ -590,6 +594,7 @@ class PROTOBUF_EXPORT UntypedMapBase { friend struct MapTestPeer; friend struct MapBenchmarkPeer; friend class UntypedMapIterator; + friend class RustMapHelper; struct NodeAndBucket { NodeBase* node; @@ -924,6 +929,7 @@ class KeyMapBase : public UntypedMapBase { friend class TcParser; friend struct MapTestPeer; friend struct MapBenchmarkPeer; + friend class RustMapHelper; PROTOBUF_NOINLINE void erase_no_destroy(map_index_t b, KeyNode* node) { TreeIterator tree_it; @@ -1139,6 +1145,60 @@ bool InitializeMapKey(T*, K&&, Arena*) { } +// The purpose of this class is to give the Rust implementation visibility into +// some of the internals of C++ proto maps. We need access to these internals +// to be able to implement Rust map operations without duplicating the same +// functionality for every message type. +class RustMapHelper { + public: + using NodeAndBucket = UntypedMapBase::NodeAndBucket; + using ClearInput = UntypedMapBase::ClearInput; + + template + static constexpr MapNodeSizeInfoT SizeInfo() { + return Map::Node::size_info(); + } + + enum { + kKeyIsString = UntypedMapBase::kKeyIsString, + kValueIsProto = UntypedMapBase::kValueIsProto, + }; + + static NodeBase* AllocNode(UntypedMapBase* m, MapNodeSizeInfoT size_info) { + return m->AllocNode(size_info); + } + + static void DeallocNode(UntypedMapBase* m, NodeBase* node, + MapNodeSizeInfoT size_info) { + return m->DeallocNode(node, size_info); + } + + template + static NodeAndBucket FindHelper(Map* m, Key key) { + return m->FindHelper(key); + } + + template + static typename Map::KeyNode* InsertOrReplaceNode(Map* m, NodeBase* node) { + return m->InsertOrReplaceNode(static_cast(node)); + } + + template + static void EraseNoDestroy(Map* m, map_index_t bucket, NodeBase* node) { + m->erase_no_destroy(bucket, static_cast(node)); + } + + static void DestroyMessage(MessageLite* m) { m->DestroyInstance(false); } + + static void ClearTable(UntypedMapBase* m, ClearInput input) { + m->ClearTable(input); + } + + static bool IsGlobalEmptyTable(const UntypedMapBase* m) { + return m->num_buckets_ == kGlobalEmptyTableSize; + } +}; + } // namespace internal // This is the class for Map's internal value_type. @@ -1252,6 +1312,11 @@ class Map : private internal::KeyMapBase> { internal::is_internal_map_value_type>::value, "We only support scalar, Message, and designated internal " "mapped types."); + // The Rust implementation that wraps C++ protos relies on the ability to + // create an UntypedMapBase and cast a pointer of it to google::protobuf::Map*. + static_assert( + sizeof(Map) == sizeof(internal::UntypedMapBase), + "Map must not have any data members beyond what is in UntypedMapBase."); } template @@ -1702,6 +1767,7 @@ class Map : private internal::KeyMapBase> { friend class internal::TcParser; friend struct internal::MapTestPeer; friend struct internal::MapBenchmarkPeer; + friend class internal::RustMapHelper; }; namespace internal { diff --git a/src/google/protobuf/map_entry.h b/src/google/protobuf/map_entry.h index 71dda48b9f..8704b63286 100644 --- a/src/google/protobuf/map_entry.h +++ b/src/google/protobuf/map_entry.h @@ -116,7 +116,7 @@ class MapEntry : public Message { // sharing easier. struct { HasBits<1> _has_bits_{}; - mutable CachedSize _cached_size_{}; + CachedSize _cached_size_{}; KeyOnMemory key_{KeyTypeHandler::Constinit()}; ValueOnMemory value_{ValueTypeHandler::Constinit()}; diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index 27c82f7869..f60818f424 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -185,7 +185,7 @@ size_t Message::ByteSizeLong() const { #endif // !PROTOBUF_CUSTOM_VTABLE size_t Message::ComputeUnknownFieldsSize( - size_t total_size, internal::CachedSize* cached_size) const { + size_t total_size, const internal::CachedSize* cached_size) const { total_size += WireFormat::ComputeUnknownFieldsSize( _internal_metadata_.unknown_fields( UnknownFieldSet::default_instance)); @@ -194,7 +194,7 @@ size_t Message::ComputeUnknownFieldsSize( } size_t Message::MaybeComputeUnknownFieldsSize( - size_t total_size, internal::CachedSize* cached_size) const { + size_t total_size, const internal::CachedSize* cached_size) const { if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { return ComputeUnknownFieldsSize(total_size, cached_size); } diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 45eeea451c..85bb049a30 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -383,10 +383,10 @@ class PROTOBUF_EXPORT Message : public MessageLite { // For CODE_SIZE types static bool IsInitializedImpl(const MessageLite&); - size_t ComputeUnknownFieldsSize(size_t total_size, - internal::CachedSize* cached_size) const; - size_t MaybeComputeUnknownFieldsSize(size_t total_size, - internal::CachedSize* cached_size) const; + size_t ComputeUnknownFieldsSize( + size_t total_size, const internal::CachedSize* cached_size) const; + size_t MaybeComputeUnknownFieldsSize( + size_t total_size, const internal::CachedSize* cached_size) const; // Reflection based version for reflection based types. static absl::string_view GetTypeNameImpl(const ClassData* data); diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 22fe7d9634..9abbf76576 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -64,23 +64,6 @@ void MessageLite::CheckTypeAndMergeFrom(const MessageLite& other) { data->merge_to_from(*this, other); } -#if defined(PROTOBUF_CUSTOM_VTABLE) -uint8_t* MessageLite::_InternalSerialize( - uint8_t* ptr, io::EpsCopyOutputStream* stream) const { - return _class_data_->serialize(*this, ptr, stream); -} - -MessageLite* MessageLite::New(Arena* arena) const { - return static_cast(_class_data_->new_message(this, arena)); -} - -void MessageLite::Clear() { _class_data_->clear(*this); } - -size_t MessageLite::ByteSizeLong() const { - return _class_data_->byte_size_long(*this); -} -#endif // PROTOBUF_CUSTOM_VTABLE - bool MessageLite::IsInitialized() const { auto* data = GetClassData(); return data->is_initialized != nullptr ? data->is_initialized(*this) : true; @@ -137,15 +120,9 @@ std::string MessageLite::DebugString() const { return absl::StrCat("MessageLite at 0x", absl::Hex(this)); } +#if !defined(PROTOBUF_CUSTOM_VTABLE) int MessageLite::GetCachedSize() const { return AccessCachedSize().Get(); } - -internal::CachedSize& MessageLite::AccessCachedSize() const { - auto* data = GetClassData(); - ABSL_DCHECK(data != nullptr); - ABSL_DCHECK(data->cached_size_offset != 0); - return *reinterpret_cast(const_cast( - reinterpret_cast(this) + data->cached_size_offset)); -} +#endif namespace { diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index cae4e6bbef..94e18c1284 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -88,6 +88,7 @@ class PROTOBUF_EXPORT CachedSize { constexpr CachedSize() noexcept : atom_(Scalar{}) {} // NOLINTNEXTLINE(google-explicit-constructor) constexpr CachedSize(Scalar desired) noexcept : atom_(desired) {} + #if PROTOBUF_BUILTIN_ATOMIC constexpr CachedSize(const CachedSize& other) = default; @@ -95,7 +96,16 @@ class PROTOBUF_EXPORT CachedSize { return __atomic_load_n(&atom_, __ATOMIC_RELAXED); } - void Set(Scalar desired) noexcept { + void Set(Scalar desired) const noexcept { + // Avoid writing the value when it is zero. This prevents writing to gloabl + // default instances, which might be in readonly memory. + if (ABSL_PREDICT_FALSE(desired == 0)) { + if (Get() == 0) return; + } + __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); + } + + void SetNonZero(Scalar desired) const noexcept { __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); } #else @@ -109,16 +119,25 @@ class PROTOBUF_EXPORT CachedSize { return atom_.load(std::memory_order_relaxed); } - void Set(Scalar desired) noexcept { + void Set(Scalar desired) const noexcept { + // Avoid writing the value when it is zero. This prevents writing to gloabl + // default instances, which might be in readonly memory. + if (ABSL_PREDICT_FALSE(desired == 0)) { + if (Get() == 0) return; + } + atom_.store(desired, std::memory_order_relaxed); + } + + void SetNonZero(Scalar desired) const noexcept { atom_.store(desired, std::memory_order_relaxed); } #endif private: #if PROTOBUF_BUILTIN_ATOMIC - Scalar atom_; + mutable Scalar atom_; #else - std::atomic atom_; + mutable std::atomic atom_; #endif }; @@ -139,6 +158,7 @@ class TcParser; struct TcParseTableBase; class WireFormatLite; class WeakFieldMap; +class RustMapHelper; template class GenericTypeHandler; // defined in repeated_field.h @@ -229,7 +249,9 @@ class PROTOBUF_EXPORT MessageLite { // Construct a new instance on the arena. Ownership is passed to the caller // if arena is a nullptr. #if defined(PROTOBUF_CUSTOM_VTABLE) - MessageLite* New(Arena* arena) const; + MessageLite* New(Arena* arena) const { + return static_cast(_class_data_->new_message(this, arena)); + } #else virtual MessageLite* New(Arena* arena) const = 0; #endif // PROTOBUF_CUSTOM_VTABLE @@ -246,7 +268,7 @@ class PROTOBUF_EXPORT MessageLite { // will likely be needed again, so the memory used may not be freed. // To ensure that all memory used by a Message is freed, you must delete it. #if defined(PROTOBUF_CUSTOM_VTABLE) - void Clear(); + void Clear() { _class_data_->clear(*this); } #else virtual void Clear() = 0; #endif // PROTOBUF_CUSTOM_VTABLE @@ -470,7 +492,7 @@ class PROTOBUF_EXPORT MessageLite { // ByteSizeLong() is generally linear in the number of fields defined for the // proto. #if defined(PROTOBUF_CUSTOM_VTABLE) - size_t ByteSizeLong() const; + size_t ByteSizeLong() const { return _class_data_->byte_size_long(*this); } #else virtual size_t ByteSizeLong() const = 0; #endif // PROTOBUF_CUSTOM_VTABLE @@ -512,7 +534,11 @@ class PROTOBUF_EXPORT MessageLite { // sub-message is changed, all of its parents' cached sizes would need to be // invalidated, which is too much work for an otherwise inlined setter // method.) +#if defined(PROTOBUF_CUSTOM_VTABLE) + int GetCachedSize() const { return AccessCachedSize().Get(); } +#else int GetCachedSize() const; +#endif const char* _InternalParse(const char* ptr, internal::ParseContext* ctx); @@ -770,7 +796,11 @@ class PROTOBUF_EXPORT MessageLite { // Return the cached size object as described by // ClassData::cached_size_offset. - internal::CachedSize& AccessCachedSize() const; + const internal::CachedSize& AccessCachedSize() const { + return *reinterpret_cast( + reinterpret_cast(this) + + GetClassData()->cached_size_offset); + } public: enum ParseFlags { @@ -791,7 +821,9 @@ class PROTOBUF_EXPORT MessageLite { // uint8_t* _InternalSerialize(uint8_t* ptr) const; #if defined(PROTOBUF_CUSTOM_VTABLE) uint8_t* _InternalSerialize(uint8_t* ptr, - io::EpsCopyOutputStream* stream) const; + io::EpsCopyOutputStream* stream) const { + return _class_data_->serialize(*this, ptr, stream); + } #else // PROTOBUF_CUSTOM_VTABLE virtual uint8_t* _InternalSerialize( uint8_t* ptr, io::EpsCopyOutputStream* stream) const = 0; @@ -826,6 +858,7 @@ class PROTOBUF_EXPORT MessageLite { friend class internal::UntypedMapBase; friend class internal::WeakFieldMap; friend class internal::WireFormatLite; + friend class internal::RustMapHelper; template friend class Arena::InternalHelper; diff --git a/src/google/protobuf/no_field_presence_test.cc b/src/google/protobuf/no_field_presence_test.cc index 56586731c7..a5192ca502 100644 --- a/src/google/protobuf/no_field_presence_test.cc +++ b/src/google/protobuf/no_field_presence_test.cc @@ -8,7 +8,10 @@ #include #include "google/protobuf/descriptor.pb.h" +#include #include +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/unittest.pb.h" #include "google/protobuf/unittest_no_field_presence.pb.h" @@ -17,6 +20,8 @@ namespace google { namespace protobuf { namespace { +using ::testing::StrEq; + // Helper: checks that all fields have default (zero/empty) values. void CheckDefaultValues( const proto2_nofieldpresence_unittest::TestAllTypes& m) { @@ -457,6 +462,74 @@ TEST(NoFieldPresenceTest, MergeFromIfNonzeroTest) { EXPECT_EQ("test2", dest.optional_string()); } +TEST(NoFieldPresenceTest, ExtraZeroesInWireParseTest) { + // check extra serialized zeroes on the wire are parsed into the object. + proto2_nofieldpresence_unittest::ForeignMessage dest; + dest.set_c(42); + ASSERT_EQ(42, dest.c()); + + // ExplicitForeignMessage has the same fields as ForeignMessage, but with + // explicit presence instead of implicit presence. + proto2_nofieldpresence_unittest::ExplicitForeignMessage source; + source.set_c(0); + std::string wire = source.SerializeAsString(); + ASSERT_THAT(wire, StrEq(absl::string_view{"\x08\x00", 2})); + + // The "parse" operation clears all fields before merging from wire. + ASSERT_TRUE(dest.ParseFromString(wire)); + EXPECT_EQ(0, dest.c()); + std::string dest_data; + EXPECT_TRUE(dest.SerializeToString(&dest_data)); + EXPECT_TRUE(dest_data.empty()); +} + +TEST(NoFieldPresenceTest, ExtraZeroesInWireMergeTest) { + // check explicit zeros on the wire are merged into an implicit one. + proto2_nofieldpresence_unittest::ForeignMessage dest; + dest.set_c(42); + ASSERT_EQ(42, dest.c()); + + // ExplicitForeignMessage has the same fields as ForeignMessage, but with + // explicit presence instead of implicit presence. + proto2_nofieldpresence_unittest::ExplicitForeignMessage source; + source.set_c(0); + std::string wire = source.SerializeAsString(); + ASSERT_THAT(wire, StrEq(absl::string_view{"\x08\x00", 2})); + + // TODO: b/356132170 -- Add conformance tests to ensure this behaviour is + // well-defined. + // As implemented, the C++ "merge" operation does not distinguish between + // implicit and explicit fields when reading from the wire. + ASSERT_TRUE(dest.MergeFromString(wire)); + // If zero is present on the wire, the original value is overwritten, even + // though this is specified as an "implicit presence" field. + EXPECT_EQ(0, dest.c()); + std::string dest_data; + EXPECT_TRUE(dest.SerializeToString(&dest_data)); + EXPECT_TRUE(dest_data.empty()); +} + +TEST(NoFieldPresenceTest, ExtraZeroesInWireLastWins) { + // check that, when the same field is present multiple times on the wire, we + // always take the last one -- even if it is a zero. + + absl::string_view wire{"\x08\x01\x08\x00", /*len=*/4}; // note the null-byte. + proto2_nofieldpresence_unittest::ForeignMessage dest; + + // TODO: b/356132170 -- Add conformance tests to ensure this behaviour is + // well-defined. + // As implemented, the C++ "merge" operation does not distinguish between + // implicit and explicit fields when reading from the wire. + ASSERT_TRUE(dest.MergeFromString(wire)); + // If the same field is present multiple times on the wire, "last one wins". + // i.e. -- the last seen field content will always overwrite, even if it's + // zero and the field is implicit presence. + EXPECT_EQ(0, dest.c()); + std::string dest_data; + EXPECT_TRUE(dest.SerializeToString(&dest_data)); + EXPECT_TRUE(dest_data.empty()); +} + TEST(NoFieldPresenceTest, IsInitializedTest) { // Check that IsInitialized works properly. proto2_nofieldpresence_unittest::TestProto2Required message; diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 151c0b6cfc..5ff41a4c15 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -56,6 +56,7 @@ namespace google { namespace protobuf { class Message; +class UnknownField; // For the allowlist namespace internal { @@ -141,6 +142,7 @@ class RepeatedField final absl::disjunction, internal::is_supported_floating_point_type, std::is_same, + std::is_same, is_proto_enum>::value, "We only support non-string scalars in RepeatedField."); } @@ -387,9 +389,9 @@ class RepeatedField final // Reserves space to expand the field to at least the given size. // If the array is grown, it will always be at least doubled in size. // If `annotate_size` is true (the default), then this function will annotate - // the old container from `old_size` to `capacity_` (unpoison memory) + // the old container from `old_size` to `Capacity()` (unpoison memory) // directly before it is being released, and annotate the new container from - // `capacity_` to `old_size` (poison unused memory). + // `Capacity()` to `old_size` (poison unused memory). void Grow(int old_size, int new_size); void GrowNoAnnotate(int old_size, int new_size); @@ -410,10 +412,10 @@ class RepeatedField final } } - // Replaces size_ with new_size and returns the previous value of - // size_. This function is intended to be the only place where - // size_ is modified, with the exception of `AddInputIterator()` - // where the size of added items is not known in advance. + // Replaces size with new_size and returns the previous value of size. This + // function is intended to be the only place where size is modified, with the + // exception of `AddInputIterator()` where the size of added items is not + // known in advance. inline int ExchangeCurrentSize(int new_size) { const int prev_size = size(); AnnotateSize(prev_size, new_size); @@ -422,7 +424,7 @@ class RepeatedField final } // Returns a pointer to elements array. - // pre-condition: the array must have been allocated. + // pre-condition: Capacity() > 0. Element* elements() const { ABSL_DCHECK_GT(Capacity(), 0); // Because of above pre-condition this cast is safe. @@ -612,12 +614,14 @@ inline Element* RepeatedField::AddNAlreadyReserved(int n) template inline void RepeatedField::Resize(int new_size, const Element& value) { ABSL_DCHECK_GE(new_size, 0); - if (new_size > size()) { - if (new_size > Capacity()) Grow(size(), new_size); + const int old_size = size(); + if (new_size > old_size) { + if (new_size > Capacity()) Grow(old_size, new_size); Element* first = elements() + ExchangeCurrentSize(new_size); - std::uninitialized_fill(first, elements() + size(), value); - } else if (new_size < size()) { - Destroy(unsafe_elements() + new_size, unsafe_elements() + size()); + std::uninitialized_fill(first, elements() + new_size, value); + } else if (new_size < old_size) { + Element* elem = unsafe_elements(); + Destroy(elem + new_size, elem + old_size); ExchangeCurrentSize(new_size); } } @@ -656,21 +660,20 @@ inline Element* RepeatedField::Mutable(int index) template inline void RepeatedField::Set(int index, const Element& value) { - ABSL_DCHECK_GE(index, 0); - ABSL_DCHECK_LT(index, size()); - elements()[index] = value; + *Mutable(index) = value; } template inline void RepeatedField::Add(Element value) { + const int old_size = size(); int capacity = Capacity(); Element* elem = unsafe_elements(); - if (ABSL_PREDICT_FALSE(size() == capacity)) { - Grow(size(), size() + 1); + if (ABSL_PREDICT_FALSE(old_size == capacity)) { + Grow(old_size, old_size + 1); capacity = Capacity(); elem = unsafe_elements(); } - int new_size = size() + 1; + int new_size = old_size + 1; void* p = elem + ExchangeCurrentSize(new_size); ::new (p) Element(std::move(value)); @@ -682,21 +685,23 @@ inline void RepeatedField::Add(Element value) { template inline Element* RepeatedField::Add() ABSL_ATTRIBUTE_LIFETIME_BOUND { - if (ABSL_PREDICT_FALSE(size() == Capacity())) { - Grow(size(), size() + 1); + const int old_size = size(); + if (ABSL_PREDICT_FALSE(old_size == Capacity())) { + Grow(old_size, old_size + 1); } - void* p = unsafe_elements() + ExchangeCurrentSize(size() + 1); + void* p = unsafe_elements() + ExchangeCurrentSize(old_size + 1); return ::new (p) Element; } template template inline void RepeatedField::AddForwardIterator(Iter begin, Iter end) { + const int old_size = size(); int capacity = Capacity(); Element* elem = unsafe_elements(); - int new_size = size() + static_cast(std::distance(begin, end)); + int new_size = old_size + static_cast(std::distance(begin, end)); if (ABSL_PREDICT_FALSE(new_size > capacity)) { - Grow(size(), new_size); + Grow(old_size, new_size); elem = unsafe_elements(); capacity = Capacity(); } @@ -711,24 +716,27 @@ inline void RepeatedField::AddForwardIterator(Iter begin, Iter end) { template template inline void RepeatedField::AddInputIterator(Iter begin, Iter end) { - Element* first = unsafe_elements() + size(); - Element* last = unsafe_elements() + Capacity(); + Element* elem = unsafe_elements(); + Element* first = elem + size(); + Element* last = elem + Capacity(); AnnotateSize(size(), Capacity()); while (begin != end) { if (ABSL_PREDICT_FALSE(first == last)) { - int size = first - unsafe_elements(); + int size = first - elem; GrowNoAnnotate(size, size + 1); - first = unsafe_elements() + size; - last = unsafe_elements() + Capacity(); + elem = unsafe_elements(); + first = elem + size; + last = elem + Capacity(); } ::new (static_cast(first)) Element(*begin); ++begin; ++first; } - set_size(first - unsafe_elements()); - AnnotateSize(Capacity(), size()); + const int new_size = first - elem; + set_size(new_size); + AnnotateSize(Capacity(), new_size); } template @@ -745,9 +753,10 @@ inline void RepeatedField::Add(Iter begin, Iter end) { template inline void RepeatedField::RemoveLast() { - ABSL_DCHECK_GT(size(), 0); - elements()[size() - 1].~Element(); - ExchangeCurrentSize(size() - 1); + const int old_size = size(); + ABSL_DCHECK_GT(old_size, 0); + elements()[old_size - 1].~Element(); + ExchangeCurrentSize(old_size - 1); } template @@ -755,33 +764,38 @@ void RepeatedField::ExtractSubrange(int start, int num, Element* elements) { ABSL_DCHECK_GE(start, 0); ABSL_DCHECK_GE(num, 0); - ABSL_DCHECK_LE(start + num, size()); + const int old_size = size(); + ABSL_DCHECK_LE(start + num, old_size); + Element* elem = unsafe_elements(); // Save the values of the removed elements if requested. if (elements != nullptr) { - for (int i = 0; i < num; ++i) elements[i] = Get(i + start); + for (int i = 0; i < num; ++i) elements[i] = std::move(elem[i + start]); } // Slide remaining elements down to fill the gap. if (num > 0) { - for (int i = start + num; i < size(); ++i) Set(i - num, Get(i)); - Truncate(size() - num); + for (int i = start + num; i < old_size; ++i) + elem[i - num] = std::move(elem[i]); + Truncate(old_size - num); } } template inline void RepeatedField::Clear() { - Destroy(unsafe_elements(), unsafe_elements() + size()); + Element* elem = unsafe_elements(); + Destroy(elem, elem + size()); ExchangeCurrentSize(0); } template inline void RepeatedField::MergeFrom(const RepeatedField& other) { ABSL_DCHECK_NE(&other, this); - if (auto size = other.size()) { - Reserve(this->size() + size); - Element* dst = elements() + ExchangeCurrentSize(this->size() + size); - UninitializedCopyN(other.elements(), size, dst); + if (auto other_size = other.size()) { + const int old_size = size(); + Reserve(old_size + other_size); + Element* dst = elements() + ExchangeCurrentSize(old_size + other_size); + UninitializedCopyN(other.elements(), other_size, dst); } } @@ -905,8 +919,8 @@ RepeatedField::cend() const ABSL_ATTRIBUTE_LIFETIME_BOUND { template inline size_t RepeatedField::SpaceUsedExcludingSelfLong() const { - return Capacity() > 0 ? (Capacity() * sizeof(Element) + kHeapRepHeaderSize) - : 0; + const int capacity = Capacity(); + return capacity > 0 ? capacity * sizeof(Element) + kHeapRepHeaderSize : 0; } namespace internal { @@ -1016,9 +1030,11 @@ PROTOBUF_NOINLINE void RepeatedField::Grow(int old_size, template inline void RepeatedField::Truncate(int new_size) { - ABSL_DCHECK_LE(new_size, size()); - if (new_size < size()) { - Destroy(unsafe_elements() + new_size, unsafe_elements() + size()); + const int old_size = size(); + ABSL_DCHECK_LE(new_size, old_size); + if (new_size < old_size) { + Element* elem = unsafe_elements(); + Destroy(elem + new_size, elem + old_size); ExchangeCurrentSize(new_size); } } diff --git a/src/google/protobuf/repeated_field_unittest.cc b/src/google/protobuf/repeated_field_unittest.cc index 4b4d328111..50e43f594e 100644 --- a/src/google/protobuf/repeated_field_unittest.cc +++ b/src/google/protobuf/repeated_field_unittest.cc @@ -1241,8 +1241,8 @@ TEST(RepeatedField, HardenAgainstBadTruncate) { for (int size = 0; size < 10; ++size) { field.Truncate(size); #if GTEST_HAS_DEATH_TEST - EXPECT_DEBUG_DEATH(field.Truncate(size + 1), "new_size <= size"); - EXPECT_DEBUG_DEATH(field.Truncate(size + 2), "new_size <= size"); + EXPECT_DEBUG_DEATH(field.Truncate(size + 1), "new_size <= old_size"); + EXPECT_DEBUG_DEATH(field.Truncate(size + 2), "new_size <= old_size"); #elif defined(NDEBUG) field.Truncate(size + 1); field.Truncate(size + 1); diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index 9dfa8e8801..fcd99edc1b 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -576,6 +576,7 @@ class PROTOBUF_EXPORT RepeatedPtrFieldBase { // subclass. friend class google::protobuf::Reflection; friend class internal::SwapFieldHelper; + friend class LazyRepeatedPtrField; // Concrete Arena enabled copy function used to copy messages instances. // This follows the `Arena::CopyConstruct` signature so that the compiler diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 4f082b546e..806d61478c 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -249,7 +249,7 @@ class PROTOBUF_EXPORT SourceContext final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const SourceContext& from_msg); ::google::protobuf::internal::ArenaStringPtr file_name_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index e77dafad1c..9fceeb3759 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -295,7 +295,7 @@ class PROTOBUF_EXPORT ListValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const ListValue& from_msg); ::google::protobuf::RepeatedPtrField< ::google::protobuf::Value > values_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -489,7 +489,7 @@ class PROTOBUF_EXPORT Struct final : public ::google::protobuf::Message ::google::protobuf::internal::WireFormatLite::TYPE_STRING, ::google::protobuf::internal::WireFormatLite::TYPE_MESSAGE> fields_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -820,7 +820,7 @@ class PROTOBUF_EXPORT Value final : public ::google::protobuf::Message ::google::protobuf::Struct* struct_value_; ::google::protobuf::ListValue* list_value_; } kind_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::uint32_t _oneof_case_[1]; PROTOBUF_TSAN_DECLARE_MEMBER }; diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 161829f1c6..1b9cc67ae9 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -255,7 +255,7 @@ class PROTOBUF_EXPORT Timestamp final : public ::google::protobuf::Message const Timestamp& from_msg); ::int64_t seconds_; ::int32_t nanos_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index fbaae9d642..dbf5dc060c 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -793,8 +793,7 @@ PROTOBUF_NOINLINE void Type::Clear() { } { // .google.protobuf.SourceContext source_context = 5; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.source_context_); @@ -1679,8 +1678,7 @@ PROTOBUF_NOINLINE void Enum::Clear() { } { // .google.protobuf.SourceContext source_context = 4; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.source_context_); @@ -2269,8 +2267,7 @@ PROTOBUF_NOINLINE void Option::Clear() { } { // .google.protobuf.Any value = 2; - cached_has_bits = - this_._impl_._has_bits_[0]; + cached_has_bits = this_._impl_._has_bits_[0]; if (cached_has_bits & 0x00000001u) { total_size += 1 + ::google::protobuf::internal::WireFormatLite::MessageSize(*this_._impl_.value_); diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index e555ee56fe..21ecb391f6 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -399,7 +399,7 @@ class PROTOBUF_EXPORT Option final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Option& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::internal::ArenaStringPtr name_; ::google::protobuf::Any* value_; PROTOBUF_TSAN_DECLARE_MEMBER @@ -783,7 +783,7 @@ class PROTOBUF_EXPORT Field final : public ::google::protobuf::Message ::int32_t number_; ::int32_t oneof_index_; bool packed_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1006,7 +1006,7 @@ class PROTOBUF_EXPORT EnumValue final : public ::google::protobuf::Message ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::internal::ArenaStringPtr name_; ::int32_t number_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1301,7 +1301,7 @@ class PROTOBUF_EXPORT Type final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Type& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Field > fields_; ::google::protobuf::RepeatedPtrField oneofs_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; @@ -1580,7 +1580,7 @@ class PROTOBUF_EXPORT Enum final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Enum& from_msg); ::google::protobuf::internal::HasBits<1> _has_bits_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::EnumValue > enumvalue_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::internal::ArenaStringPtr name_; diff --git a/src/google/protobuf/unittest_lite.proto b/src/google/protobuf/unittest_lite.proto index b3fcfa431c..4bc78c4de6 100644 --- a/src/google/protobuf/unittest_lite.proto +++ b/src/google/protobuf/unittest_lite.proto @@ -625,3 +625,7 @@ message RecursiveMessage { RecursiveMessage recurse = 1; bytes payload = 2; } + +message RecursiveGroup { + RecursiveGroup recurse = 1 [features.message_encoding = DELIMITED]; +} diff --git a/src/google/protobuf/unittest_lite_edition_2024.proto b/src/google/protobuf/unittest_lite_edition_2024.proto new file mode 100644 index 0000000000..b56f7d026a --- /dev/null +++ b/src/google/protobuf/unittest_lite_edition_2024.proto @@ -0,0 +1,10 @@ +edition = "2024"; + +package protobuf_unittest; + +option optimize_for = LITE_RUNTIME; + +enum EnumNameStringView { + ENUM_NAME_STRING_VIEW_DEFAULT = 0; + ENUM_NAME_STRING_VIEW_ANOTHER_VALUE = 1; +} diff --git a/src/google/protobuf/unittest_no_field_presence.proto b/src/google/protobuf/unittest_no_field_presence.proto index f6880a58e9..cc02acc913 100644 --- a/src/google/protobuf/unittest_no_field_presence.proto +++ b/src/google/protobuf/unittest_no_field_presence.proto @@ -92,7 +92,7 @@ message TestAllTypes { repeated string repeated_string_piece = 54 [ctype = STRING_PIECE]; repeated string repeated_cord = 55 [ctype = CORD]; - repeated NestedMessage repeated_lazy_message = 57 ; + repeated NestedMessage repeated_lazy_message = 57; oneof oneof_field { uint32 oneof_uint32 = 111; @@ -112,6 +112,12 @@ message ForeignMessage { int32 c = 1; } +// Same as ForeignMessage, but all fields have explicit presence. +// It can be useful for testing explicit-implicit presence interop behaviour. +message ExplicitForeignMessage { + int32 c = 1 [features.field_presence = EXPLICIT]; +} + enum ForeignEnum { FOREIGN_FOO = 0; FOREIGN_BAR = 1; diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index 852a73f2d6..ecf66f5e82 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -16,7 +16,6 @@ #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/string_view.h" #include "google/protobuf/extension_set.h" -#include "google/protobuf/generated_message_tctable_decl.h" #include "google/protobuf/generated_message_tctable_impl.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream.h" @@ -26,45 +25,29 @@ #include "google/protobuf/wire_format.h" #include "google/protobuf/wire_format_lite.h" - // Must be included last. #include "google/protobuf/port_def.inc" namespace google { namespace protobuf { -const UnknownFieldSet& UnknownFieldSet::default_instance() { - static auto instance = internal::OnShutdownDelete(new UnknownFieldSet()); - return *instance; -} - void UnknownFieldSet::ClearFallback() { ABSL_DCHECK(!fields_.empty()); - int n = fields_.size(); - do { - (fields_)[--n].Delete(); - } while (n > 0); - fields_.clear(); -} - -void UnknownFieldSet::InternalMergeFrom(const UnknownFieldSet& other) { - int other_field_count = other.field_count(); - if (other_field_count > 0) { - fields_.reserve(fields_.size() + other_field_count); - for (int i = 0; i < other_field_count; i++) { - fields_.push_back((other.fields_)[i]); - fields_.back().DeepCopy((other.fields_)[i]); - } + if (arena() == nullptr) { + int n = fields_.size(); + do { + (fields_)[--n].Delete(); + } while (n > 0); } + fields_.Clear(); } void UnknownFieldSet::MergeFrom(const UnknownFieldSet& other) { int other_field_count = other.field_count(); if (other_field_count > 0) { - fields_.reserve(fields_.size() + other_field_count); - for (int i = 0; i < other_field_count; i++) { - fields_.push_back((other.fields_)[i]); - fields_.back().DeepCopy((other.fields_)[i]); + fields_.Reserve(fields_.size() + other_field_count); + for (auto elem : other.fields_) { + fields_.Add(elem.DeepCopy(arena())); } } } @@ -72,14 +55,14 @@ void UnknownFieldSet::MergeFrom(const UnknownFieldSet& other) { // A specialized MergeFrom for performance when we are merging from an UFS that // is temporary and can be destroyed in the process. void UnknownFieldSet::MergeFromAndDestroy(UnknownFieldSet* other) { - if (fields_.empty()) { - fields_ = std::move(other->fields_); + if (arena() != other->arena()) { + MergeFrom(*other); + } else if (fields_.empty()) { + fields_.Swap(&other->fields_); } else { - fields_.insert(fields_.end(), - std::make_move_iterator(other->fields_.begin()), - std::make_move_iterator(other->fields_.end())); + fields_.MergeFrom(other->fields_); + other->fields_.Clear(); } - other->fields_.clear(); } void UnknownFieldSet::MergeToInternalMetadata( @@ -90,14 +73,14 @@ void UnknownFieldSet::MergeToInternalMetadata( size_t UnknownFieldSet::SpaceUsedExcludingSelfLong() const { if (fields_.empty()) return 0; - size_t total_size = sizeof(UnknownField) * fields_.capacity(); + size_t total_size = fields_.SpaceUsedExcludingSelfLong(); for (const UnknownField& field : fields_) { switch (field.type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - total_size += sizeof(*field.data_.length_delimited_.string_value) + + total_size += sizeof(*field.data_.string_value) + internal::StringSpaceUsedExcludingSelfLong( - *field.data_.length_delimited_.string_value); + *field.data_.string_value); break; case UnknownField::TYPE_GROUP: total_size += field.data_.group_->SpaceUsedLong(); @@ -114,66 +97,54 @@ size_t UnknownFieldSet::SpaceUsedLong() const { } void UnknownFieldSet::AddVarint(int number, uint64_t value) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_VARINT); field.data_.varint_ = value; } void UnknownFieldSet::AddFixed32(int number, uint32_t value) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_FIXED32); field.data_.fixed32_ = value; } void UnknownFieldSet::AddFixed64(int number, uint64_t value) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_FIXED64); field.data_.fixed64_ = value; } std::string* UnknownFieldSet::AddLengthDelimited(int number) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_LENGTH_DELIMITED); - field.data_.length_delimited_.string_value = new std::string; - return field.data_.length_delimited_.string_value; + field.data_.string_value = Arena::Create(arena()); + return field.data_.string_value; } - UnknownFieldSet* UnknownFieldSet::AddGroup(int number) { - fields_.emplace_back(); - auto& field = fields_.back(); + auto& field = *fields_.Add(); field.number_ = number; field.SetType(UnknownField::TYPE_GROUP); - field.data_.group_ = new UnknownFieldSet; + field.data_.group_ = Arena::Create(arena()); return field.data_.group_; } void UnknownFieldSet::AddField(const UnknownField& field) { - fields_.push_back(field); - fields_.back().DeepCopy(field); + fields_.Add(field.DeepCopy(arena())); } void UnknownFieldSet::DeleteSubrange(int start, int num) { - // Delete the specified fields. - for (int i = 0; i < num; ++i) { - (fields_)[i + start].Delete(); - } - // Slide down the remaining fields. - for (size_t i = start + num; i < fields_.size(); ++i) { - (fields_)[i - num] = (fields_)[i]; - } - // Pop off the # of deleted fields. - for (int i = 0; i < num; ++i) { - fields_.pop_back(); + if (arena() == nullptr) { + // Delete the specified fields. + for (int i = 0; i < num; ++i) { + (fields_)[i + start].Delete(); + } } + fields_.ExtractSubrange(start, num, nullptr); } void UnknownFieldSet::DeleteByNumber(int number) { @@ -181,7 +152,9 @@ void UnknownFieldSet::DeleteByNumber(int number) { for (size_t i = 0; i < fields_.size(); ++i) { UnknownField* field = &(fields_)[i]; if (field->number() == number) { - field->Delete(); + if (arena() == nullptr) { + field->Delete(); + } } else { if (i != left) { (fields_)[left] = (fields_)[i]; @@ -189,7 +162,7 @@ void UnknownFieldSet::DeleteByNumber(int number) { ++left; } } - fields_.resize(left); + fields_.Truncate(left); } bool UnknownFieldSet::MergeFromCodedStream(io::CodedInputStream* input) { @@ -249,7 +222,7 @@ bool UnknownFieldSet::SerializeToCord(absl::Cord* output) const { void UnknownField::Delete() { switch (type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - delete data_.length_delimited_.string_value; + delete data_.string_value; break; case UnknownField::TYPE_GROUP: delete data_.group_; @@ -259,29 +232,38 @@ void UnknownField::Delete() { } } -void UnknownField::DeepCopy(const UnknownField& other) { - (void)other; // Parameter is used by Google-internal code. +UnknownField UnknownField::DeepCopy(Arena* arena) const { + UnknownField copy = *this; switch (type()) { case UnknownField::TYPE_LENGTH_DELIMITED: - data_.length_delimited_.string_value = - new std::string(*data_.length_delimited_.string_value); + copy.data_.string_value = + Arena::Create(arena, *data_.string_value); break; case UnknownField::TYPE_GROUP: { - UnknownFieldSet* group = new UnknownFieldSet(); - group->InternalMergeFrom(*data_.group_); - data_.group_ = group; + UnknownFieldSet* group = Arena::Create(arena); + group->MergeFrom(*data_.group_); + copy.data_.group_ = group; break; } default: break; } + return copy; } +void UnknownFieldSet::SwapSlow(UnknownFieldSet* other) { + UnknownFieldSet tmp; + tmp.MergeFrom(*this); + this->Clear(); + this->MergeFrom(*other); + other->Clear(); + other->MergeFrom(tmp); +} uint8_t* UnknownField::InternalSerializeLengthDelimitedNoTag( uint8_t* target, io::EpsCopyOutputStream* stream) const { ABSL_DCHECK_EQ(TYPE_LENGTH_DELIMITED, type()); - const absl::string_view data = *data_.length_delimited_.string_value; + const absl::string_view data = *data_.string_value; target = io::CodedOutputStream::WriteVarint32ToArray(data.size(), target); target = stream->WriteRaw(data.data(), data.size(), target); return target; diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index e985b0f4d4..e4cdb70a9a 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -19,17 +19,19 @@ #include #include -#include #include "google/protobuf/stubs/common.h" #include "absl/log/absl_check.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" +#include "google/protobuf/arena.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/message_lite.h" +#include "google/protobuf/metadata_lite.h" #include "google/protobuf/parse_context.h" #include "google/protobuf/port.h" +#include "google/protobuf/repeated_field.h" // Must be included last. #include "google/protobuf/port_def.inc" @@ -54,7 +56,67 @@ using UFSStringView = const std::string&; } // namespace internal class Message; // message.h -class UnknownField; // below + +// Represents one field in an UnknownFieldSet. +class PROTOBUF_EXPORT UnknownField { + public: + enum Type { + TYPE_VARINT, + TYPE_FIXED32, + TYPE_FIXED64, + TYPE_LENGTH_DELIMITED, + TYPE_GROUP + }; + + // The field's field number, as seen on the wire. + inline int number() const; + + // The field type. + inline Type type() const; + + // Accessors ------------------------------------------------------- + // Each method works only for UnknownFields of the corresponding type. + + inline uint64_t varint() const; + inline uint32_t fixed32() const; + inline uint64_t fixed64() const; + inline internal::UFSStringView length_delimited() const; + inline const UnknownFieldSet& group() const; + + inline void set_varint(uint64_t value); + inline void set_fixed32(uint32_t value); + inline void set_fixed64(uint64_t value); + inline void set_length_delimited(absl::string_view value); + inline std::string* mutable_length_delimited(); + inline UnknownFieldSet* mutable_group(); + + inline size_t GetLengthDelimitedSize() const; + uint8_t* InternalSerializeLengthDelimitedNoTag( + uint8_t* target, io::EpsCopyOutputStream* stream) const; + + private: + friend class UnknownFieldSet; + + // If this UnknownField contains a pointer, delete it. + void Delete(); + + // Make a deep copy of any pointers in this UnknownField. + UnknownField DeepCopy(Arena* arena) const; + + // Set the wire type of this UnknownField. Should only be used when this + // UnknownField is being created. + inline void SetType(Type type); + + uint32_t number_; + uint32_t type_; + union { + uint64_t varint_; + uint32_t fixed32_; + uint64_t fixed64_; + std::string* string_value; + UnknownFieldSet* group_; + } data_; +}; // An UnknownFieldSet contains fields that were encountered while parsing a // message but were not defined by its type. Keeping track of these can be @@ -70,7 +132,7 @@ class UnknownField; // below // the Reflection interface which is independent of any serialization scheme. class PROTOBUF_EXPORT UnknownFieldSet { public: - UnknownFieldSet(); + constexpr UnknownFieldSet(); UnknownFieldSet(const UnknownFieldSet&) = delete; UnknownFieldSet& operator=(const UnknownFieldSet&) = delete; ~UnknownFieldSet(); @@ -167,13 +229,20 @@ class PROTOBUF_EXPORT UnknownFieldSet { bool SerializeToCodedStream(io::CodedOutputStream* output) const; static const UnknownFieldSet& default_instance(); + UnknownFieldSet(internal::InternalVisibility, Arena* arena) + : UnknownFieldSet(arena) {} + private: - // For InternalMergeFrom - friend class UnknownField; - // Merges from other UnknownFieldSet. This method assumes, that this object - // is newly created and has no fields. - void InternalMergeFrom(const UnknownFieldSet& other); + using InternalArenaConstructable_ = void; + using DestructorSkippable_ = void; + + friend class google::protobuf::Arena; + explicit UnknownFieldSet(Arena* arena) : fields_(arena) {} + + Arena* arena() { return fields_.GetArena(); } + void ClearFallback(); + void SwapSlow(UnknownFieldSet* other); template fields_; + RepeatedField fields_; }; namespace internal { @@ -218,76 +287,19 @@ const char* UnknownFieldParse(uint64_t tag, UnknownFieldSet* unknown, } // namespace internal -// Represents one field in an UnknownFieldSet. -class PROTOBUF_EXPORT UnknownField { - public: - enum Type { - TYPE_VARINT, - TYPE_FIXED32, - TYPE_FIXED64, - TYPE_LENGTH_DELIMITED, - TYPE_GROUP - }; - - // The field's field number, as seen on the wire. - inline int number() const; - - // The field type. - inline Type type() const; - - // Accessors ------------------------------------------------------- - // Each method works only for UnknownFields of the corresponding type. - - inline uint64_t varint() const; - inline uint32_t fixed32() const; - inline uint64_t fixed64() const; - inline internal::UFSStringView length_delimited() const; - inline const UnknownFieldSet& group() const; - - inline void set_varint(uint64_t value); - inline void set_fixed32(uint32_t value); - inline void set_fixed64(uint64_t value); - inline void set_length_delimited(absl::string_view value); - inline std::string* mutable_length_delimited(); - inline UnknownFieldSet* mutable_group(); - - inline size_t GetLengthDelimitedSize() const; - uint8_t* InternalSerializeLengthDelimitedNoTag( - uint8_t* target, io::EpsCopyOutputStream* stream) const; - - - // If this UnknownField contains a pointer, delete it. - void Delete(); - - // Make a deep copy of any pointers in this UnknownField. - void DeepCopy(const UnknownField& other); - - // Set the wire type of this UnknownField. Should only be used when this - // UnknownField is being created. - inline void SetType(Type type); - - union LengthDelimited { - std::string* string_value; - }; - - uint32_t number_; - uint32_t type_; - union { - uint64_t varint_; - uint32_t fixed32_; - uint64_t fixed64_; - mutable union LengthDelimited length_delimited_; - UnknownFieldSet* group_; - } data_; -}; - // =================================================================== // inline implementations -inline UnknownFieldSet::UnknownFieldSet() {} +constexpr UnknownFieldSet::UnknownFieldSet() = default; inline UnknownFieldSet::~UnknownFieldSet() { Clear(); } +inline const UnknownFieldSet& UnknownFieldSet::default_instance() { + PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT static const UnknownFieldSet + instance; + return instance; +} + inline void UnknownFieldSet::ClearAndFreeMemory() { Clear(); } inline void UnknownFieldSet::Clear() { @@ -299,7 +311,12 @@ inline void UnknownFieldSet::Clear() { inline bool UnknownFieldSet::empty() const { return fields_.empty(); } inline void UnknownFieldSet::Swap(UnknownFieldSet* x) { - fields_.swap(x->fields_); + if (arena() == x->arena()) { + fields_.Swap(&x->fields_); + } else { + // We might need to do a deep copy, so use Merge instead + SwapSlow(x); + } } inline int UnknownFieldSet::field_count() const { @@ -317,9 +334,6 @@ inline void UnknownFieldSet::AddLengthDelimited(int number, AddLengthDelimited(number)->assign(value.data(), value.size()); } - - - inline int UnknownField::number() const { return static_cast(number_); } inline UnknownField::Type UnknownField::type() const { return static_cast(type_); @@ -339,7 +353,7 @@ inline uint64_t UnknownField::fixed64() const { } inline internal::UFSStringView UnknownField::length_delimited() const { assert(type() == TYPE_LENGTH_DELIMITED); - return *data_.length_delimited_.string_value; + return *data_.string_value; } inline const UnknownFieldSet& UnknownField::group() const { assert(type() == TYPE_GROUP); @@ -360,11 +374,11 @@ inline void UnknownField::set_fixed64(uint64_t value) { } inline void UnknownField::set_length_delimited(const absl::string_view value) { assert(type() == TYPE_LENGTH_DELIMITED); - data_.length_delimited_.string_value->assign(value.data(), value.size()); + data_.string_value->assign(value.data(), value.size()); } inline std::string* UnknownField::mutable_length_delimited() { assert(type() == TYPE_LENGTH_DELIMITED); - return data_.length_delimited_.string_value; + return data_.string_value; } inline UnknownFieldSet* UnknownField::mutable_group() { assert(type() == TYPE_GROUP); @@ -376,16 +390,28 @@ bool UnknownFieldSet::MergeFromMessage(const MessageType& message) { return InternalMergeFromMessage(message); } - inline size_t UnknownField::GetLengthDelimitedSize() const { ABSL_DCHECK_EQ(TYPE_LENGTH_DELIMITED, type()); - return data_.length_delimited_.string_value->size(); + return data_.string_value->size(); } -inline void UnknownField::SetType(Type type) { - type_ = type; -} +inline void UnknownField::SetType(Type type) { type_ = type; } + +namespace internal { +// Add specialization of InternalMetadata::Container to provide arena support. +template <> +struct InternalMetadata::Container + : public InternalMetadata::ContainerBase { + UnknownFieldSet unknown_fields; + + explicit Container(Arena* input_arena) + : unknown_fields(InternalVisibility{}, input_arena) {} + + using InternalArenaConstructable_ = void; + using DestructorSkippable_ = void; +}; +} // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/unknown_field_set_unittest.cc b/src/google/protobuf/unknown_field_set_unittest.cc index dcd20e5ad6..27923675f1 100644 --- a/src/google/protobuf/unknown_field_set_unittest.cc +++ b/src/google/protobuf/unknown_field_set_unittest.cc @@ -37,7 +37,6 @@ #include "google/protobuf/unittest_lite.pb.h" #include "google/protobuf/wire_format.h" - namespace google { namespace protobuf { @@ -176,6 +175,94 @@ TEST_F(UnknownFieldSetTest, Group) { EXPECT_EQ(all_fields_.optionalgroup().a(), nested_field.varint()); } +static void PopulateUFS(UnknownFieldSet& set) { + UnknownFieldSet* node = &set; + for (int i = 0; i < 3; ++i) { + node->AddVarint(1, 100); + const char* long_str = "This is a very long string, not sso"; + node->AddLengthDelimited(2, long_str); + *node->AddLengthDelimited(3) = long_str; + // Test some recursion too. + node = node->AddGroup(4); + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithMergeFrom) { + Arena arena; + + for (bool lhs_arena : {false, true}) { + for (bool rhs_arena : {false, true}) { + UnknownFieldSet lhs_stack, rhs_stack; + auto& lhs = + lhs_arena ? *Arena::Create(&arena) : lhs_stack; + auto& rhs = + rhs_arena ? *Arena::Create(&arena) : rhs_stack; + PopulateUFS(rhs); + lhs.MergeFrom(rhs); + } + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithMergeAndDestroy) { + Arena arena; + + for (bool lhs_arena : {false, true}) { + for (bool populate_lhs : {false, true}) { + for (bool rhs_arena : {false, true}) { + for (bool populate_rhs : {false, true}) { + UnknownFieldSet lhs_stack, rhs_stack; + auto& lhs = + lhs_arena ? *Arena::Create(&arena) : lhs_stack; + auto& rhs = + rhs_arena ? *Arena::Create(&arena) : rhs_stack; + if (populate_lhs) PopulateUFS(lhs); + if (populate_rhs) PopulateUFS(rhs); + lhs.MergeFromAndDestroy(&rhs); + } + } + } + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithSwap) { + Arena arena; + + for (bool lhs_arena : {false, true}) { + for (bool rhs_arena : {false, true}) { + UnknownFieldSet lhs_stack, rhs_stack; + auto& lhs = + lhs_arena ? *Arena::Create(&arena) : lhs_stack; + auto& rhs = + rhs_arena ? *Arena::Create(&arena) : rhs_stack; + PopulateUFS(lhs); + lhs.Swap(&rhs); + } + } +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksWithClear) { + Arena arena; + auto* ufs = Arena::Create(&arena); + PopulateUFS(*ufs); + // Clear should not try to delete memory from the arena. + ufs->Clear(); +} + +TEST_F(UnknownFieldSetTest, ArenaSupportWorksDelete) { + Arena arena; + + auto* ufs = Arena::Create(&arena); + PopulateUFS(*ufs); + + while (ufs->field_count() != 0) { + ufs->DeleteByNumber(ufs->field(0).number()); + } + + ufs = Arena::Create(&arena); + PopulateUFS(*ufs); + ufs->DeleteSubrange(0, ufs->field_count()); +} + TEST_F(UnknownFieldSetTest, SerializeFastAndSlowAreEquivalent) { int size = WireFormat::ComputeUnknownFieldsSize(empty_message_.unknown_fields()); @@ -335,7 +422,6 @@ TEST_F(UnknownFieldSetTest, MergeFromMessageLite) { EXPECT_EQ(unknown_field.fixed32(), 42); } - TEST_F(UnknownFieldSetTest, Clear) { // Clear the set. empty_message_.Clear(); @@ -518,13 +604,15 @@ TEST_F(UnknownFieldSetTest, UnknownEnumValue) { TEST_F(UnknownFieldSetTest, SpaceUsedExcludingSelf) { UnknownFieldSet empty; empty.AddVarint(1, 0); - EXPECT_EQ(sizeof(UnknownField), empty.SpaceUsedExcludingSelf()); + RepeatedField rep; + rep.Add(); + EXPECT_EQ(rep.SpaceUsedExcludingSelf(), empty.SpaceUsedExcludingSelf()); } TEST_F(UnknownFieldSetTest, SpaceUsed) { // Keep shadow vectors to avoid making assumptions about its capacity growth. // We imitate the push back calls here to determine the expected capacity. - std::vector shadow_vector, shadow_vector_group; + RepeatedField shadow_vector, shadow_vector_group; unittest::TestEmptyMessage empty_message; // Make sure an unknown field set has zero space used until a field is @@ -534,8 +622,8 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { UnknownFieldSet* group = nullptr; const auto total = [&] { size_t result = base; - result += shadow_vector.capacity() * sizeof(UnknownField); - result += shadow_vector_group.capacity() * sizeof(UnknownField); + result += shadow_vector.SpaceUsedExcludingSelfLong(); + result += shadow_vector_group.SpaceUsedExcludingSelfLong(); if (str != nullptr) { result += sizeof(std::string); static const size_t sso_capacity = std::string().capacity(); @@ -552,30 +640,29 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) { // Make sure each thing we add to the set increases the SpaceUsedLong(). unknown_fields->AddVarint(1, 0); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var"; str = unknown_fields->AddLengthDelimited(1); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str"; str->assign(sizeof(std::string) + 1, 'x'); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Str2"; group = unknown_fields->AddGroup(1); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Group"; group->AddVarint(1, 0); - shadow_vector_group.emplace_back(); + shadow_vector_group.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Group2"; unknown_fields->AddVarint(1, 0); - shadow_vector.emplace_back(); + shadow_vector.Add(); EXPECT_EQ(total(), empty_message.SpaceUsedLong()) << "Var2"; } - TEST_F(UnknownFieldSetTest, Empty) { UnknownFieldSet unknown_fields; EXPECT_TRUE(unknown_fields.empty()); @@ -732,6 +819,5 @@ TEST_F(UnknownFieldSetTest, SerializeToCord_TestPackedTypes) { } } // namespace - } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/wire_format_lite.cc b/src/google/protobuf/wire_format_lite.cc index 8fc29af555..6a9caa9041 100644 --- a/src/google/protobuf/wire_format_lite.cc +++ b/src/google/protobuf/wire_format_lite.cc @@ -21,6 +21,7 @@ #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "google/protobuf/message_lite.h" #include "utf8_validity.h" @@ -830,6 +831,91 @@ size_t WireFormatLite::SInt64Size(const RepeatedField& value) { #endif +size_t WireFormatLite::Int32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = Int32Size(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::Int64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = Int64Size(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::UInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = UInt32Size(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::UInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = UInt64Size(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::SInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = SInt32Size(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::SInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = SInt64Size(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} +size_t WireFormatLite::EnumSizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size) { + if (value.empty()) { + cached_size.Set(0); + return 0; + } + size_t res; + PROTOBUF_ALWAYS_INLINE_CALL res = EnumSize(value); + cached_size.SetNonZero(ToCachedSize(res)); + return tag_size + res + Int32Size(static_cast(res)); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/wire_format_lite.h b/src/google/protobuf/wire_format_lite.h index ad0167c850..42634885cd 100644 --- a/src/google/protobuf/wire_format_lite.h +++ b/src/google/protobuf/wire_format_lite.h @@ -700,6 +700,28 @@ class PROTOBUF_EXPORT WireFormatLite { static size_t SInt64Size(const RepeatedField& value); static size_t EnumSize(const RepeatedField& value); + static size_t Int32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t Int64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t UInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t UInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t SInt32SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t SInt64SizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + static size_t EnumSizeWithPackedTagSize( + const RepeatedField& value, size_t tag_size, + const internal::CachedSize& cached_size); + // These types always have the same size. static constexpr size_t kFixed32Size = 4; static constexpr size_t kFixed64Size = 8; diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index bd627bee1a..6e156ce6f7 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -267,7 +267,7 @@ class PROTOBUF_EXPORT UInt64Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const UInt64Value& from_msg); ::uint64_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -453,7 +453,7 @@ class PROTOBUF_EXPORT UInt32Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const UInt32Value& from_msg); ::uint32_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -645,7 +645,7 @@ class PROTOBUF_EXPORT StringValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const StringValue& from_msg); ::google::protobuf::internal::ArenaStringPtr value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -831,7 +831,7 @@ class PROTOBUF_EXPORT Int64Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Int64Value& from_msg); ::int64_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1017,7 +1017,7 @@ class PROTOBUF_EXPORT Int32Value final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const Int32Value& from_msg); ::int32_t value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1203,7 +1203,7 @@ class PROTOBUF_EXPORT FloatValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const FloatValue& from_msg); float value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1389,7 +1389,7 @@ class PROTOBUF_EXPORT DoubleValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const DoubleValue& from_msg); double value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1581,7 +1581,7 @@ class PROTOBUF_EXPORT BytesValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const BytesValue& from_msg); ::google::protobuf::internal::ArenaStringPtr value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; @@ -1767,7 +1767,7 @@ class PROTOBUF_EXPORT BoolValue final : public ::google::protobuf::Message ::google::protobuf::Arena* arena, const Impl_& from, const BoolValue& from_msg); bool value_; - mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::internal::CachedSize _cached_size_; PROTOBUF_TSAN_DECLARE_MEMBER }; union { Impl_ _impl_; }; diff --git a/upb/BUILD b/upb/BUILD index 42797dc2ec..b46ee4b477 100644 --- a/upb/BUILD +++ b/upb/BUILD @@ -288,6 +288,7 @@ upb_amalgamation( "//upb/reflection:internal", ], strip_import_prefix = ["src"], + visibility = ["//rust:__pkg__"], ) cc_library( diff --git a/upb/cmake/BUILD.bazel b/upb/cmake/BUILD.bazel index e17ce8450a..fdf0f23226 100644 --- a/upb/cmake/BUILD.bazel +++ b/upb/cmake/BUILD.bazel @@ -5,6 +5,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load( "//upb/bazel:build_defs.bzl", "make_shell_script", @@ -106,3 +107,26 @@ sh_test( }), deps = ["@bazel_tools//tools/bash/runfiles"], ) + +pkg_files( + name = "upb_cmake_dist", + srcs = [ + ":copy_protos", + ":gen_cmakelists", + "//third_party/utf8_range:utf8_range_srcs", + "//upb:source_files", + "//upb/base:source_files", + "//upb/hash:source_files", + "//upb/lex:source_files", + "//upb/mem:source_files", + "//upb/message:source_files", + "//upb/mini_descriptor:source_files", + "//upb/mini_table:source_files", + "//upb/port:source_files", + "//upb/reflection:source_files", + "//upb/text:source_files", + "//upb/wire:source_files", + ], + strip_prefix = strip_prefix.from_root(""), + visibility = ["//rust:__pkg__"], +) diff --git a/upb/wire/eps_copy_input_stream_test.cc b/upb/wire/eps_copy_input_stream_test.cc index c28d1457e4..42664748e3 100644 --- a/upb/wire/eps_copy_input_stream_test.cc +++ b/upb/wire/eps_copy_input_stream_test.cc @@ -281,7 +281,7 @@ TEST(EpsCopyInputStreamTest, ZeroSize) { // } // // // Test with: -// // $ blaze run --config=fuzztest third_party/upb:eps_copy_input_stream_test \ +// // $ bazel run --config=fuzztest third_party/upb:eps_copy_input_stream_test \ // // -- --gunit_fuzz= // FUZZ_TEST(EpsCopyFuzzTest, TestAgainstFakeStream) // .WithDomains(ArbitraryEpsCopyTestScript());