From 7599210683886181b774f05271d59fc99627ecad Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 5 Jul 2023 17:06:09 -0700 Subject: [PATCH] Editions test enhancements to cover utf8 handling. This replaces the cc_utf8_verification and enforce_utf8 options with the corresponding feature values, consistent with C++ behavior. Further runtimes will be supported by refactoring the string_field_validation feature in a later change. PiperOrigin-RevId: 545824813 --- src/google/protobuf/descriptor.cc | 20 ++++---- .../protobuf/editions/codegen_tests/BUILD | 11 ++++ .../codegen_tests/proto2_utf8_disabled.proto | 3 +- .../codegen_tests/proto2_utf8_lite.proto | 40 +++++++++++++++ .../codegen_tests/proto3_utf8_disabled.proto | 1 + .../codegen_tests/proto3_utf8_strict.proto | 1 + .../golden/editions_transform_proto2.proto | 15 +++++- .../editions_transform_proto2_lite.proto | 51 +++++++++++++++++++ ...tions_transform_proto2_utf8_disabled.proto | 50 ++++++++++++++++++ .../golden/editions_transform_proto3.proto | 48 +++++++++++++++++ ...tions_transform_proto3_utf8_disabled.proto | 45 ++++++++++++++++ .../golden/test_messages_proto2.proto | 2 +- .../proto/editions_transform_proto2.proto | 25 +++++++++ .../editions_transform_proto2_lite.proto | 45 ++++++++++++++++ ...tions_transform_proto2_utf8_disabled.proto | 44 ++++++++++++++++ .../proto/editions_transform_proto3.proto | 47 +++++++++++++++++ ...tions_transform_proto3_utf8_disabled.proto | 44 ++++++++++++++++ src/google/protobuf/editions/transform.awk | 41 ++++++++++++--- 18 files changed, 512 insertions(+), 21 deletions(-) create mode 100644 src/google/protobuf/editions/codegen_tests/proto2_utf8_lite.proto create mode 100644 src/google/protobuf/editions/golden/editions_transform_proto2_lite.proto create mode 100644 src/google/protobuf/editions/golden/editions_transform_proto2_utf8_disabled.proto create mode 100644 src/google/protobuf/editions/golden/editions_transform_proto3.proto create mode 100644 src/google/protobuf/editions/golden/editions_transform_proto3_utf8_disabled.proto create mode 100644 src/google/protobuf/editions/proto/editions_transform_proto2_lite.proto create mode 100644 src/google/protobuf/editions/proto/editions_transform_proto2_utf8_disabled.proto create mode 100644 src/google/protobuf/editions/proto/editions_transform_proto3.proto create mode 100644 src/google/protobuf/editions/proto/editions_transform_proto3_utf8_disabled.proto diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 7f55bfcb1a..8c9403f598 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3875,19 +3875,18 @@ bool FieldDescriptor::is_packed() const { } } -static bool FieldEnforceUtf8(const FieldDescriptor* field) { - return +static bool IsStrictUtf8(const FieldDescriptor* field) { #ifdef PROTOBUF_FUTURE_EDITIONS - internal::InternalFeatureHelper::GetFeatures(*field) - .string_field_validation() == FeatureSet::MANDATORY; + return internal::InternalFeatureHelper::GetFeatures(*field) + .string_field_validation() == FeatureSet::MANDATORY; #else // PROTOBUF_FUTURE_EDITIONS - FileDescriptorLegacy(field->file()).syntax() == - FileDescriptorLegacy::Syntax::SYNTAX_PROTO3; + return FileDescriptorLegacy(field->file()).syntax() == + FileDescriptorLegacy::Syntax::SYNTAX_PROTO3; #endif // PROTOBUF_FUTURE_EDITIONS } bool FieldDescriptor::requires_utf8_validation() const { - return type() == TYPE_STRING && FieldEnforceUtf8(this); + return type() == TYPE_STRING && IsStrictUtf8(this); } bool FieldDescriptor::has_presence() const { @@ -9567,15 +9566,16 @@ bool HasHasbit(const FieldDescriptor* field) { !field->options().weak(); } -static bool FileUtf8Verification(const FileDescriptor* file) { +static bool IsVerifyUtf8(const FieldDescriptor* field, bool is_lite) { + if (is_lite) return false; return true; } // Which level of UTF-8 enforcemant is placed on this file. Utf8CheckMode GetUtf8CheckMode(const FieldDescriptor* field, bool is_lite) { - if (FieldEnforceUtf8(field)) { + if (IsStrictUtf8(field)) { return Utf8CheckMode::kStrict; - } else if (!is_lite && FileUtf8Verification(field->file())) { + } else if (IsVerifyUtf8(field, is_lite)) { return Utf8CheckMode::kVerify; } else { return Utf8CheckMode::kNone; diff --git a/src/google/protobuf/editions/codegen_tests/BUILD b/src/google/protobuf/editions/codegen_tests/BUILD index 5bda10080b..f3ab7544fc 100644 --- a/src/google/protobuf/editions/codegen_tests/BUILD +++ b/src/google/protobuf/editions/codegen_tests/BUILD @@ -103,6 +103,17 @@ cc_proto_library( deps = [":proto2_utf8_disabled_proto"], ) +proto_library( + name = "proto2_utf8_lite_proto", + srcs = ["proto2_utf8_lite.proto"], + strip_import_prefix = "/src", +) + +cc_proto_library( + name = "proto2_utf8_lite_cc_proto", + deps = [":proto2_utf8_lite_proto"], +) + proto_library( name = "proto2_proto3_enum_proto", srcs = ["proto2_proto3_enum.proto"], diff --git a/src/google/protobuf/editions/codegen_tests/proto2_utf8_disabled.proto b/src/google/protobuf/editions/codegen_tests/proto2_utf8_disabled.proto index 2d2033bacb..84618151b9 100644 --- a/src/google/protobuf/editions/codegen_tests/proto2_utf8_disabled.proto +++ b/src/google/protobuf/editions/codegen_tests/proto2_utf8_disabled.proto @@ -33,6 +33,7 @@ syntax = "proto2"; package protobuf_editions_test.proto2; -message Proto2Utf8Strict { +message Proto2Utf8Disabled { optional string string_field = 1; + map map_field = 2; } diff --git a/src/google/protobuf/editions/codegen_tests/proto2_utf8_lite.proto b/src/google/protobuf/editions/codegen_tests/proto2_utf8_lite.proto new file mode 100644 index 0000000000..2fc109dd55 --- /dev/null +++ b/src/google/protobuf/editions/codegen_tests/proto2_utf8_lite.proto @@ -0,0 +1,40 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_editions_test.proto2; + +option optimize_for = LITE_RUNTIME; + +message Proto2Utf8Lite { + optional string string_field = 1; + map map_field = 2; +} diff --git a/src/google/protobuf/editions/codegen_tests/proto3_utf8_disabled.proto b/src/google/protobuf/editions/codegen_tests/proto3_utf8_disabled.proto index 11ad005ce7..ad12ffd98e 100644 --- a/src/google/protobuf/editions/codegen_tests/proto3_utf8_disabled.proto +++ b/src/google/protobuf/editions/codegen_tests/proto3_utf8_disabled.proto @@ -34,4 +34,5 @@ package protobuf_editions_test.proto3; message Proto3Utf8Disabled { string string_field = 1; + map map_field = 10; } diff --git a/src/google/protobuf/editions/codegen_tests/proto3_utf8_strict.proto b/src/google/protobuf/editions/codegen_tests/proto3_utf8_strict.proto index 018d09810d..fbdb1d6088 100644 --- a/src/google/protobuf/editions/codegen_tests/proto3_utf8_strict.proto +++ b/src/google/protobuf/editions/codegen_tests/proto3_utf8_strict.proto @@ -34,4 +34,5 @@ package protobuf_editions_test.proto3; message Proto3Utf8Strict { string string_field = 1; + map map_field = 10; } diff --git a/src/google/protobuf/editions/golden/editions_transform_proto2.proto b/src/google/protobuf/editions/golden/editions_transform_proto2.proto index 0857e8bc3c..1320a2be91 100644 --- a/src/google/protobuf/editions/golden/editions_transform_proto2.proto +++ b/src/google/protobuf/editions/golden/editions_transform_proto2.proto @@ -32,7 +32,7 @@ edition = "2023"; import "google/protobuf/cpp_features.proto"; option features.enum_type = CLOSED; option features.repeated_field_encoding = EXPANDED; -option features.string_field_validation = NONE; +option features.string_field_validation = HINT; option features.json_format = LEGACY_BEST_EFFORT; option features.(pb.cpp).legacy_closed_enum = true; @@ -81,6 +81,19 @@ extend ParentMessage.ExtendedMessage { } message TestMessage { + string string_field = 1; + string string_field_utf = 2; + string string_field_noutf = 3; + string options_strip_beginning = 4 [ ctype = STRING_PIECE, default = "hello world abcd" ]; + string options_strip_middle = 5 [ ctype = STRING_PIECE, default = "hello world abcd" ]; + string options_strip_end = 6 [ ctype = STRING_PIECE, default = "hello world abcd" ]; + + map string_map_field = 7; + + repeated int32 int_field = 8; + repeated int32 int_field_packed = 9 [features.repeated_field_encoding = PACKED]; + repeated int32 int_field_unpacked = 10; + message OptionalGroup { int32 a = 17; } diff --git a/src/google/protobuf/editions/golden/editions_transform_proto2_lite.proto b/src/google/protobuf/editions/golden/editions_transform_proto2_lite.proto new file mode 100644 index 0000000000..404cdde914 --- /dev/null +++ b/src/google/protobuf/editions/golden/editions_transform_proto2_lite.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +edition = "2023"; +import "google/protobuf/cpp_features.proto"; +option features.enum_type = CLOSED; +option features.repeated_field_encoding = EXPANDED; +option features.string_field_validation = HINT; +option features.json_format = LEGACY_BEST_EFFORT; +option features.(pb.cpp).legacy_closed_enum = true; + +package protobuf_editions_test; + +option optimize_for = LITE_RUNTIME; + +message TestMessageLite { + string string_field = 1; + string string_field_utf = 2; + string string_field_noutf = 3; + + map string_map_field = 4; + + int32 int_field = 5; +} diff --git a/src/google/protobuf/editions/golden/editions_transform_proto2_utf8_disabled.proto b/src/google/protobuf/editions/golden/editions_transform_proto2_utf8_disabled.proto new file mode 100644 index 0000000000..e58ca31a0d --- /dev/null +++ b/src/google/protobuf/editions/golden/editions_transform_proto2_utf8_disabled.proto @@ -0,0 +1,50 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +edition = "2023"; +import "google/protobuf/cpp_features.proto"; +option features.enum_type = CLOSED; +option features.repeated_field_encoding = EXPANDED; +option features.string_field_validation = HINT; +option features.json_format = LEGACY_BEST_EFFORT; +option features.(pb.cpp).legacy_closed_enum = true; + +package protobuf_editions_test; + + +message TestMessageUtf8Disabled { + string string_field = 1 [features.string_field_validation = NONE]; + string string_field_utf = 2 [features.string_field_validation = NONE]; + string string_field_noutf = 3 [features.string_field_validation = NONE]; + + map string_map_field = 4 [features.string_field_validation = NONE]; + + int32 int_field = 5; +} diff --git a/src/google/protobuf/editions/golden/editions_transform_proto3.proto b/src/google/protobuf/editions/golden/editions_transform_proto3.proto new file mode 100644 index 0000000000..2310bd796a --- /dev/null +++ b/src/google/protobuf/editions/golden/editions_transform_proto3.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +edition = "2023"; +option features.field_presence = IMPLICIT; + +package protobuf_editions_test; + +message TestMessageProto3 { + string string_field = 1; + string string_field_utf = 2; + string string_field_noutf = 3 [features.string_field_validation = HINT]; + + map string_map_field = 4; + map string_map_field_utf = 5; + map string_map_field_noutf = 6 [features.string_field_validation = HINT]; + + repeated int32 int_field = 7; + repeated int32 int_field_packed = 8; + repeated int32 int_field_unpacked = 9 [features.repeated_field_encoding = EXPANDED]; +} diff --git a/src/google/protobuf/editions/golden/editions_transform_proto3_utf8_disabled.proto b/src/google/protobuf/editions/golden/editions_transform_proto3_utf8_disabled.proto new file mode 100644 index 0000000000..2ef6447bb8 --- /dev/null +++ b/src/google/protobuf/editions/golden/editions_transform_proto3_utf8_disabled.proto @@ -0,0 +1,45 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +edition = "2023"; +option features.field_presence = IMPLICIT; + +package protobuf_editions_test; + + +message TestMessageProto3 { + string string_field = 1 [features.string_field_validation = NONE]; + string string_field_utf = 2 [features.string_field_validation = NONE]; + string string_field_noutf = 3 [features.string_field_validation = NONE]; + + map string_map_field = 4 [features.string_field_validation = NONE]; + + repeated int32 int_field = 7; +} diff --git a/src/google/protobuf/editions/golden/test_messages_proto2.proto b/src/google/protobuf/editions/golden/test_messages_proto2.proto index e20b0987eb..f5f59dacf6 100644 --- a/src/google/protobuf/editions/golden/test_messages_proto2.proto +++ b/src/google/protobuf/editions/golden/test_messages_proto2.proto @@ -39,7 +39,7 @@ edition = "2023"; import "google/protobuf/cpp_features.proto"; option features.enum_type = CLOSED; option features.repeated_field_encoding = EXPANDED; -option features.string_field_validation = NONE; +option features.string_field_validation = HINT; option features.json_format = LEGACY_BEST_EFFORT; option features.(pb.cpp).legacy_closed_enum = true; diff --git a/src/google/protobuf/editions/proto/editions_transform_proto2.proto b/src/google/protobuf/editions/proto/editions_transform_proto2.proto index 829cd6468b..ce399a2874 100644 --- a/src/google/protobuf/editions/proto/editions_transform_proto2.proto +++ b/src/google/protobuf/editions/proto/editions_transform_proto2.proto @@ -76,6 +76,31 @@ extend ParentMessage.ExtendedMessage { } message TestMessage { + optional string string_field = 1; + optional string string_field_utf = 2; + optional string string_field_noutf = 3; + optional string options_strip_beginning = 4 [ + enforce_utf8 = false, + ctype = STRING_PIECE, + default = "hello world abcd" + ]; + optional string options_strip_middle = 5 [ + ctype = STRING_PIECE, + enforce_utf8 = false, + default = "hello world abcd" + ]; + optional string options_strip_end = 6 [ + ctype = STRING_PIECE, + default = "hello world abcd", + enforce_utf8 = false + ]; + + map string_map_field = 7; + + repeated int32 int_field = 8; + repeated int32 int_field_packed = 9 [packed = true]; + repeated int32 int_field_unpacked = 10 [packed = false]; + optional group OptionalGroup = 16 { optional int32 a = 17; } diff --git a/src/google/protobuf/editions/proto/editions_transform_proto2_lite.proto b/src/google/protobuf/editions/proto/editions_transform_proto2_lite.proto new file mode 100644 index 0000000000..fa1056736e --- /dev/null +++ b/src/google/protobuf/editions/proto/editions_transform_proto2_lite.proto @@ -0,0 +1,45 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_editions_test; + +option optimize_for = LITE_RUNTIME; + +message TestMessageLite { + optional string string_field = 1; + optional string string_field_utf = 2; + optional string string_field_noutf = 3; + + map string_map_field = 4; + + optional int32 int_field = 5; +} diff --git a/src/google/protobuf/editions/proto/editions_transform_proto2_utf8_disabled.proto b/src/google/protobuf/editions/proto/editions_transform_proto2_utf8_disabled.proto new file mode 100644 index 0000000000..169a08b3d5 --- /dev/null +++ b/src/google/protobuf/editions/proto/editions_transform_proto2_utf8_disabled.proto @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package protobuf_editions_test; + + +message TestMessageUtf8Disabled { + optional string string_field = 1; + optional string string_field_utf = 2; + optional string string_field_noutf = 3; + + map string_map_field = 4; + + optional int32 int_field = 5; +} diff --git a/src/google/protobuf/editions/proto/editions_transform_proto3.proto b/src/google/protobuf/editions/proto/editions_transform_proto3.proto new file mode 100644 index 0000000000..cd2e086190 --- /dev/null +++ b/src/google/protobuf/editions/proto/editions_transform_proto3.proto @@ -0,0 +1,47 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package protobuf_editions_test; + +message TestMessageProto3 { + string string_field = 1; + string string_field_utf = 2; + string string_field_noutf = 3; + + map string_map_field = 4; + map string_map_field_utf = 5; + map string_map_field_noutf = 6; + + repeated int32 int_field = 7; + repeated int32 int_field_packed = 8 [packed = true]; + repeated int32 int_field_unpacked = 9 [packed = false]; +} diff --git a/src/google/protobuf/editions/proto/editions_transform_proto3_utf8_disabled.proto b/src/google/protobuf/editions/proto/editions_transform_proto3_utf8_disabled.proto new file mode 100644 index 0000000000..1c8e74af2a --- /dev/null +++ b/src/google/protobuf/editions/proto/editions_transform_proto3_utf8_disabled.proto @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package protobuf_editions_test; + + +message TestMessageProto3 { + string string_field = 1; + string string_field_utf = 2; + string string_field_noutf = 3; + + map string_map_field = 4; + + repeated int32 int_field = 7; +} diff --git a/src/google/protobuf/editions/transform.awk b/src/google/protobuf/editions/transform.awk index 58e3ae9f94..23ead050ee 100644 --- a/src/google/protobuf/editions/transform.awk +++ b/src/google/protobuf/editions/transform.awk @@ -41,6 +41,16 @@ function join(array, len, sep) return result } +function strip_option(option, options_list) +{ + # First try to strip out matching commas + sub("\\<" option "\\s*,", "", options_list) + sub(",\\s*" option "\\>", "", options_list) + # Fallback to just stripping the option + sub(option, "", options_list) + return options_list +} + function transform_field(field) { if (!match(field, /\w+\s*=\s*[0-9-]+/)) { @@ -59,20 +69,28 @@ function transform_field(field) num_options = 0 if(syntax == 2) { - sub(/\/, "features.repeated_field_encoding = PACKED", existing_options) - sub(/\,/, "", existing_options) - sub(/,?packed = false\>/, "", existing_options) - if (match($0, /\/)) { - sub(/\/)) { + sub(/\)/)) { + options[++num_options] = "features.string_field_validation = NONE" + } } if(syntax == 3) { + if (disable_utf8 && match(field_def, /^\s*(string|repeated\s*string|map)/)) { + options[++num_options] = "features.string_field_validation = NONE" + } else { + sub(/\/, "features.string_field_validation = HINT", existing_options) + } sub(/\/, "features.repeated_field_encoding = EXPANDED", existing_options) - sub(/\,/, "", existing_options) - sub(/,?packed = true\>/, "", existing_options) + existing_options = strip_option("packed = true", existing_options) + existing_options = strip_option("enforce_utf8 = (true|false)", existing_options) if (match($0, /\/)) { sub(/\/)) {