From 4019e254322b2312cc7431b45ae71bb8e3dfa035 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 6 Sep 2023 16:19:08 -0700 Subject: [PATCH] Editions: Introduce functionality to protoc for generating edition feature set defaults. This can be used by non-C++ runtimes and generators to seed feature resolution. The output binary proto message will contain a mapping from edition to the default feature set, including whichever generator feature extensions are passed to protoc. PiperOrigin-RevId: 563246376 --- .../compiler/command_line_interface.cc | 122 ++++++++- .../compiler/command_line_interface.h | 8 + .../command_line_interface_unittest.cc | 238 ++++++++++++++++++ 3 files changed, 367 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index a17cf83a87..4b08256585 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -40,6 +40,7 @@ #include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "absl/types/span.h" #include "google/protobuf/compiler/allowlists/allowlists.h" #include "google/protobuf/descriptor_legacy.h" @@ -98,6 +99,7 @@ #include "google/protobuf/compiler/subprocess.h" #include "google/protobuf/compiler/zip_writer.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" #include "google/protobuf/dynamic_message.h" #include "google/protobuf/io/coded_stream.h" #include "google/protobuf/io/io_win32.h" @@ -1403,6 +1405,12 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) { } } + if (!experimental_edition_defaults_out_name_.empty()) { + if (!WriteExperimentalEditionDefaults(*descriptor_pool)) { + return 1; + } + } + if (mode_ == MODE_ENCODE || mode_ == MODE_DECODE) { if (codec_type_.empty()) { // HACK: Define an EmptyMessage type to use for decoding. @@ -1684,6 +1692,9 @@ void CommandLineInterface::Clear() { dependency_out_name_.clear(); experimental_editions_ = false; + experimental_edition_defaults_out_name_.clear(); + experimental_edition_defaults_minimum_ = EDITION_UNKNOWN; + experimental_edition_defaults_maximum_ = EDITION_UNKNOWN; mode_ = MODE_COMPILE; print_mode_ = PRINT_NONE; @@ -1915,7 +1926,8 @@ CommandLineInterface::ParseArgumentStatus CommandLineInterface::ParseArguments( return PARSE_ARGUMENT_FAIL; } if (mode_ == MODE_COMPILE && output_directives_.empty() && - descriptor_set_out_name_.empty()) { + descriptor_set_out_name_.empty() && + experimental_edition_defaults_out_name_.empty()) { std::cerr << "Missing output directives." << std::endl; return PARSE_ARGUMENT_FAIL; } @@ -2335,6 +2347,43 @@ CommandLineInterface::InterpretArgument(const std::string& name, // experimental, undocumented, unsupported flag. Enable it at your own risk // (or, just don't!). experimental_editions_ = true; + } else if (name == "--experimental_edition_defaults_out") { + if (!experimental_edition_defaults_out_name_.empty()) { + std::cerr << name << " may only be passed once." << std::endl; + return PARSE_ARGUMENT_FAIL; + } + if (value.empty()) { + std::cerr << name << " requires a non-empty value." << std::endl; + return PARSE_ARGUMENT_FAIL; + } + if (mode_ != MODE_COMPILE) { + std::cerr + << "Cannot use --encode or --decode and generate defaults at the " + "same time." + << std::endl; + return PARSE_ARGUMENT_FAIL; + } + experimental_edition_defaults_out_name_ = value; + } else if (name == "--experimental_edition_defaults_minimum") { + if (experimental_edition_defaults_minimum_ != EDITION_UNKNOWN) { + std::cerr << name << " may only be passed once." << std::endl; + return PARSE_ARGUMENT_FAIL; + } + if (!Edition_Parse(absl::StrCat("EDITION_", value), + &experimental_edition_defaults_minimum_)) { + std::cerr << name << " unknown edition \"" << value << "\"." << std::endl; + return PARSE_ARGUMENT_FAIL; + } + } else if (name == "--experimental_edition_defaults_maximum") { + if (experimental_edition_defaults_maximum_ != EDITION_UNKNOWN) { + std::cerr << name << " may only be passed once." << std::endl; + return PARSE_ARGUMENT_FAIL; + } + if (!Edition_Parse(absl::StrCat("EDITION_", value), + &experimental_edition_defaults_maximum_)) { + std::cerr << name << " unknown edition \"" << value << "\"." << std::endl; + return PARSE_ARGUMENT_FAIL; + } } else { // Some other flag. Look it up in the generators list. const GeneratorInfo* generator_info = FindGeneratorByFlag(name); @@ -2616,6 +2665,10 @@ bool CommandLineInterface::GenerateDependencyManifestFile( output_filenames.push_back(descriptor_set_out_name_); } + if (!experimental_edition_defaults_out_name_.empty()) { + output_filenames.push_back(experimental_edition_defaults_out_name_); + } + // Create the depfile, even if it will be empty. int fd; do { @@ -2909,6 +2962,73 @@ bool CommandLineInterface::WriteDescriptorSet( return true; } +bool CommandLineInterface::WriteExperimentalEditionDefaults( + const DescriptorPool& pool) { + const Descriptor* feature_set = + pool.FindMessageTypeByName("google.protobuf.FeatureSet"); + if (feature_set == nullptr) { + std::cerr << experimental_edition_defaults_out_name_ + << ": Could not find FeatureSet in descriptor pool. Please make " + "sure descriptor.proto is in your import path" + << std::endl; + return false; + } + std::vector extensions; + pool.FindAllExtensions(feature_set, &extensions); + + Edition minimum = PROTOBUF_MINIMUM_EDITION; + if (experimental_edition_defaults_minimum_ != EDITION_UNKNOWN) { + minimum = experimental_edition_defaults_minimum_; + } + Edition maximum = PROTOBUF_MAXIMUM_EDITION; + if (experimental_edition_defaults_maximum_ != EDITION_UNKNOWN) { + maximum = experimental_edition_defaults_maximum_; + } + + absl::StatusOr defaults = + FeatureResolver::CompileDefaults(feature_set, extensions, minimum, + maximum); + if (!defaults.ok()) { + std::cerr << experimental_edition_defaults_out_name_ << ": " + << defaults.status().message() << std::endl; + return false; + } + + int fd; + do { + fd = open(experimental_edition_defaults_out_name_.c_str(), + O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666); + } while (fd < 0 && errno == EINTR); + + if (fd < 0) { + perror(experimental_edition_defaults_out_name_.c_str()); + return false; + } + + io::FileOutputStream out(fd); + + { + io::CodedOutputStream coded_out(&out); + // Determinism is useful here because build outputs are sometimes checked + // into version control. + coded_out.SetSerializationDeterministic(true); + if (!defaults->SerializeToCodedStream(&coded_out)) { + std::cerr << experimental_edition_defaults_out_name_ << ": " + << strerror(out.GetErrno()) << std::endl; + out.Close(); + return false; + } + } + + if (!out.Close()) { + std::cerr << experimental_edition_defaults_out_name_ << ": " + << strerror(out.GetErrno()) << std::endl; + return false; + } + + return true; +} + const CommandLineInterface::GeneratorInfo* CommandLineInterface::FindGeneratorByFlag(const std::string& name) const { auto it = generators_by_flag_name_.find(name); diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h index 65c3f58d9b..ca806e87eb 100644 --- a/src/google/protobuf/compiler/command_line_interface.h +++ b/src/google/protobuf/compiler/command_line_interface.h @@ -49,6 +49,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" +#include "google/protobuf/descriptor.pb.h" #include "google/protobuf/port.h" // Must be included last. @@ -311,6 +312,9 @@ class PROTOC_EXPORT CommandLineInterface { bool WriteDescriptorSet( const std::vector& parsed_files); + // Implements the --experimental_edition_defaults_out option. + bool WriteExperimentalEditionDefaults(const DescriptorPool& pool); + // Implements the --dependency_out option bool GenerateDependencyManifestFile( const std::vector& parsed_files, @@ -448,6 +452,10 @@ class PROTOC_EXPORT CommandLineInterface { // FileDescriptorSet should be written. Otherwise, empty. std::string descriptor_set_out_name_; + std::string experimental_edition_defaults_out_name_; + Edition experimental_edition_defaults_minimum_; + Edition experimental_edition_defaults_maximum_; + // If --dependency_out was given, this is the path to the file where the // dependency file will be written. Otherwise, empty. std::string dependency_out_name_; diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index 0f74d4c8b5..1e90dde7ab 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -206,6 +206,8 @@ class CommandLineInterfaceTest : public CommandLineInterfaceTester { void WriteDescriptorSet(absl::string_view filename, const FileDescriptorSet* descriptor_set); + FeatureSetDefaults ReadEditionDefaults(absl::string_view filename); + // The default code generators support all features. Use this to create a // code generator that omits the given feature(s). void CreateGeneratorWithMissingFeatures(const std::string& name, @@ -357,6 +359,15 @@ void CommandLineInterfaceTest::ReadDescriptorSet( } } +FeatureSetDefaults CommandLineInterfaceTest::ReadEditionDefaults( + absl::string_view filename) { + FeatureSetDefaults defaults; + std::string file_contents = ReadFile(filename); + ABSL_CHECK(defaults.ParseFromString(file_contents)) + << "Could not parse file contents: " << filename; + return defaults; +} + void CommandLineInterfaceTest::WriteDescriptorSet( absl::string_view filename, const FileDescriptorSet* descriptor_set) { std::string binary_proto; @@ -1749,6 +1760,233 @@ TEST_F(CommandLineInterfaceTest, PluginNoEditionsSupport) { "code generator prefix-gen-plug hasn't been updated to support editions"); } +TEST_F(CommandLineInterfaceTest, EditionDefaults) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "google/protobuf/descriptor.proto"); + ExpectNoErrors(); + + FeatureSetDefaults defaults = ReadEditionDefaults("defaults"); + EXPECT_THAT(defaults, EqualsProto(R"pb( + defaults { + edition_enum: EDITION_2023 + features { + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + } + } + minimum_edition_enum: EDITION_2023 + maximum_edition_enum: EDITION_2023 + )pb")); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsWithMaximum) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "--experimental_edition_defaults_maximum=99997_TEST_ONLY " + "google/protobuf/descriptor.proto"); + ExpectNoErrors(); + + FeatureSetDefaults defaults = ReadEditionDefaults("defaults"); + EXPECT_THAT(defaults, EqualsProto(R"pb( + defaults { + edition_enum: EDITION_2023 + features { + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + } + } + minimum_edition_enum: EDITION_2023 + maximum_edition_enum: EDITION_99997_TEST_ONLY + )pb")); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsWithMinimum) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "--experimental_edition_defaults_minimum=99997_TEST_ONLY " + "--experimental_edition_defaults_maximum=99999_TEST_ONLY " + "google/protobuf/descriptor.proto"); + ExpectNoErrors(); + + FeatureSetDefaults defaults = ReadEditionDefaults("defaults"); + EXPECT_THAT(defaults, EqualsProto(R"pb( + defaults { + edition_enum: EDITION_2023 + features { + field_presence: EXPLICIT + enum_type: OPEN + repeated_field_encoding: PACKED + message_encoding: LENGTH_PREFIXED + json_format: ALLOW + } + } + minimum_edition_enum: EDITION_99997_TEST_ONLY + maximum_edition_enum: EDITION_99999_TEST_ONLY + )pb")); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsWithExtension) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + CreateTempFile("features.proto", + pb::TestFeatures::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "--experimental_edition_defaults_maximum=99999_TEST_ONLY " + "features.proto google/protobuf/descriptor.proto"); + ExpectNoErrors(); + + FeatureSetDefaults defaults = ReadEditionDefaults("defaults"); + EXPECT_EQ(defaults.minimum_edition_enum(), EDITION_2023); + EXPECT_EQ(defaults.maximum_edition_enum(), EDITION_99999_TEST_ONLY); + ASSERT_EQ(defaults.defaults_size(), 3); + EXPECT_EQ(defaults.defaults(0).edition_enum(), EDITION_2023); + EXPECT_EQ(defaults.defaults(1).edition_enum(), EDITION_99997_TEST_ONLY); + EXPECT_EQ(defaults.defaults(2).edition_enum(), EDITION_99998_TEST_ONLY); + EXPECT_EQ( + defaults.defaults(0).features().GetExtension(pb::test).int_file_feature(), + 1); + EXPECT_EQ( + defaults.defaults(1).features().GetExtension(pb::test).int_file_feature(), + 2); + EXPECT_EQ( + defaults.defaults(2).features().GetExtension(pb::test).int_file_feature(), + 3); +} + +#ifndef _WIN32 +TEST_F(CommandLineInterfaceTest, EditionDefaultsDependencyManifest) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + CreateTempFile("features.proto", + pb::TestFeatures::descriptor()->file()->DebugString()); + + Run("protocol_compiler --dependency_out=$tmpdir/manifest " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "--proto_path=$tmpdir features.proto"); + + ExpectNoErrors(); + + ExpectFileContent( + "manifest", + "$tmpdir/defaults: " + "$tmpdir/google/protobuf/descriptor.proto\\\n $tmpdir/features.proto"); +} +#endif // _WIN32 + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMissingDescriptor) { + CreateTempFile("features.proto", R"schema( + syntax = "proto2"; + message Foo {} + )schema"); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "features.proto"); + ExpectErrorSubstring("Could not find FeatureSet in descriptor pool"); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidTwice) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring( + "experimental_edition_defaults_out may only be passed once"); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidEmpty) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_out= " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring( + "experimental_edition_defaults_out requires a non-empty value"); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidCompile) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--encode=pb.CppFeatures " + "--experimental_edition_defaults_out=$tmpdir/defaults " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring("Cannot use --encode or --decode and generate defaults"); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMinimumTwice) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_minimum=2023 " + "--experimental_edition_defaults_minimum=2023 " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring( + "experimental_edition_defaults_minimum may only be passed once"); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMinimumEmpty) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_minimum= " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring("unknown edition \"\""); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMinimumUnknown) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_minimum=2022 " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring("unknown edition \"2022\""); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMaximumTwice) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_maximum=2023 " + "--experimental_edition_defaults_maximum=2023 " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring( + "experimental_edition_defaults_maximum may only be passed once"); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMaximumEmpty) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_maximum= " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring("unknown edition \"\""); +} + +TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMaximumUnknown) { + CreateTempFile("google/protobuf/descriptor.proto", + google::protobuf::DescriptorProto::descriptor()->file()->DebugString()); + Run("protocol_compiler --proto_path=$tmpdir " + "--experimental_edition_defaults_maximum=2022 " + "google/protobuf/descriptor.proto"); + ExpectErrorSubstring("unknown edition \"2022\""); +} + TEST_F(CommandLineInterfaceTest, DirectDependencies_Missing_EmptyList) { CreateTempFile("foo.proto",