Editions: Introduce functionality to protoc for generating edition feature set defaults.

This can be used by non-C++ runtimes and generators to seed feature resolution.  The output binary proto message will contain a mapping from edition to the default feature set, including whichever generator feature extensions are passed to protoc.

PiperOrigin-RevId: 563246376

src/google/protobuf/compiler/command_line_interface.cc

@@ -40,6 +40,7 @@
 #include "absl/container/btree_set.h"
 #include "absl/container/flat_hash_map.h"
 #include "absl/status/statusor.h"
+#include "absl/strings/str_cat.h"
 #include "absl/types/span.h"
 #include "google/protobuf/compiler/allowlists/allowlists.h"
 #include "google/protobuf/descriptor_legacy.h"
@@ -98,6 +99,7 @@
 #include "google/protobuf/compiler/subprocess.h"
 #include "google/protobuf/compiler/zip_writer.h"
 #include "google/protobuf/descriptor.h"
+#include "google/protobuf/descriptor.pb.h"
 #include "google/protobuf/dynamic_message.h"
 #include "google/protobuf/io/coded_stream.h"
 #include "google/protobuf/io/io_win32.h"
@@ -1403,6 +1405,12 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) {
     }
   }
 
+  if (!experimental_edition_defaults_out_name_.empty()) {
+    if (!WriteExperimentalEditionDefaults(*descriptor_pool)) {
+      return 1;
+    }
+  }
+
   if (mode_ == MODE_ENCODE || mode_ == MODE_DECODE) {
     if (codec_type_.empty()) {
       // HACK:  Define an EmptyMessage type to use for decoding.
@@ -1684,6 +1692,9 @@ void CommandLineInterface::Clear() {
   dependency_out_name_.clear();
 
   experimental_editions_ = false;
+  experimental_edition_defaults_out_name_.clear();
+  experimental_edition_defaults_minimum_ = EDITION_UNKNOWN;
+  experimental_edition_defaults_maximum_ = EDITION_UNKNOWN;
 
   mode_ = MODE_COMPILE;
   print_mode_ = PRINT_NONE;
@@ -1915,7 +1926,8 @@ CommandLineInterface::ParseArgumentStatus CommandLineInterface::ParseArguments(
     return PARSE_ARGUMENT_FAIL;
   }
   if (mode_ == MODE_COMPILE && output_directives_.empty() &&
-      descriptor_set_out_name_.empty()) {
+      descriptor_set_out_name_.empty() &&
+      experimental_edition_defaults_out_name_.empty()) {
     std::cerr << "Missing output directives." << std::endl;
     return PARSE_ARGUMENT_FAIL;
   }
@@ -2335,6 +2347,43 @@ CommandLineInterface::InterpretArgument(const std::string& name,
     // experimental, undocumented, unsupported flag. Enable it at your own risk
     // (or, just don't!).
     experimental_editions_ = true;
+  } else if (name == "--experimental_edition_defaults_out") {
+    if (!experimental_edition_defaults_out_name_.empty()) {
+      std::cerr << name << " may only be passed once." << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
+    if (value.empty()) {
+      std::cerr << name << " requires a non-empty value." << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
+    if (mode_ != MODE_COMPILE) {
+      std::cerr
+          << "Cannot use --encode or --decode and generate defaults at the "
+             "same time."
+          << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
+    experimental_edition_defaults_out_name_ = value;
+  } else if (name == "--experimental_edition_defaults_minimum") {
+    if (experimental_edition_defaults_minimum_ != EDITION_UNKNOWN) {
+      std::cerr << name << " may only be passed once." << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
+    if (!Edition_Parse(absl::StrCat("EDITION_", value),
+                       &experimental_edition_defaults_minimum_)) {
+      std::cerr << name << " unknown edition \"" << value << "\"." << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
+  } else if (name == "--experimental_edition_defaults_maximum") {
+    if (experimental_edition_defaults_maximum_ != EDITION_UNKNOWN) {
+      std::cerr << name << " may only be passed once." << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
+    if (!Edition_Parse(absl::StrCat("EDITION_", value),
+                       &experimental_edition_defaults_maximum_)) {
+      std::cerr << name << " unknown edition \"" << value << "\"." << std::endl;
+      return PARSE_ARGUMENT_FAIL;
+    }
   } else {
     // Some other flag.  Look it up in the generators list.
     const GeneratorInfo* generator_info = FindGeneratorByFlag(name);
@@ -2616,6 +2665,10 @@ bool CommandLineInterface::GenerateDependencyManifestFile(
     output_filenames.push_back(descriptor_set_out_name_);
   }
 
+  if (!experimental_edition_defaults_out_name_.empty()) {
+    output_filenames.push_back(experimental_edition_defaults_out_name_);
+  }
+
   // Create the depfile, even if it will be empty.
   int fd;
   do {
@@ -2909,6 +2962,73 @@ bool CommandLineInterface::WriteDescriptorSet(
   return true;
 }
 
+bool CommandLineInterface::WriteExperimentalEditionDefaults(
+    const DescriptorPool& pool) {
+  const Descriptor* feature_set =
+      pool.FindMessageTypeByName("google.protobuf.FeatureSet");
+  if (feature_set == nullptr) {
+    std::cerr << experimental_edition_defaults_out_name_
+              << ": Could not find FeatureSet in descriptor pool.  Please make "
+                 "sure descriptor.proto is in your import path"
+              << std::endl;
+    return false;
+  }
+  std::vector<const FieldDescriptor*> extensions;
+  pool.FindAllExtensions(feature_set, &extensions);
+
+  Edition minimum = PROTOBUF_MINIMUM_EDITION;
+  if (experimental_edition_defaults_minimum_ != EDITION_UNKNOWN) {
+    minimum = experimental_edition_defaults_minimum_;
+  }
+  Edition maximum = PROTOBUF_MAXIMUM_EDITION;
+  if (experimental_edition_defaults_maximum_ != EDITION_UNKNOWN) {
+    maximum = experimental_edition_defaults_maximum_;
+  }
+
+  absl::StatusOr<FeatureSetDefaults> defaults =
+      FeatureResolver::CompileDefaults(feature_set, extensions, minimum,
+                                       maximum);
+  if (!defaults.ok()) {
+    std::cerr << experimental_edition_defaults_out_name_ << ": "
+              << defaults.status().message() << std::endl;
+    return false;
+  }
+
+  int fd;
+  do {
+    fd = open(experimental_edition_defaults_out_name_.c_str(),
+              O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
+  } while (fd < 0 && errno == EINTR);
+
+  if (fd < 0) {
+    perror(experimental_edition_defaults_out_name_.c_str());
+    return false;
+  }
+
+  io::FileOutputStream out(fd);
+
+  {
+    io::CodedOutputStream coded_out(&out);
+    // Determinism is useful here because build outputs are sometimes checked
+    // into version control.
+    coded_out.SetSerializationDeterministic(true);
+    if (!defaults->SerializeToCodedStream(&coded_out)) {
+      std::cerr << experimental_edition_defaults_out_name_ << ": "
+                << strerror(out.GetErrno()) << std::endl;
+      out.Close();
+      return false;
+    }
+  }
+
+  if (!out.Close()) {
+    std::cerr << experimental_edition_defaults_out_name_ << ": "
+              << strerror(out.GetErrno()) << std::endl;
+    return false;
+  }
+
+  return true;
+}
+
 const CommandLineInterface::GeneratorInfo*
 CommandLineInterface::FindGeneratorByFlag(const std::string& name) const {
   auto it = generators_by_flag_name_.find(name);

src/google/protobuf/compiler/command_line_interface.h

@@ -49,6 +49,7 @@
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
 #include "absl/strings/string_view.h"
+#include "google/protobuf/descriptor.pb.h"
 #include "google/protobuf/port.h"
 
 // Must be included last.
@@ -311,6 +312,9 @@ class PROTOC_EXPORT CommandLineInterface {
   bool WriteDescriptorSet(
       const std::vector<const FileDescriptor*>& parsed_files);
 
+  // Implements the --experimental_edition_defaults_out option.
+  bool WriteExperimentalEditionDefaults(const DescriptorPool& pool);
+
   // Implements the --dependency_out option
   bool GenerateDependencyManifestFile(
       const std::vector<const FileDescriptor*>& parsed_files,
@@ -448,6 +452,10 @@ class PROTOC_EXPORT CommandLineInterface {
   // FileDescriptorSet should be written.  Otherwise, empty.
   std::string descriptor_set_out_name_;
 
+  std::string experimental_edition_defaults_out_name_;
+  Edition experimental_edition_defaults_minimum_;
+  Edition experimental_edition_defaults_maximum_;
+
   // If --dependency_out was given, this is the path to the file where the
   // dependency file will be written. Otherwise, empty.
   std::string dependency_out_name_;

src/google/protobuf/compiler/command_line_interface_unittest.cc

@@ -206,6 +206,8 @@ class CommandLineInterfaceTest : public CommandLineInterfaceTester {
   void WriteDescriptorSet(absl::string_view filename,
                           const FileDescriptorSet* descriptor_set);
 
+  FeatureSetDefaults ReadEditionDefaults(absl::string_view filename);
+
   // The default code generators support all features. Use this to create a
   // code generator that omits the given feature(s).
   void CreateGeneratorWithMissingFeatures(const std::string& name,
@@ -357,6 +359,15 @@ void CommandLineInterfaceTest::ReadDescriptorSet(
   }
 }
 
+FeatureSetDefaults CommandLineInterfaceTest::ReadEditionDefaults(
+    absl::string_view filename) {
+  FeatureSetDefaults defaults;
+  std::string file_contents = ReadFile(filename);
+  ABSL_CHECK(defaults.ParseFromString(file_contents))
+      << "Could not parse file contents: " << filename;
+  return defaults;
+}
+
 void CommandLineInterfaceTest::WriteDescriptorSet(
     absl::string_view filename, const FileDescriptorSet* descriptor_set) {
   std::string binary_proto;
@@ -1749,6 +1760,233 @@ TEST_F(CommandLineInterfaceTest, PluginNoEditionsSupport) {
       "code generator prefix-gen-plug hasn't been updated to support editions");
 }
 
+TEST_F(CommandLineInterfaceTest, EditionDefaults) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "google/protobuf/descriptor.proto");
+  ExpectNoErrors();
+
+  FeatureSetDefaults defaults = ReadEditionDefaults("defaults");
+  EXPECT_THAT(defaults, EqualsProto(R"pb(
+                defaults {
+                  edition_enum: EDITION_2023
+                  features {
+                    field_presence: EXPLICIT
+                    enum_type: OPEN
+                    repeated_field_encoding: PACKED
+                    message_encoding: LENGTH_PREFIXED
+                    json_format: ALLOW
+                  }
+                }
+                minimum_edition_enum: EDITION_2023
+                maximum_edition_enum: EDITION_2023
+              )pb"));
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsWithMaximum) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "--experimental_edition_defaults_maximum=99997_TEST_ONLY "
+      "google/protobuf/descriptor.proto");
+  ExpectNoErrors();
+
+  FeatureSetDefaults defaults = ReadEditionDefaults("defaults");
+  EXPECT_THAT(defaults, EqualsProto(R"pb(
+                defaults {
+                  edition_enum: EDITION_2023
+                  features {
+                    field_presence: EXPLICIT
+                    enum_type: OPEN
+                    repeated_field_encoding: PACKED
+                    message_encoding: LENGTH_PREFIXED
+                    json_format: ALLOW
+                  }
+                }
+                minimum_edition_enum: EDITION_2023
+                maximum_edition_enum: EDITION_99997_TEST_ONLY
+              )pb"));
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsWithMinimum) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "--experimental_edition_defaults_minimum=99997_TEST_ONLY "
+      "--experimental_edition_defaults_maximum=99999_TEST_ONLY "
+      "google/protobuf/descriptor.proto");
+  ExpectNoErrors();
+
+  FeatureSetDefaults defaults = ReadEditionDefaults("defaults");
+  EXPECT_THAT(defaults, EqualsProto(R"pb(
+                defaults {
+                  edition_enum: EDITION_2023
+                  features {
+                    field_presence: EXPLICIT
+                    enum_type: OPEN
+                    repeated_field_encoding: PACKED
+                    message_encoding: LENGTH_PREFIXED
+                    json_format: ALLOW
+                  }
+                }
+                minimum_edition_enum: EDITION_99997_TEST_ONLY
+                maximum_edition_enum: EDITION_99999_TEST_ONLY
+              )pb"));
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsWithExtension) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  CreateTempFile("features.proto",
+                 pb::TestFeatures::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "--experimental_edition_defaults_maximum=99999_TEST_ONLY "
+      "features.proto google/protobuf/descriptor.proto");
+  ExpectNoErrors();
+
+  FeatureSetDefaults defaults = ReadEditionDefaults("defaults");
+  EXPECT_EQ(defaults.minimum_edition_enum(), EDITION_2023);
+  EXPECT_EQ(defaults.maximum_edition_enum(), EDITION_99999_TEST_ONLY);
+  ASSERT_EQ(defaults.defaults_size(), 3);
+  EXPECT_EQ(defaults.defaults(0).edition_enum(), EDITION_2023);
+  EXPECT_EQ(defaults.defaults(1).edition_enum(), EDITION_99997_TEST_ONLY);
+  EXPECT_EQ(defaults.defaults(2).edition_enum(), EDITION_99998_TEST_ONLY);
+  EXPECT_EQ(
+      defaults.defaults(0).features().GetExtension(pb::test).int_file_feature(),
+      1);
+  EXPECT_EQ(
+      defaults.defaults(1).features().GetExtension(pb::test).int_file_feature(),
+      2);
+  EXPECT_EQ(
+      defaults.defaults(2).features().GetExtension(pb::test).int_file_feature(),
+      3);
+}
+
+#ifndef _WIN32
+TEST_F(CommandLineInterfaceTest, EditionDefaultsDependencyManifest) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  CreateTempFile("features.proto",
+                 pb::TestFeatures::descriptor()->file()->DebugString());
+
+  Run("protocol_compiler --dependency_out=$tmpdir/manifest "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "--proto_path=$tmpdir features.proto");
+
+  ExpectNoErrors();
+
+  ExpectFileContent(
+      "manifest",
+      "$tmpdir/defaults: "
+      "$tmpdir/google/protobuf/descriptor.proto\\\n $tmpdir/features.proto");
+}
+#endif  // _WIN32
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMissingDescriptor) {
+  CreateTempFile("features.proto", R"schema(
+    syntax = "proto2";
+    message Foo {}
+  )schema");
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "features.proto");
+  ExpectErrorSubstring("Could not find FeatureSet in descriptor pool");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidTwice) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring(
+      "experimental_edition_defaults_out may only be passed once");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidEmpty) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_out= "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring(
+      "experimental_edition_defaults_out requires a non-empty value");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidCompile) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--encode=pb.CppFeatures "
+      "--experimental_edition_defaults_out=$tmpdir/defaults "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring("Cannot use --encode or --decode and generate defaults");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMinimumTwice) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_minimum=2023 "
+      "--experimental_edition_defaults_minimum=2023 "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring(
+      "experimental_edition_defaults_minimum may only be passed once");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMinimumEmpty) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_minimum= "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring("unknown edition \"\"");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMinimumUnknown) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_minimum=2022 "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring("unknown edition \"2022\"");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMaximumTwice) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_maximum=2023 "
+      "--experimental_edition_defaults_maximum=2023 "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring(
+      "experimental_edition_defaults_maximum may only be passed once");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMaximumEmpty) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_maximum= "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring("unknown edition \"\"");
+}
+
+TEST_F(CommandLineInterfaceTest, EditionDefaultsInvalidMaximumUnknown) {
+  CreateTempFile("google/protobuf/descriptor.proto",
+                 google::protobuf::DescriptorProto::descriptor()->file()->DebugString());
+  Run("protocol_compiler --proto_path=$tmpdir "
+      "--experimental_edition_defaults_maximum=2022 "
+      "google/protobuf/descriptor.proto");
+  ExpectErrorSubstring("unknown edition \"2022\"");
+}
+
 
 TEST_F(CommandLineInterfaceTest, DirectDependencies_Missing_EmptyList) {
   CreateTempFile("foo.proto",