From 3ced818640df9b30efd25faf34084d3339806766 Mon Sep 17 00:00:00 2001 From: Marcel Hlopko Date: Thu, 20 Jul 2023 00:30:29 -0700 Subject: [PATCH] Add support for proto_library with more than one srcs If a proto_library has more than one srcs, we designate the first one as the primary (that file will be passed to rustc as the crate root). All other files will represent (internal) submodules of the crate. In general, Rust users won't see which .proto file a message came from, they will only see a crate corresponding to the entire proto_library, and in it public submodules for all `package` statements in all .proto files in the proto_library sources. Therefore in this CL we reexport all messages from non primary sources into their corresponding public modules (= packages declared in their owning .proto files). Besides the common case this CL also handles .proto files without package statement, and a subset of behaviors needed for public import functionality. PiperOrigin-RevId: 549543321 --- rust/test/BUILD | 57 ++++- rust/test/no_package.proto | 2 + rust/test/no_package_import.proto | 35 ++++ rust/test/no_package_other.proto | 35 ++++ rust/test/package.proto | 37 ++++ rust/test/package_import.proto | 35 ++++ rust/test/package_other.proto | 35 ++++ rust/test/package_other_different.proto | 35 ++++ rust/test/shared/BUILD | 4 +- rust/test/shared/package_test.rs | 24 ++- src/google/protobuf/compiler/rust/BUILD.bazel | 25 ++- .../protobuf/compiler/rust/generator.cc | 198 +++++++++++++++--- src/google/protobuf/compiler/rust/naming.cc | 10 + src/google/protobuf/compiler/rust/naming.h | 1 + .../protobuf/compiler/rust/relative_path.cc | 103 +++++++++ .../protobuf/compiler/rust/relative_path.h | 77 +++++++ .../compiler/rust/relative_path_test.cc | 67 ++++++ 17 files changed, 740 insertions(+), 40 deletions(-) create mode 100644 rust/test/no_package_import.proto create mode 100644 rust/test/no_package_other.proto create mode 100644 rust/test/package.proto create mode 100644 rust/test/package_import.proto create mode 100644 rust/test/package_other.proto create mode 100644 rust/test/package_other_different.proto create mode 100644 src/google/protobuf/compiler/rust/relative_path.cc create mode 100644 src/google/protobuf/compiler/rust/relative_path.h create mode 100644 src/google/protobuf/compiler/rust/relative_path_test.cc diff --git a/rust/test/BUILD b/rust/test/BUILD index 0ec7aad5b9..85a00eea57 100644 --- a/rust/test/BUILD +++ b/rust/test/BUILD @@ -144,10 +144,21 @@ rust_upb_proto_library( deps = [":dots_in_package_proto"], ) +proto_library( + name = "no_package_import_proto", + testonly = True, + srcs = ["no_package_import.proto"], +) + proto_library( name = "no_package_proto", testonly = True, - srcs = ["no_package.proto"], + srcs = [ + "no_package.proto", + "no_package_other.proto", + ], + exports = [":no_package_import_proto"], + deps = [":no_package_import_proto"], ) cc_proto_library( @@ -176,6 +187,50 @@ rust_upb_proto_library( deps = [":no_package_proto"], ) +proto_library( + name = "package_import_proto", + testonly = True, + srcs = ["package_import.proto"], +) + +proto_library( + name = "package_proto", + testonly = True, + srcs = [ + "package.proto", + "package_other.proto", + "package_other_different.proto", + ], + exports = [":package_import_proto"], + deps = [":package_import_proto"], +) + +cc_proto_library( + name = "package_cc_proto", + testonly = True, + deps = [":package_proto"], +) + +rust_cc_proto_library( + name = "package_cc_rust_proto", + testonly = True, + visibility = [ + "//rust/test/cpp:__subpackages__", + "//rust/test/shared:__subpackages__", + ], + deps = [":package_cc_proto"], +) + +rust_upb_proto_library( + name = "package_upb_rust_proto", + testonly = True, + visibility = [ + "//rust/test/shared:__subpackages__", + "//rust/test/upb:__subpackages__", + ], + deps = [":package_proto"], +) + proto_library( name = "reserved_proto", testonly = True, diff --git a/rust/test/no_package.proto b/rust/test/no_package.proto index c5068a579d..b01df5ef0b 100644 --- a/rust/test/no_package.proto +++ b/rust/test/no_package.proto @@ -32,4 +32,6 @@ syntax = "proto2"; // package intentionally left unspecified. +import public "google/protobuf/rust/test/no_package_import.proto"; + message MsgWithoutPackage {} diff --git a/rust/test/no_package_import.proto b/rust/test/no_package_import.proto new file mode 100644 index 0000000000..1a2495a7de --- /dev/null +++ b/rust/test/no_package_import.proto @@ -0,0 +1,35 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +// package intentionally left unspecified. + +message ImportedMsgWithoutPackage {} diff --git a/rust/test/no_package_other.proto b/rust/test/no_package_other.proto new file mode 100644 index 0000000000..decf2199de --- /dev/null +++ b/rust/test/no_package_other.proto @@ -0,0 +1,35 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +// package intentionally left unspecified. + +message OtherMsgWithoutPackage {} diff --git a/rust/test/package.proto b/rust/test/package.proto new file mode 100644 index 0000000000..21e6c53c2a --- /dev/null +++ b/rust/test/package.proto @@ -0,0 +1,37 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package testing_packages; + +import public "google/protobuf/rust/test/package_import.proto"; + +message MsgWithPackage {} diff --git a/rust/test/package_import.proto b/rust/test/package_import.proto new file mode 100644 index 0000000000..75c2607cc5 --- /dev/null +++ b/rust/test/package_import.proto @@ -0,0 +1,35 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package testing_packages; + +message ImportedMsgWithPackage {} diff --git a/rust/test/package_other.proto b/rust/test/package_other.proto new file mode 100644 index 0000000000..f3061dd3f8 --- /dev/null +++ b/rust/test/package_other.proto @@ -0,0 +1,35 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package testing_packages; + +message OtherMsgWithPackage {} diff --git a/rust/test/package_other_different.proto b/rust/test/package_other_different.proto new file mode 100644 index 0000000000..6645591771 --- /dev/null +++ b/rust/test/package_other_different.proto @@ -0,0 +1,35 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package testing_other_packages; + +message OtherMsgInDifferentPackage {} diff --git a/rust/test/shared/BUILD b/rust/test/shared/BUILD index 140944cf8b..0f65d1eff9 100644 --- a/rust/test/shared/BUILD +++ b/rust/test/shared/BUILD @@ -40,7 +40,7 @@ rust_test( deps = [ "//rust/test:dots_in_package_cc_rust_proto", "//rust/test:no_package_cc_rust_proto", - "//rust/test:unittest_cc_rust_proto", + "//rust/test:package_cc_rust_proto", ], ) @@ -50,7 +50,7 @@ rust_test( deps = [ "//rust/test:dots_in_package_upb_rust_proto", "//rust/test:no_package_upb_rust_proto", - "//rust/test:unittest_upb_rust_proto", + "//rust/test:package_upb_rust_proto", ], ) diff --git a/rust/test/shared/package_test.rs b/rust/test/shared/package_test.rs index 3b2a0abd3f..49fc8771a1 100644 --- a/rust/test/shared/package_test.rs +++ b/rust/test/shared/package_test.rs @@ -31,16 +31,20 @@ /// Tests covering proto packages. #[test] -fn test_package_specified() { - let _foo: unittest_proto::proto2_unittest::TestAllTypes; -} - -#[test] -fn test_empty_package() { +fn test_packages() { + // empty package, message declared in the first .proto source let _foo: no_package_proto::MsgWithoutPackage; -} + // empty package, message declared in other .proto source + let _foo: no_package_proto::OtherMsgWithoutPackage; + // empty package, import public of a message + let _foo: no_package_proto::ImportedMsgWithoutPackage; -#[test] -fn test_dots_in_package() { - let _foo: dots_in_package_proto::package::uses::dots::submodule::separator::Msg; + // package, message declared in the first .proto source + let _foo: package_proto::testing_packages::MsgWithPackage; + // package, message declared in the other .proto source with the same package + let _foo: package_proto::testing_packages::OtherMsgWithPackage; + // package, message declared in the other .proto source with a different package + let _foo: package_proto::testing_other_packages::OtherMsgInDifferentPackage; + // package, import public of a message + let _foo: package_proto::testing_packages::ImportedMsgWithPackage; } diff --git a/src/google/protobuf/compiler/rust/BUILD.bazel b/src/google/protobuf/compiler/rust/BUILD.bazel index 7c1f5b8568..96ee032f09 100644 --- a/src/google/protobuf/compiler/rust/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/BUILD.bazel @@ -19,6 +19,7 @@ cc_library( ":context", ":message", ":naming", + ":relative_path", "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler/cpp:names", @@ -95,4 +96,26 @@ cc_library( "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], -) \ No newline at end of file +) + +cc_library( + name = "relative_path", + srcs = ["relative_path.cc"], + hdrs = ["relative_path.h"], + include_prefix = "google/protobuf/compiler/rust", + deps = [ + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "relative_path_test", + srcs = ["relative_path_test.cc"], + deps = [ + ":relative_path", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/google/protobuf/compiler/rust/generator.cc b/src/google/protobuf/compiler/rust/generator.cc index 5b6340ce8f..a659124bb9 100644 --- a/src/google/protobuf/compiler/rust/generator.cc +++ b/src/google/protobuf/compiler/rust/generator.cc @@ -30,19 +30,25 @@ #include "google/protobuf/compiler/rust/generator.h" +#include #include #include #include #include "absl/algorithm/container.h" +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/memory/memory.h" +#include "absl/status/statusor.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/cpp/names.h" #include "google/protobuf/compiler/rust/context.h" #include "google/protobuf/compiler/rust/message.h" #include "google/protobuf/compiler/rust/naming.h" +#include "google/protobuf/compiler/rust/relative_path.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" @@ -53,9 +59,22 @@ namespace protobuf { namespace compiler { namespace rust { namespace { -void EmitOpeningOfPackageModules(Context file) { - if (file.desc().package().empty()) return; - for (absl::string_view segment : absl::StrSplit(file.desc().package(), '.')) { + +// Emits openings for a tree of submodules for a given `pkg`. +// +// For example for `package.uses.dots.submodule.separator` this function +// generates: +// ``` +// pub mod package { +// pub mod uses { +// pub mod dots { +// pub mod submodule { +// pub mod separator { +// ``` +void EmitOpeningOfPackageModules(absl::string_view pkg, + Context file) { + if (pkg.empty()) return; + for (absl::string_view segment : absl::StrSplit(pkg, '.')) { file.Emit({{"segment", segment}}, R"rs( pub mod $segment$ { @@ -63,10 +82,21 @@ void EmitOpeningOfPackageModules(Context file) { } } -void EmitClosingOfPackageModules(Context file) { - if (file.desc().package().empty()) return; - std::vector segments = - absl::StrSplit(file.desc().package(), '.'); +// Emits closing curly brace for a tree of submodules for a given `pkg`. +// +// For example for `package.uses.dots.submodule.separator` this function +// generates: +// ``` +// } // mod separator +// } // mod submodule +// } // mod dots +// } // mod uses +// } // mod package +// ``` +void EmitClosingOfPackageModules(absl::string_view pkg, + Context file) { + if (pkg.empty()) return; + std::vector segments = absl::StrSplit(pkg, '.'); absl::c_reverse(segments); for (absl::string_view segment : segments) { @@ -75,6 +105,114 @@ void EmitClosingOfPackageModules(Context file) { )rs"); } } + +// Emits `pub use ::Msg` for all messages of a +// `non_primary_src` into the `primary_file`. +// +// `non_primary_src` has to be a non-primary src of the current `proto_library`. +void EmitPubUseOfOwnMessages(Context& primary_file, + const Context& non_primary_src) { + for (int i = 0; i < non_primary_src.desc().message_type_count(); ++i) { + auto msg = primary_file.WithDesc(non_primary_src.desc().message_type(i)); + auto mod = RustInternalModuleName(non_primary_src); + auto name = msg.desc().name(); + primary_file.Emit({{"mod", mod}, {"Msg", name}}, + R"rs( + pub use crate::$mod$::$Msg$; + )rs"); + } +} + +// Emits `pub use ::::Msg` for all messages of a +// `dep` into the `primary_file`. +// +// `dep` is a primary src of a dependency of the current `proto_library`. +// TODO(b/270124215): Add support for public import of non-primary srcs of deps. +void EmitPubUseForImportedMessages(Context& primary_file, + const Context& dep) { + std::string crate_name = GetCrateName(dep); + for (int i = 0; i < dep.desc().message_type_count(); ++i) { + auto msg = primary_file.WithDesc(dep.desc().message_type(i)); + auto path = GetCrateRelativeQualifiedPath(msg); + primary_file.Emit({{"crate", crate_name}, {"pkg::Msg", path}}, + R"rs( + pub use $crate$::$pkg::Msg$; + )rs"); + } +} + +// Emits all public imports of the current file +void EmitPublicImports( + Context& primary_file, + const std::vector& files_in_current_crate) { + absl::flat_hash_set files( + files_in_current_crate.begin(), files_in_current_crate.end()); + for (int i = 0; i < primary_file.desc().public_dependency_count(); ++i) { + auto dep_file = primary_file.desc().public_dependency(i); + // If the publicly imported file is a src of the current `proto_library` + // we don't need to emit `pub use` here, we already do it for all srcs in + // RustGenerator::Generate. In other words, all srcs are implicitly publicly + // imported into the primary file for Protobuf Rust. + // TODO(b/270124215): Handle the case where a non-primary src with the same + // declared package as the primary src publicly imports a file that the + // primary doesn't. + if (files.contains(dep_file)) continue; + auto dep = primary_file.WithDesc(dep_file); + EmitPubUseForImportedMessages(primary_file, dep); + } +} + +// Emits submodule declarations so `rustc` can find non primary sources from the +// primary file. +void DeclareSubmodulesForNonPrimarySrcs( + Context& primary_file, + absl::Span> non_primary_srcs) { + std::string primary_file_path = GetRsFile(primary_file); + RelativePath primary_relpath(primary_file_path); + for (const auto& non_primary_src : non_primary_srcs) { + std::string non_primary_file_path = GetRsFile(non_primary_src); + std::string relative_mod_path = + primary_relpath.Relative(RelativePath(non_primary_file_path)); + primary_file.Emit({{"file_path", relative_mod_path}, + {"foo", primary_file_path}, + {"bar", non_primary_file_path}, + {"mod_name", RustInternalModuleName(non_primary_src)}}, + R"rs( + #[path="$file_path$"] + pub mod $mod_name$; + )rs"); + } +} + +// Emits `pub use <...>::Msg` for all messages in non primary sources into their +// corresponding packages (each source file can declare a different package). +// +// Returns the non-primary sources that should be reexported from the package of +// the primary file. +std::vector*> ReexportMessagesFromSubmodules( + Context& primary_file, + absl::Span> non_primary_srcs) { + absl::btree_map*>> + packages; + for (const Context& ctx : non_primary_srcs) { + packages[ctx.desc().package()].push_back(&ctx); + } + for (const auto& pair : packages) { + // We will deal with messages for the package of the primary file later. + auto fds = pair.second; + absl::string_view package = fds[0]->desc().package(); + if (package == primary_file.desc().package()) continue; + + EmitOpeningOfPackageModules(package, primary_file); + for (const Context* c : fds) { + EmitPubUseOfOwnMessages(primary_file, *c); + } + EmitClosingOfPackageModules(package, primary_file); + } + + return packages[primary_file.desc().package()]; +} } // namespace bool RustGenerator::Generate(const FileDescriptor* file_desc, @@ -107,27 +245,33 @@ bool RustGenerator::Generate(const FileDescriptor* file_desc, extern crate std as __std; )rs"); - EmitOpeningOfPackageModules(file); - - // TODO(b/270124215): Delete the following "placeholder impl" of `import - // public`. Also make sure to figure out how to map FileDescriptor#name to - // Rust crate names (currently Bazel labels). - for (int i = 0; i < file.desc().public_dependency_count(); ++i) { - auto dep = file.WithDesc(file.desc().public_dependency(i)); - std::string crate_name = GetCrateName(dep); - for (int j = 0; j < dep.desc().message_type_count(); ++j) { - auto msg = file.WithDesc(dep.desc().message_type(j)); - file.Emit( - { - {"crate", crate_name}, - {"pkg::Msg", GetCrateRelativeQualifiedPath(msg)}, - }, - R"rs( - pub use $crate$::$pkg::Msg$; - )rs"); + + std::vector files_in_current_crate; + generator_context->ListParsedFiles(&files_in_current_crate); + std::vector> file_contexts; + for (const FileDescriptor* f : files_in_current_crate) { + file_contexts.push_back(file.WithDesc(*f)); + } + + // Generating the primary file? + if (file_desc == files_in_current_crate.front()) { + auto non_primary_srcs = absl::MakeConstSpan(file_contexts).subspan(1); + DeclareSubmodulesForNonPrimarySrcs(file, non_primary_srcs); + + std::vector*> + non_primary_srcs_in_primary_package = + ReexportMessagesFromSubmodules(file, non_primary_srcs); + + EmitOpeningOfPackageModules(file.desc().package(), file); + + for (const Context* non_primary_file : + non_primary_srcs_in_primary_package) { + EmitPubUseOfOwnMessages(file, *non_primary_file); } } + EmitPublicImports(file, files_in_current_crate); + std::unique_ptr thunks_cc; std::unique_ptr thunks_printer; if (file.is_cpp()) { @@ -158,7 +302,9 @@ bool RustGenerator::Generate(const FileDescriptor* file_desc, thunks_msg.printer().PrintRaw("\n"); } } - EmitClosingOfPackageModules(file); + if (file_desc == files_in_current_crate.front()) { + EmitClosingOfPackageModules(file.desc().package(), file); + } return true; } diff --git a/src/google/protobuf/compiler/rust/naming.cc b/src/google/protobuf/compiler/rust/naming.cc index 224fd3f0aa..2cb3897b10 100644 --- a/src/google/protobuf/compiler/rust/naming.cc +++ b/src/google/protobuf/compiler/rust/naming.cc @@ -36,6 +36,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" #include "absl/strings/substitute.h" #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/rust/context.h" @@ -164,7 +165,16 @@ std::string RustModule(Context msg) { return absl::StrCat("", absl::StrReplaceAll(package, {{".", "::"}})); } +std::string RustInternalModuleName(Context file) { + // TODO(b/291853557): Introduce a more robust mangling here to avoid conflicts + // between `foo/bar/baz.proto` and `foo_bar/baz.proto`. + return absl::StrReplaceAll(StripProto(file.desc().name()), {{"/", "_"}}); +} + std::string GetCrateRelativeQualifiedPath(Context msg) { + if (msg.desc().file()->package().empty()) { + return msg.desc().name(); + } return absl::StrCat(RustModule(msg), "::", msg.desc().name()); } diff --git a/src/google/protobuf/compiler/rust/naming.h b/src/google/protobuf/compiler/rust/naming.h index e6b343c8b2..07dba8796e 100644 --- a/src/google/protobuf/compiler/rust/naming.h +++ b/src/google/protobuf/compiler/rust/naming.h @@ -58,6 +58,7 @@ absl::string_view PrimitiveRsTypeName(Context field); std::string FieldInfoComment(Context field); std::string RustModule(Context msg); +std::string RustInternalModuleName(Context file); std::string GetCrateRelativeQualifiedPath(Context msg); } // namespace rust diff --git a/src/google/protobuf/compiler/rust/relative_path.cc b/src/google/protobuf/compiler/rust/relative_path.cc new file mode 100644 index 0000000000..3af43819c6 --- /dev/null +++ b/src/google/protobuf/compiler/rust/relative_path.cc @@ -0,0 +1,103 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/rust/relative_path.h" + +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/strings/match.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace rust { + +std::vector RelativePath::Segments() const { + return absl::StrSplit(this->path_, '/', absl::SkipWhitespace()); +} + +bool RelativePath::IsDirectory() const { + return absl::EndsWith(this->path_, "/"); +} + +std::string RelativePath::Relative(const RelativePath& dest) const { + ABSL_CHECK(!dest.IsDirectory()) + << "`dest` has to be a file path, but is a directory."; + std::vector current_segments = this->Segments(); + + if (!current_segments.empty() && !this->IsDirectory()) { + // `this` represents a file path, skip the last segment to get its + // directory. + current_segments.pop_back(); + } + + std::vector dest_segments = dest.Segments(); + + // Find the lowest common ancestor. + absl::c_reverse(current_segments); + absl::c_reverse(dest_segments); + while (true) { + if (current_segments.empty()) break; + if (dest_segments.empty()) break; + if (current_segments.back() != dest_segments.back()) break; + + current_segments.pop_back(); + dest_segments.pop_back(); + } + + // Construct the relative path in reverse order. + std::vector result; + result.reserve(current_segments.size() + dest_segments.size()); + // Push the segments from the `dest` to the common ancestor. + for (const auto& segment : dest_segments) { + result.push_back(segment); + } + // Push `..` from the common ancestor to the current path. + for (int i = 0; i < current_segments.size(); ++i) { + result.push_back(".."); + } + absl::c_reverse(result); + if (dest.IsDirectory()) { + // Convince the `StrJoin` below to add a trailing `/`. + result.push_back(""); + } + return absl::StrJoin(result, "/"); +} + +} // namespace rust +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/rust/relative_path.h b/src/google/protobuf/compiler/rust/relative_path.h new file mode 100644 index 0000000000..f56b09ad39 --- /dev/null +++ b/src/google/protobuf/compiler/rust/relative_path.h @@ -0,0 +1,77 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_RUST_RELATIVE_PATH_H__ +#define GOOGLE_PROTOBUF_COMPILER_RUST_RELATIVE_PATH_H__ + +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/log/absl_check.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace rust { + +// Relative path using '/' as a separator. +class RelativePath final { + public: + explicit RelativePath(absl::string_view path) : path_(path) { + ABSL_CHECK(!absl::StartsWith(path, "/")) + << "only relative paths are supported"; + // `..` and `.` not supported, since there's no use case for that right now. + for (absl::string_view segment : Segments()) { + ABSL_CHECK(segment != "..") << "`..` segments are not supported"; + ABSL_CHECK(segment != ".") << "`.` segments are not supported"; + } + } + + // Returns a path getting us from the current relative path to the `dest` + // path. + // + // Supports both files and directories. + std::string Relative(const RelativePath& dest) const; + std::vector Segments() const; + bool IsDirectory() const; + + private: + absl::string_view path_; +}; + +} // namespace rust +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_COMPILER_RUST_RELATIVE_PATH_H__ diff --git a/src/google/protobuf/compiler/rust/relative_path_test.cc b/src/google/protobuf/compiler/rust/relative_path_test.cc new file mode 100644 index 0000000000..82bab106f7 --- /dev/null +++ b/src/google/protobuf/compiler/rust/relative_path_test.cc @@ -0,0 +1,67 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/rust/relative_path.h" + +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace rust { +namespace { + +using testing::Eq; + +TEST(RelativePathTest, GetRelativePath) { + auto relative = [](absl::string_view from_path, absl::string_view to_path) { + return RelativePath(from_path).Relative(RelativePath(to_path)); + }; + EXPECT_EQ(relative("foo/bar/baz.txt", "foo/bar/file.txt"), "file.txt"); + EXPECT_EQ(relative("foo/bar/", "foo/bar/file.txt"), "file.txt"); + + EXPECT_EQ(relative("foo/bar/baz.txt", "foo/file.txt"), "../file.txt"); + EXPECT_EQ(relative("foo/bar/", "foo/file.txt"), "../file.txt"); + + EXPECT_EQ(relative("foo/baz.txt", "foo/bar/baz/file.txt"), + "bar/baz/file.txt"); + EXPECT_EQ(relative("foo/", "foo/bar/baz/file.txt"), "bar/baz/file.txt"); + + EXPECT_EQ(relative("baz.txt", "foo/bar/file.txt"), "foo/bar/file.txt"); + EXPECT_EQ(relative("", "foo/bar/file.txt"), "foo/bar/file.txt"); +} + +} // namespace + +} // namespace rust +} // namespace compiler +} // namespace protobuf +} // namespace google