build: Move cc_proto_descriptor_library into api/ (#29837)

Signed-off-by: Ryan Hamilton <rch@google.com>

Mirrored from https://github.com/envoyproxy/envoy @ 56c88dd4ad33a611a2e622edf7eb3dbaadb59ce1
main
update-envoy[bot] 1 year ago
parent 5e2e6438f4
commit 2e958bf11a
  1. 2
      bazel/api_build_system.bzl
  2. 66
      bazel/cc_proto_descriptor_library/BUILD
  3. 349
      bazel/cc_proto_descriptor_library/builddefs.bzl
  4. 31
      bazel/cc_proto_descriptor_library/create_dynamic_message.cc
  5. 26
      bazel/cc_proto_descriptor_library/create_dynamic_message.h
  6. 176
      bazel/cc_proto_descriptor_library/file_descriptor_generator.cc
  7. 17
      bazel/cc_proto_descriptor_library/file_descriptor_generator.h
  8. 10
      bazel/cc_proto_descriptor_library/file_descriptor_generator_main.cc
  9. 29
      bazel/cc_proto_descriptor_library/file_descriptor_info.cc
  10. 21
      bazel/cc_proto_descriptor_library/file_descriptor_info.h
  11. 135
      bazel/cc_proto_descriptor_library/testdata/BUILD
  12. 38
      bazel/cc_proto_descriptor_library/testdata/create_dynamic_message_test.cc
  13. 41
      bazel/cc_proto_descriptor_library/testdata/global_fallback_test.cc
  14. 9
      bazel/cc_proto_descriptor_library/testdata/test-extension.proto
  15. 12
      bazel/cc_proto_descriptor_library/testdata/test.proto
  16. 9
      bazel/cc_proto_descriptor_library/testdata/test1.proto
  17. 100
      bazel/cc_proto_descriptor_library/testdata/text_format_transcoder_test.cc
  18. 128
      bazel/cc_proto_descriptor_library/text_format_transcoder.cc
  19. 65
      bazel/cc_proto_descriptor_library/text_format_transcoder.h

@ -11,7 +11,7 @@ load(
"EXTERNAL_PROTO_PY_BAZEL_DEP_MAP",
)
load(
"@envoy//bazel/cc_proto_descriptor_library:builddefs.bzl",
"//bazel/cc_proto_descriptor_library:builddefs.bzl",
"cc_proto_descriptor_library",
)

@ -0,0 +1,66 @@
licenses(["notice"]) # Apache 2
cc_library(
name = "file_descriptor_info",
hdrs = ["file_descriptor_info.h"],
visibility = ["//visibility:public"],
deps = [
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "text_format_transcoder",
srcs = [
"create_dynamic_message.h",
"text_format_transcoder.cc",
],
hdrs = ["text_format_transcoder.h"],
visibility = ["//visibility:public"],
deps = [
":file_descriptor_info",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_protobuf//:protobuf",
],
)
cc_library(
name = "file_descriptor_generator_lib",
srcs = ["file_descriptor_generator.cc"],
hdrs = ["file_descriptor_generator.h"],
deps = [
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_protobuf//:protobuf",
"@com_google_protobuf//src/google/protobuf/compiler:code_generator",
"@com_google_protobuf//src/google/protobuf/compiler:retention",
],
)
cc_binary(
name = "file_descriptor_generator",
srcs = ["file_descriptor_generator_main.cc"],
visibility = ["//visibility:public"],
deps = [
":file_descriptor_generator_lib",
"@com_google_protobuf//:protobuf",
"@com_google_protobuf//:protoc_lib",
"@com_google_protobuf//src/google/protobuf/compiler:code_generator",
],
)
cc_library(
name = "create_dynamic_message",
srcs = ["create_dynamic_message.cc"],
hdrs = ["create_dynamic_message.h"],
visibility = ["//visibility:public"],
deps = [
":text_format_transcoder",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_protobuf//:protobuf",
],
)

@ -0,0 +1,349 @@
"""Public rules for using cc proto descriptor library with protos:
- cc_proto_descriptor_library()
"""
load("@bazel_skylib//lib:paths.bzl", "paths")
# begin:google_only
# load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain", "use_cpp_toolchain")
# end:google_only
# begin:github_only
# Compatibility code for Bazel 4.x. Remove this when we drop support for Bazel 4.x.
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
def use_cpp_toolchain():
return ["@bazel_tools//tools/cpp:toolchain_type"]
# end:github_only
# Generic support code #########################################################
# begin:github_only
_is_google3 = False
# end:github_only
# begin:google_only
# _is_google3 = True
# end:google_only
def _get_real_short_path(file):
# For some reason, files from other archives have short paths that look like:
# ../com_google_protobuf/google/protobuf/descriptor.proto
short_path = file.short_path
if short_path.startswith("../"):
second_slash = short_path.index("/", 3)
short_path = short_path[second_slash + 1:]
# Sometimes it has another few prefixes like:
# _virtual_imports/any_proto/google/protobuf/any.proto
# benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto
# We want just google/protobuf/any.proto.
virtual_imports = "_virtual_imports/"
if virtual_imports in short_path:
short_path = short_path.split(virtual_imports)[1].split("/", 1)[1]
return short_path
def _get_real_root(ctx, file):
real_short_path = _get_real_short_path(file)
root = file.path[:-len(real_short_path) - 1]
if not _is_google3 and ctx.rule.attr.strip_import_prefix:
root = paths.join(root, ctx.rule.attr.strip_import_prefix[1:])
return root
def _generate_output_file(ctx, src, extension):
package = ctx.label.package
if not _is_google3:
strip_import_prefix = ctx.rule.attr.strip_import_prefix
if strip_import_prefix and strip_import_prefix != "/":
if not package.startswith(strip_import_prefix[1:]):
fail("%s does not begin with prefix %s" % (package, strip_import_prefix))
package = package[len(strip_import_prefix):]
real_short_path = _get_real_short_path(src)
real_short_path = paths.relativize(real_short_path, package)
output_filename = paths.replace_extension(real_short_path, extension)
ret = ctx.actions.declare_file(output_filename)
return ret
def _generate_include_path(src, out, extension):
short_path = _get_real_short_path(src)
short_path = paths.replace_extension(short_path, extension)
if not out.path.endswith(short_path):
fail("%s does not end with %s" % (out.path, short_path))
return out.path[:-len(short_path)]
def _filter_none(elems):
out = []
for elem in elems:
if elem:
out.append(elem)
return out
def _cc_library_func(ctx, name, hdrs, srcs, copts, includes, dep_ccinfos):
"""Like cc_library(), but callable from rules.
Args:
ctx: Rule context.
name: Unique name used to generate output files.
hdrs: Public headers that can be #included from other rules.
srcs: C/C++ source files.
copts: Additional options for cc compilation.
includes: Additional include paths.
dep_ccinfos: CcInfo providers of dependencies we should build/link against.
Returns:
CcInfo provider for this compilation.
"""
compilation_contexts = [info.compilation_context for info in dep_ccinfos]
linking_contexts = [info.linking_context for info in dep_ccinfos]
toolchain = find_cpp_toolchain(ctx)
feature_configuration = cc_common.configure_features(
ctx = ctx,
cc_toolchain = toolchain,
requested_features = ctx.features,
unsupported_features = ctx.disabled_features,
)
blaze_only_args = {}
if _is_google3:
blaze_only_args["grep_includes"] = ctx.file._grep_includes
(compilation_context, compilation_outputs) = cc_common.compile(
actions = ctx.actions,
feature_configuration = feature_configuration,
cc_toolchain = toolchain,
name = name,
srcs = srcs,
includes = includes,
public_hdrs = hdrs,
user_compile_flags = copts,
compilation_contexts = compilation_contexts,
**blaze_only_args
)
# buildifier: disable=unused-variable
(linking_context, linking_outputs) = cc_common.create_linking_context_from_compilation_outputs(
actions = ctx.actions,
name = name,
feature_configuration = feature_configuration,
cc_toolchain = toolchain,
compilation_outputs = compilation_outputs,
linking_contexts = linking_contexts,
disallow_dynamic_library = cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "targets_windows"),
**blaze_only_args
)
return CcInfo(
compilation_context = compilation_context,
linking_context = linking_context,
)
# Dummy rule to expose select() copts to aspects ##############################
CcProtoDescriptorLibraryCoptsInfo = provider(
"Provides copts for cc proto descriptor library targets",
fields = {
"copts": "copts for cc_proto_descriptor_library()",
},
)
def cc_proto_descriptor_library_copts_impl(ctx):
return CcProtoDescriptorLibraryCoptsInfo(copts = ctx.attr.copts)
cc_proto_descriptor_library_copts = rule(
implementation = cc_proto_descriptor_library_copts_impl,
attrs = {"copts": attr.string_list(default = [])},
)
# cc_proto_descriptor_library shared code #################
GeneratedSrcsInfo = provider(
"Provides generated headers and sources",
fields = {
"srcs": "list of srcs",
"hdrs": "list of hdrs",
"includes": "list of extra includes",
},
)
CcProtoDescriptorWrappedCcInfo = provider("Provider for cc_info for protos", fields = ["cc_info"])
_CcProtoDescriptorWrappedGeneratedSrcsInfo = provider("Provider for generated sources", fields = ["srcs"])
def _compile_protos(ctx, generator, proto_info, proto_sources):
if len(proto_sources) == 0:
return GeneratedSrcsInfo(srcs = [], hdrs = [], includes = [])
ext = "_" + generator
tool = getattr(ctx.executable, "_gen_" + generator)
srcs = [_generate_output_file(ctx, name, ext + ".pb.cc") for name in proto_sources]
hdrs = [_generate_output_file(ctx, name, ext + ".pb.h") for name in proto_sources]
transitive_sets = proto_info.transitive_descriptor_sets.to_list()
args = ctx.actions.args()
args.use_param_file(param_file_arg = "@%s")
args.set_param_file_format("multiline")
args.add("--" + generator + "_out=" + _get_real_root(ctx, srcs[0]))
args.add("--plugin=protoc-gen-" + generator + "=" + tool.path)
args.add("--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets]))
args.add_all(proto_sources, map_each = _get_real_short_path)
ctx.actions.run(
inputs = depset(
direct = [proto_info.direct_descriptor_set],
transitive = [proto_info.transitive_descriptor_sets],
),
tools = [tool],
outputs = srcs + hdrs,
executable = ctx.executable._protoc,
arguments = [args],
progress_message = "Generating descriptor protos for :" + ctx.label.name,
mnemonic = "GenDescriptorProtos",
)
return GeneratedSrcsInfo(
srcs = srcs,
hdrs = hdrs,
includes = [_generate_include_path(proto_sources[0], hdrs[0], "_descriptor.pb.h")],
)
def _cc_proto_descriptor_rule_impl(ctx):
if len(ctx.attr.deps) != 1:
fail("only one deps dependency allowed.")
dep = ctx.attr.deps[0]
if _CcProtoDescriptorWrappedGeneratedSrcsInfo in dep:
srcs = dep[_CcProtoDescriptorWrappedGeneratedSrcsInfo].srcs
else:
fail("proto_library rule must generate _CcProtoDescriptorWrappedGeneratedSrcsInfo " +
"(aspect should have handled this).")
if CcProtoDescriptorWrappedCcInfo in dep:
cc_info = dep[CcProtoDescriptorWrappedCcInfo].cc_info
else:
fail("proto_library rule must generate CcProtoDescriptorWrappedCcInfo " +
"(aspect should have handled this).")
lib = cc_info.linking_context.linker_inputs.to_list()[0].libraries[0]
files = _filter_none([
lib.static_library,
lib.pic_static_library,
lib.dynamic_library,
])
return [
DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs)),
srcs,
cc_info,
]
def _cc_proto_descriptor_aspect_impl(target, ctx, generator, cc_provider, file_provider, provide_cc_shared_library_hints = True):
proto_info = target[ProtoInfo]
files = _compile_protos(ctx, generator, proto_info, proto_info.direct_sources)
deps = ctx.rule.attr.deps + getattr(ctx.attr, "_" + generator)
dep_ccinfos = [dep[CcInfo] for dep in deps if CcInfo in dep]
dep_ccinfos += [dep[CcProtoDescriptorWrappedCcInfo].cc_info for dep in deps if CcProtoDescriptorWrappedCcInfo in dep]
name = ctx.rule.attr.name + "." + generator
owners = [ctx.label.relative(name)]
cc_info = _cc_library_func(
ctx = ctx,
name = name,
hdrs = files.hdrs,
srcs = files.srcs,
includes = files.includes,
#copts = ctx.attr._copts[CcProtoDescriptorLibraryCoptsInfo].copts,
copts = [],
dep_ccinfos = dep_ccinfos,
)
wrapped_cc_info = cc_provider(
cc_info = cc_info,
)
providers = [
wrapped_cc_info,
file_provider(srcs = files),
]
if provide_cc_shared_library_hints:
if hasattr(cc_common, "CcSharedLibraryHintInfo"):
providers.append(cc_common.CcSharedLibraryHintInfo(owners = owners))
elif hasattr(cc_common, "CcSharedLibraryHintInfo_6_X_constructor_do_not_use"):
# This branch can be deleted once 6.X is not supported by rules
providers.append(cc_common.CcSharedLibraryHintInfo_6_X_constructor_do_not_use(owners = owners))
return providers
def cc_proto_descriptor_library_aspect_impl(target, ctx):
return _cc_proto_descriptor_aspect_impl(target, ctx, "descriptor", CcProtoDescriptorWrappedCcInfo, _CcProtoDescriptorWrappedGeneratedSrcsInfo)
def _maybe_add(d):
if _is_google3:
d["_grep_includes"] = attr.label(
allow_single_file = True,
cfg = "exec",
default = "@bazel_tools//tools/cpp:grep-includes",
)
return d
# cc_proto_descriptor_library() ##########################################################
def _get_cc_proto_descriptor_library_aspect_provides():
provides = [
CcProtoDescriptorWrappedCcInfo,
_CcProtoDescriptorWrappedGeneratedSrcsInfo,
]
if hasattr(cc_common, "CcSharedLibraryHintInfo"):
provides.append(cc_common.CcSharedLibraryHintInfo)
elif hasattr(cc_common, "CcSharedLibraryHintInfo_6_X_getter_do_not_use"):
# This branch can be deleted once 6.X is not supported by upb rules
provides.append(cc_common.CcSharedLibraryHintInfo_6_X_getter_do_not_use)
return provides
cc_proto_descriptor_library_aspect = aspect(
attrs = _maybe_add({
#"_copts": attr.label(
# default = "//:upb_proto_library_copts__for_generated_code_only_do_not_use",
#),
"_gen_descriptor": attr.label(
executable = True,
cfg = "exec",
default = "//bazel/cc_proto_descriptor_library:file_descriptor_generator",
),
"_protoc": attr.label(
executable = True,
cfg = "exec",
default = "@com_google_protobuf//:protoc",
),
"_cc_toolchain": attr.label(
default = "@bazel_tools//tools/cpp:current_cc_toolchain",
),
"_descriptor": attr.label_list(
default = [
Label("//bazel/cc_proto_descriptor_library:file_descriptor_info"),
Label("@com_google_absl//absl/base:core_headers"),
],
),
}),
implementation = cc_proto_descriptor_library_aspect_impl,
provides = _get_cc_proto_descriptor_library_aspect_provides(),
attr_aspects = ["deps"],
fragments = ["cpp"],
toolchains = use_cpp_toolchain(),
incompatible_use_toolchain_transition = True,
)
cc_proto_descriptor_library = rule(
output_to_genfiles = True,
implementation = _cc_proto_descriptor_rule_impl,
attrs = {
"deps": attr.label_list(
aspects = [cc_proto_descriptor_library_aspect],
allow_rules = ["proto_library"],
providers = [ProtoInfo],
),
},
provides = [CcInfo],
)

@ -0,0 +1,31 @@
#include "bazel/cc_proto_descriptor_library/create_dynamic_message.h"
#include <memory>
#include "absl/memory/memory.h"
#include "absl/strings/escaping.h"
#include "absl/strings/str_format.h"
#include "bazel/cc_proto_descriptor_library/text_format_transcoder.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/descriptor.pb.h"
#include "google/protobuf/dynamic_message.h"
#include "google/protobuf/message.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
std::unique_ptr<google::protobuf::Message>
createDynamicMessage(const TextFormatTranscoder& transcoder,
const google::protobuf::MessageLite& message,
google::protobuf::io::ErrorCollector* error_collector /*= nullptr*/) {
auto dynamic_message =
transcoder.createEmptyDynamicMessage(message.GetTypeName(), error_collector);
if (dynamic_message) {
dynamic_message->ParsePartialFromString(message.SerializePartialAsString());
}
return dynamic_message;
}
} // namespace cc_proto_descriptor_library

@ -0,0 +1,26 @@
#pragma once
#include <memory>
#include "google/protobuf/io/tokenizer.h"
#include "google/protobuf/io/zero_copy_stream.h"
#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
#include "google/protobuf/message.h"
#include "google/protobuf/message_lite.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
// Forward declare to make the friendship handshake easier.
class TextFormatTranscoder;
// Creates a DynamicMessage based off the passed in MessageLite. DynamicMessage
// gives you access to the Descriptor for the message. The message must have
// been registered with the passed in TextFormatTranscoder. Returned messages
// cannot outlive transcoder.
std::unique_ptr<google::protobuf::Message>
createDynamicMessage(const TextFormatTranscoder& transcoder,
const google::protobuf::MessageLite& message,
google::protobuf::io::ErrorCollector* error_collector = nullptr);
} // namespace cc_proto_descriptor_library

@ -0,0 +1,176 @@
#include "bazel/cc_proto_descriptor_library/file_descriptor_generator.h"
#include <memory>
#include <sstream>
#include <string>
#include "absl/strings/escaping.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/str_split.h"
#include "absl/strings/string_view.h"
#include "absl/strings/strip.h"
#include "google/protobuf/compiler/retention.h"
#include "google/protobuf/descriptor.pb.h"
#include "google/protobuf/io/coded_stream.h"
#include "google/protobuf/io/zero_copy_stream.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
namespace {
absl::string_view getFileBaseName(const google::protobuf::FileDescriptor* file) {
absl::string_view stripped_name = file->name();
if (absl::ConsumeSuffix(&stripped_name, ".proto")) {
return stripped_name;
} else if (absl::ConsumeSuffix(&stripped_name, ".protodevel")) {
return stripped_name;
}
return stripped_name;
}
std::string getDescriptorHeaderName(const google::protobuf::FileDescriptor* file) {
return absl::StrCat(getFileBaseName(file), "_descriptor.pb.h");
}
std::string getDescriptorSourceName(const google::protobuf::FileDescriptor* file) {
return absl::StrCat(getFileBaseName(file), "_descriptor.pb.cc");
}
std::string getDescriptorNamespace(const google::protobuf::FileDescriptor* file) {
return absl::AsciiStrToLower(
absl::StrReplaceAll(getFileBaseName(file), {{"/", "_"}, {"-", "_"}}));
}
std::string getDependencyFileDescriptorInfoSymbol(const google::protobuf::FileDescriptor* file) {
return absl::StrFormat("%s::kFileDescriptorInfo", getDescriptorNamespace(file));
}
std::string getDependencyFileDescriptorHeaderGuard(const google::protobuf::FileDescriptor* file) {
std::string header_path = getDescriptorHeaderName(file);
return absl::AsciiStrToUpper(
absl::StrReplaceAll(header_path, {{"/", "_"}, {".", "_"}, {"-", "_"}}));
}
bool generateHeader(const google::protobuf::FileDescriptor* file,
google::protobuf::io::ZeroCopyOutputStream* output_stream) {
auto header_guard = getDependencyFileDescriptorHeaderGuard(file);
auto unique_namespace = getDescriptorNamespace(file);
std::stringstream contents;
contents << absl::StrFormat(R"text(
#ifndef %s
#define %s
#include "absl/base/attributes.h"
namespace cc_proto_descriptor_library {
namespace internal {
struct FileDescriptorInfo;
} // namespace internal
} // namespace cc_proto_descriptor_library
namespace protobuf {
namespace reflection {
namespace %s {
extern const ::cc_proto_descriptor_library::internal::FileDescriptorInfo kFileDescriptorInfo;
} // namespace %s
} // namespace reflection
} // namespace protobuf
#endif // %s
)text",
header_guard, header_guard, unique_namespace, unique_namespace,
header_guard);
google::protobuf::io::CodedOutputStream output(output_stream);
output.WriteString(contents.str());
output.Trim();
return !output.HadError();
}
bool generateSource(const google::protobuf::FileDescriptor* file,
google::protobuf::io::ZeroCopyOutputStream* output_stream) {
auto unique_namespace = getDescriptorNamespace(file);
std::stringstream contents;
contents << absl::StrFormat("#include \"%s\"\n", getDescriptorHeaderName(file));
for (int i = 0; i < file->dependency_count(); ++i) {
contents << absl::StrFormat("#include \"%s\"\n", getDescriptorHeaderName(file->dependency(i)));
}
contents << R"text(#include "bazel/cc_proto_descriptor_library/file_descriptor_info.h"
)text";
google::protobuf::FileDescriptorProto file_descriptor_proto =
google::protobuf::compiler::StripSourceRetentionOptions(*file);
contents << absl::StrFormat(
R"text(
namespace protobuf {
namespace reflection {
namespace %s {
)text",
unique_namespace);
contents << "static const"
"::cc_proto_descriptor_library::internal::"
"FileDescriptorInfo* kDeps[] = {\n";
for (int i = 0; i < file->dependency_count(); ++i) {
contents << absl::StrFormat("&%s,\n",
getDependencyFileDescriptorInfoSymbol(file->dependency(i)));
}
contents << "nullptr};\n";
contents << absl::StrFormat(
R"text(
const ::cc_proto_descriptor_library::internal::FileDescriptorInfo kFileDescriptorInfo{
"%s",
"%s",
kDeps
};
} // namespace %s
} // namespace reflection
} // namespace protobuf
)text",
file->name(), absl::Base64Escape(file_descriptor_proto.SerializeAsString()),
unique_namespace);
google::protobuf::io::CodedOutputStream output(output_stream);
output.WriteString(contents.str());
output.Trim();
return !output.HadError();
}
} // namespace
bool ProtoDescriptorGenerator::Generate( // NOLINT(readability-identifier-naming)
const google::protobuf::FileDescriptor* file, const std::string& parameter,
google::protobuf::compiler::GeneratorContext* generator_context, std::string* error) const {
std::string header_path = getDescriptorHeaderName(file);
std::string source_path = getDescriptorSourceName(file);
std::unique_ptr<google::protobuf::io::ZeroCopyOutputStream> header_output(
generator_context->Open(header_path));
std::unique_ptr<google::protobuf::io::ZeroCopyOutputStream> source_output(
generator_context->Open(source_path));
if (!generateHeader(file, header_output.get())) {
return false;
}
if (!generateSource(file, source_output.get())) {
return false;
}
return true;
}
} // namespace cc_proto_descriptor_library

@ -0,0 +1,17 @@
#pragma once
#include <string>
#include "google/protobuf/compiler/code_generator.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
class ProtoDescriptorGenerator : public google::protobuf::compiler::CodeGenerator {
public:
bool Generate(const google::protobuf::FileDescriptor* file, const std::string& parameter,
google::protobuf::compiler::GeneratorContext* generator_context,
std::string* error) const override;
};
} // namespace cc_proto_descriptor_library

@ -0,0 +1,10 @@
#include <cstdio>
#include "bazel/cc_proto_descriptor_library/file_descriptor_generator.h"
#include "google/protobuf/compiler/plugin.h"
// NOLINT(namespace-envoy)
int main(int argc, char* argv[]) {
cc_proto_descriptor_library::ProtoDescriptorGenerator generator;
return google::protobuf::compiler::PluginMain(argc, argv, &generator);
}

@ -0,0 +1,29 @@
#include "bazel/cc_proto_descriptor_library/file_descriptor_info.h"
#include "absl/strings/escaping.h"
#include "google/protobuf/descriptor.h"
#include "net/proto2/proto/descriptor.proto.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
namespace internal {
void loadFileDescriptors(const FileDescriptorInfo& descriptor_info,
google::protobuf::DescriptorPool* descriptor_pool) {
if (descriptor_pool->FindFileByName(descriptor_info.file_name)) {
return;
}
for (int i = 0; descriptor_info.deps[i] != nullptr; ++i) {
loadFileDescriptors(*descriptor_info.deps[i], descriptor_pool);
}
google::protobuf::FileDescriptorProto file_descriptor_proto;
std::string file_descriptor_bytes;
absl::Base64Unescape(descriptor_info.file_descriptor_bytes_base64, &file_descriptor_bytes);
file_descriptor_proto.ParseFromString(file_descriptor_bytes);
descriptor_pool->BuildFile(file_descriptor_proto);
}
} // namespace internal
} // namespace cc_proto_descriptor_library

@ -0,0 +1,21 @@
#pragma once
#include "absl/strings/string_view.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
namespace internal {
struct FileDescriptorInfo final {
// Path of the file this object represents.
const absl::string_view file_name;
// The bytes of the FileDescriptorProto for this proto file.
const absl::string_view file_descriptor_bytes_base64;
// File descriptors that this file is dependent on.
// TODO(b/143243097): absl::Span requires special casing to deal with zero
// length arrays and constexpr.
const FileDescriptorInfo** const deps;
};
} // namespace internal
} // namespace cc_proto_descriptor_library

@ -0,0 +1,135 @@
load("//bazel/cc_proto_descriptor_library:builddefs.bzl", "cc_proto_descriptor_library")
licenses(["notice"]) # Apache 2
# Tests for go/cc_proto_descriptor_library.
# These need to be in a separate directory to test visibility restrictions.
proto_library(
name = "test_proto",
testonly = True,
srcs = ["test.proto"],
visibility = ["//visibility:private"],
deps = [
"@com_google_protobuf//:any_proto",
],
)
proto_library(
name = "test1_proto",
testonly = True,
srcs = ["test1.proto"],
visibility = ["//visibility:private"],
deps = [
":test_proto",
"@com_google_protobuf//:any_proto",
],
)
proto_library(
name = "test_extension_proto",
testonly = True,
# Purprosely a dash to make sure they're handled properly in filenames
srcs = ["test-extension.proto"],
visibility = ["//visibility:private"],
deps = [
":test_proto",
"@com_google_protobuf//:any_proto",
],
)
cc_proto_library(
name = "test_cc_proto",
testonly = True,
visibility = ["//visibility:private"],
deps = ["test_proto"],
)
cc_proto_library(
name = "test1_cc_proto",
testonly = True,
visibility = ["//visibility:private"],
deps = ["test1_proto"],
)
cc_proto_library(
name = "test_extension_cc_proto",
testonly = True,
visibility = ["//visibility:private"],
deps = ["test_extension_proto"],
)
cc_proto_descriptor_library(
name = "test_descriptors",
testonly = True,
visibility = ["//visibility:private"],
deps = [
":test_proto",
],
)
cc_proto_descriptor_library(
name = "test1_descriptors",
testonly = True,
visibility = ["//visibility:private"],
deps = [
":test1_proto",
],
)
cc_proto_descriptor_library(
name = "test_extension_descriptors",
testonly = True,
visibility = ["//visibility:private"],
deps = [
":test_extension_proto",
],
)
cc_test(
name = "text_format_transcoder_test",
srcs = [
"text_format_transcoder_test.cc",
],
visibility = ["//visibility:private"],
deps = [
"test1_descriptors",
"test_cc_proto",
"test_descriptors",
"test_extension_cc_proto",
"test_extension_descriptors",
"//bazel/cc_proto_descriptor_library:text_format_transcoder",
"@com_google_googletest//:gtest_main",
"@com_google_protobuf//:protobuf",
],
)
cc_test(
name = "global_fallback_test",
srcs = [
"global_fallback_test.cc",
],
visibility = ["//visibility:private"],
deps = [
"test_cc_proto",
"//bazel/cc_proto_descriptor_library:text_format_transcoder",
"@com_google_googletest//:gtest_main",
"@com_google_protobuf//:protobuf",
],
)
cc_test(
name = "create_dynamic_message_test",
srcs = [
"create_dynamic_message_test.cc",
],
visibility = ["//visibility:private"],
deps = [
"test_cc_proto",
"test_descriptors",
"//bazel/cc_proto_descriptor_library:create_dynamic_message",
"//bazel/cc_proto_descriptor_library:text_format_transcoder",
"@com_google_googletest//:gtest_main",
"@com_google_protobuf//:protobuf",
],
)

@ -0,0 +1,38 @@
#include "bazel/cc_proto_descriptor_library/create_dynamic_message.h"
#include "bazel/cc_proto_descriptor_library/testdata/test.pb.h"
#include "bazel/cc_proto_descriptor_library/testdata/test_descriptor.pb.h"
#include "bazel/cc_proto_descriptor_library/text_format_transcoder.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
// NOLINT(namespace-envoy)
namespace {
using ::testing::Eq;
using ::testing::NotNull;
using ::testing::Test;
TEST(TextFormatTranscoderTest, CreateDynamicMessage) {
cc_proto_descriptor_library::TextFormatTranscoder reserializer;
reserializer.loadFileDescriptors(
protobuf::reflection::bazel_cc_proto_descriptor_library_testdata_test::kFileDescriptorInfo);
testdata::dynamic_descriptors::Foo concrete_message;
concrete_message.set_bar("hello world");
auto dynamic_message =
cc_proto_descriptor_library::createDynamicMessage(reserializer, concrete_message);
ASSERT_THAT(dynamic_message, NotNull());
// Access the descriptor.
auto descriptor = dynamic_message->GetDescriptor();
ASSERT_THAT(descriptor, NotNull());
ASSERT_THAT(descriptor->full_name(), Eq(concrete_message.GetTypeName()));
// Access reflection.
auto reflection = dynamic_message->GetReflection();
const auto bar_feld_descriptor = descriptor->FindFieldByName("bar");
ASSERT_THAT(bar_feld_descriptor, NotNull());
ASSERT_THAT(reflection->GetString(*dynamic_message, bar_feld_descriptor), Eq("hello world"));
}
} // namespace

@ -0,0 +1,41 @@
// Copyright 2019 Google Inc. All Rights Reserved.
// Author: kmensah@google.com (Kwasi Mensah)
#include "bazel/cc_proto_descriptor_library/testdata/test.pb.h"
#include "bazel/cc_proto_descriptor_library/text_format_transcoder.h"
#include "gmock/gmock.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/text_format.h"
#include "gtest/gtest.h"
// NOLINT(namespace-envoy)
namespace {
using ::testing::Eq;
using ::testing::Test;
TEST(TextFormatTranscoderTest, GlobalFallbackAllowed) {
cc_proto_descriptor_library::TextFormatTranscoder reserializer(
/*allow_global_fallback=*/true);
testdata::dynamic_descriptors::Foo concrete_message;
ASSERT_TRUE(reserializer.parseInto(R"text(
bar: "hello world"
)text",
&concrete_message));
ASSERT_THAT(concrete_message.bar(), Eq("hello world"));
}
TEST(TextFormatTranscoderTest, GlobalFallbackDisallowed) {
cc_proto_descriptor_library::TextFormatTranscoder reserializer(
/*allow_global_fallback=*/false);
testdata::dynamic_descriptors::Foo concrete_message;
ASSERT_FALSE(reserializer.parseInto(R"text(
bar: "hello world"
)text",
&concrete_message));
}
} // namespace

@ -0,0 +1,9 @@
syntax = "proto2";
package testdata.dynamic_descriptors;
import "bazel/cc_proto_descriptor_library/testdata/test.proto";
extend Foo {
optional int64 number = 11;
}

@ -0,0 +1,12 @@
syntax = "proto2";
import "google/protobuf/any.proto";
package testdata.dynamic_descriptors;
message Foo {
optional string bar = 1;
optional google.protobuf.Any baz = 2;
extensions 10 to 20;
}

@ -0,0 +1,9 @@
syntax = "proto2";
package testdata.dynamic_descriptors;
message FooCopy {
optional string bar = 1;
extensions 10 to 20;
}

@ -0,0 +1,100 @@
// Copyright 2019 Google Inc. All Rights Reserved.
// Author: kmensah@google.com (Kwasi Mensah)
#include <string>
#include "absl/strings/substitute.h"
#include "bazel/cc_proto_descriptor_library/testdata/test-extension.pb.h"
#include "bazel/cc_proto_descriptor_library/testdata/test-extension_descriptor.pb.h"
#include "bazel/cc_proto_descriptor_library/testdata/test.pb.h"
#include "bazel/cc_proto_descriptor_library/testdata/test1_descriptor.pb.h"
#include "bazel/cc_proto_descriptor_library/testdata/test_descriptor.pb.h"
#include "bazel/cc_proto_descriptor_library/text_format_transcoder.h"
#include "gmock/gmock.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/dynamic_message.h"
#include "google/protobuf/text_format.h"
#include "gtest/gtest.h"
// NOLINT(namespace-envoy)
namespace {
using google::protobuf::io::ColumnNumber;
class StringErrorCollector : public google::protobuf::io::ErrorCollector {
public:
// String error_text is unowned and must remain valid during the use of
// StringErrorCollector.
// If one_indexing is set to true, all line and column numbers will be
// increased by one for cases when provided indices are 0-indexed and
// 1-indexed error messages are desired.
// If ignore_warnings is set to true, warnings are discarded.
explicit StringErrorCollector(std::string& error_text, bool one_indexing = false,
bool ignore_warnings = false);
// google::protobuf::io::ErrorCollector:
void AddError(int line, ColumnNumber column, const std::string& message) override;
void AddWarning(int line, ColumnNumber column, const std::string& message) override;
private:
std::string& error_text_;
const int index_offset_;
const bool ignore_warnings_;
};
StringErrorCollector::StringErrorCollector(std::string& error_text, bool one_indexing,
bool ignore_warnings)
: error_text_(error_text), index_offset_(one_indexing ? 1 : 0),
ignore_warnings_(ignore_warnings) {}
void StringErrorCollector::AddError(int line, ColumnNumber column, const std::string& message) {
absl::SubstituteAndAppend(&error_text_, "$0($1): $2\n", line + index_offset_,
column + index_offset_, message);
}
void StringErrorCollector::AddWarning(int line, ColumnNumber column, const std::string& message) {
if (ignore_warnings_) {
return;
}
AddError(line, column, message);
}
using ::testing::Eq;
using ::testing::Test;
TEST(TextFormatTranscoderTest, TextFormatWorks) {
cc_proto_descriptor_library::TextFormatTranscoder reserializer;
reserializer.loadFileDescriptors(
protobuf::reflection::bazel_cc_proto_descriptor_library_testdata_test::kFileDescriptorInfo);
testdata::dynamic_descriptors::Foo concrete_message;
ASSERT_TRUE(reserializer.parseInto(R"text(
bar: "hello world"
)text",
&concrete_message));
ASSERT_THAT(concrete_message.bar(), Eq("hello world"));
}
TEST(TextToBinaryReserializerTest, TextFormatWithExtensionWorks) {
cc_proto_descriptor_library::TextFormatTranscoder reserializer;
const auto& file_descriptor_info = protobuf::reflection::
bazel_cc_proto_descriptor_library_testdata_test_extension::kFileDescriptorInfo;
reserializer.loadFileDescriptors(file_descriptor_info);
std::string error_text;
StringErrorCollector error_collector(error_text);
testdata::dynamic_descriptors::Foo concrete_message;
ASSERT_TRUE(reserializer.parseInto(R"text(
bar: "hello world"
[testdata.dynamic_descriptors.number]: 20
)text",
&concrete_message, &error_collector));
ASSERT_THAT(error_text, Eq(""));
ASSERT_THAT(concrete_message.bar(), Eq("hello world"));
ASSERT_THAT(concrete_message.GetExtension(testdata::dynamic_descriptors::number), Eq(20));
}
} // namespace

@ -0,0 +1,128 @@
#include "bazel/cc_proto_descriptor_library/text_format_transcoder.h"
#include <memory>
#include <string>
#include "absl/memory/memory.h"
#include "absl/strings/escaping.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "bazel/cc_proto_descriptor_library/file_descriptor_info.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/descriptor.pb.h"
#include "google/protobuf/dynamic_message.h"
#include "google/protobuf/message.h"
#include "google/protobuf/text_format.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
struct TextFormatTranscoder::InternalData {
google::protobuf::DescriptorPool descriptor_pool;
google::protobuf::DynamicMessageFactory message_factory;
};
TextFormatTranscoder::TextFormatTranscoder(bool allow_global_fallback /*= true*/)
: internals_(std::make_unique<InternalData>()), allow_global_fallback_(allow_global_fallback) {}
// Needs to be defined here so std::unique_ptr doesn't complain about
// TextToBinaryProtoReserializerInternals being incomplete in it's
// destructor.
TextFormatTranscoder::~TextFormatTranscoder() = default;
bool TextFormatTranscoder::parseInto(absl::string_view text_format,
google::protobuf::MessageLite* msg,
google::protobuf::io::ErrorCollector* error_collector) const {
google::protobuf::io::ArrayInputStream input(text_format.data(), text_format.size());
return parseInto(&input, msg, error_collector);
}
bool TextFormatTranscoder::parseInto(google::protobuf::io::ZeroCopyInputStream* input_stream,
google::protobuf::MessageLite* msg,
google::protobuf::io::ErrorCollector* error_collector) const {
std::string serialization;
if (!toBinarySerializationInternal(msg->GetTypeName(), input_stream, &serialization,
error_collector)) {
return false;
}
return msg->ParseFromString(serialization);
}
void TextFormatTranscoder::loadFileDescriptors(
const internal::FileDescriptorInfo& file_descriptor_info) {
if (internals_->descriptor_pool.FindFileByName(std::string(file_descriptor_info.file_name))) {
return;
}
for (int i = 0; file_descriptor_info.deps[i] != nullptr; ++i) {
loadFileDescriptors(*file_descriptor_info.deps[i]);
}
google::protobuf::FileDescriptorProto file_descriptor_proto;
std::string file_descriptor_bytes;
absl::Base64Unescape(file_descriptor_info.file_descriptor_bytes_base64, &file_descriptor_bytes);
file_descriptor_proto.ParseFromString(file_descriptor_bytes);
internals_->descriptor_pool.BuildFile(file_descriptor_proto);
}
bool TextFormatTranscoder::toBinarySerializationInternal(
absl::string_view type_name, google::protobuf::io::ZeroCopyInputStream* input_stream,
std::string* binary_serializtion, google::protobuf::io::ErrorCollector* error_collector) const {
auto dynamic_message = createEmptyDynamicMessage(type_name, error_collector);
if (!dynamic_message) {
// createEmptyDynamicMessage already would have written to the
// error_collector.
return false;
}
// TextFormat defaults to using the DescriptorPool of the passed in
// Message*. So we don't have to do anything special with TextFormat::Finder
// for it to use our DescriptorPool.
google::protobuf::TextFormat::Parser parser;
parser.RecordErrorsTo(error_collector);
if (!parser.Parse(input_stream, dynamic_message.get())) {
return false;
}
return dynamic_message->SerializePartialToString(binary_serializtion);
}
std::unique_ptr<google::protobuf::Message> TextFormatTranscoder::createEmptyDynamicMessage(
absl::string_view type_name, google::protobuf::io::ErrorCollector* error_collector) const {
const google::protobuf::Descriptor* descriptor =
internals_->descriptor_pool.FindMessageTypeByName(std::string(type_name));
// If you're built with the full runtime then embedding the descriptors and
// loading them would be information duplicated by the global descriptor
// pool which hurts builds like superroot that are near all the blaze/forge
// size limits. Teams that care about not silently falling into this fallback
// case should make sure their tests are also run in mobile configurations or
// set allow_global_fallback to false in TextFormatTranscoder's constructor.
if (allow_global_fallback_ && (descriptor == nullptr)) {
const auto generated_pool = google::protobuf::DescriptorPool::generated_pool();
if (generated_pool != nullptr) {
descriptor = generated_pool->FindMessageTypeByName(std::string(type_name));
}
}
if (descriptor == nullptr) {
if (error_collector) {
error_collector->RecordError(0, 0,
absl::StrFormat("Could not find descriptor for: %s", type_name));
}
return nullptr;
}
auto dynamic_message =
absl::WrapUnique(internals_->message_factory.GetPrototype(descriptor)->New());
if (!dynamic_message) {
if (error_collector) {
error_collector->RecordError(
0, 0, absl::StrFormat("Could not create dynamic message for: %s", type_name));
}
return nullptr;
}
return dynamic_message;
}
} // namespace cc_proto_descriptor_library

@ -0,0 +1,65 @@
#pragma once
#include <memory>
#include <string>
#include "absl/strings/string_view.h"
#include "bazel/cc_proto_descriptor_library/create_dynamic_message.h"
#include "google/protobuf/io/tokenizer.h"
#include "google/protobuf/io/zero_copy_stream.h"
#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
#include "google/protobuf/message.h"
// NOLINT(namespace-envoy)
namespace cc_proto_descriptor_library {
namespace internal {
struct FileDescriptorInfo;
} // namespace internal
// Takes a text formatted proto and translates it to a MessageLite proto so it
// can be used by the lite runtime. Requires the that text formatted protos and
// all of its extensions are registered with this object.
class TextFormatTranscoder final {
public:
explicit TextFormatTranscoder(
// Whether to allow using the global descriptor pool as a fallback. See
// text_format_transcoder.cc for more information.
bool allow_global_fallback = true);
~TextFormatTranscoder();
bool parseInto(absl::string_view text_format, google::protobuf::MessageLite* msg,
google::protobuf::io::ErrorCollector* error_collector = nullptr) const;
bool parseInto(google::protobuf::io::ZeroCopyInputStream* input_stream,
google::protobuf::MessageLite* msg,
google::protobuf::io::ErrorCollector* error_collector = nullptr) const;
void loadFileDescriptors(const internal::FileDescriptorInfo& file_descriptor_info);
private:
// This function needs the member function CreateEmptyDynamicMessage. The
// target the defines the function has its own visibility whitelist which
// requires a dance with these declarations.
friend std::unique_ptr<google::protobuf::Message>
createDynamicMessage(const TextFormatTranscoder& transcoder,
const google::protobuf::MessageLite& message,
google::protobuf::io::ErrorCollector* error_collector);
std::unique_ptr<google::protobuf::Message>
createEmptyDynamicMessage(absl::string_view type_name,
google::protobuf::io::ErrorCollector* error_collector) const;
bool toBinarySerializationInternal(absl::string_view type_name,
google::protobuf::io::ZeroCopyInputStream* input_stream,
std::string* binary_serializtion,
google::protobuf::io::ErrorCollector* error_collector) const;
struct InternalData;
std::unique_ptr<InternalData> internals_;
const bool allow_global_fallback_;
};
} // namespace cc_proto_descriptor_library
Loading…
Cancel
Save