From 6c121f5b14112d0ad3dca7f6daa078fa0907418a Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Wed, 13 Sep 2023 08:36:31 -0700 Subject: [PATCH] Refactored `upb_proto*_library()` rules to make future splitting of rules easier. This is in preparation for splitting MiniTables out of `upb_c_proto_library()`. PiperOrigin-RevId: 565059826 --- upb/BUILD | 7 +- upb/bazel/BUILD | 23 +- upb/bazel/upb_c_proto_library.bzl | 86 +++ upb/bazel/upb_proto_library.bzl | 504 +----------------- .../upb_proto_library_internal/aspect.bzl | 261 +++++++++ .../cc_library_func.bzl | 71 +++ .../upb_proto_library_internal/copts.bzl | 16 + upb/bazel/upb_proto_library_internal/rule.bzl | 39 ++ upb/bazel/upb_proto_reflection_library.bzl | 81 +++ 9 files changed, 593 insertions(+), 495 deletions(-) create mode 100644 upb/bazel/upb_c_proto_library.bzl create mode 100644 upb/bazel/upb_proto_library_internal/aspect.bzl create mode 100644 upb/bazel/upb_proto_library_internal/cc_library_func.bzl create mode 100644 upb/bazel/upb_proto_library_internal/copts.bzl create mode 100644 upb/bazel/upb_proto_library_internal/rule.bzl create mode 100644 upb/bazel/upb_proto_reflection_library.bzl diff --git a/upb/BUILD b/upb/BUILD index 07d6de32a8..6324ad2059 100644 --- a/upb/BUILD +++ b/upb/BUILD @@ -8,11 +8,8 @@ load("@rules_python//python:defs.bzl", "py_binary") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("//upb/bazel:build_defs.bzl", "UPB_DEFAULT_COPTS") -load( - "//upb/bazel:upb_proto_library.bzl", - "upb_proto_library_copts", - "upb_proto_reflection_library", -) +load("//upb/bazel:upb_proto_library.bzl", "upb_proto_reflection_library") +load("//upb/bazel:upb_proto_library_internal/copts.bzl", "upb_proto_library_copts") load( "//upb/upbc:bootstrap_compiler.bzl", "bootstrap_cc_library", diff --git a/upb/bazel/BUILD b/upb/bazel/BUILD index 005bb635b1..ec8421585c 100644 --- a/upb/bazel/BUILD +++ b/upb/bazel/BUILD @@ -34,12 +34,29 @@ bzl_library( ) bzl_library( - name = "upb_proto_library_bzl", - srcs = ["upb_proto_library.bzl"], - visibility = ["//visibility:public"], + name = "upb_proto_library_internal_bzl", + srcs = [ + "upb_proto_library_internal/aspect.bzl", + "upb_proto_library_internal/cc_library_func.bzl", + "upb_proto_library_internal/copts.bzl", + "upb_proto_library_internal/rule.bzl", + ], deps = [ "@bazel_skylib//lib:paths", "@bazel_tools//tools/cpp:toolchain_utils.bzl", + ], +) + +bzl_library( + name = "upb_proto_library_bzl", + srcs = [ + "upb_c_proto_library.bzl", + "upb_proto_library.bzl", + "upb_proto_reflection_library.bzl", + ], + visibility = ["//visibility:public"], + deps = [ + ":upb_proto_library_internal_bzl", "@rules_proto//proto:defs", ], ) diff --git a/upb/bazel/upb_c_proto_library.bzl b/upb/bazel/upb_c_proto_library.bzl new file mode 100644 index 0000000000..f5cd6b70da --- /dev/null +++ b/upb/bazel/upb_c_proto_library.bzl @@ -0,0 +1,86 @@ +"""upb_c_proto_library() exposes upb's generated C API for protobuf (foo.upb.h)""" + +load("//upb/bazel:upb_proto_library_internal/aspect.bzl", "upb_proto_aspect_impl") +load("//upb/bazel:upb_proto_library_internal/cc_library_func.bzl", "upb_use_cpp_toolchain") +load("//upb/bazel:upb_proto_library_internal/rule.bzl", "upb_proto_rule_impl") + +UpbWrappedCcInfo = provider( + "Provider for cc_info for protos", + fields = ["cc_info", "cc_info_with_thunks"], +) + +_UpbWrappedGeneratedSrcsInfo = provider( + "Provider for generated sources", + fields = ["srcs"], +) + +def _upb_c_proto_library_aspect_impl(target, ctx): + return upb_proto_aspect_impl( + target = target, + ctx = ctx, + generator = "upb", + cc_provider = UpbWrappedCcInfo, + dep_cc_provider = None, + file_provider = _UpbWrappedGeneratedSrcsInfo, + ) + +def _get_upb_c_proto_library_aspect_provides(): + provides = [ + UpbWrappedCcInfo, + _UpbWrappedGeneratedSrcsInfo, + ] + + if hasattr(cc_common, "CcSharedLibraryHintInfo"): + provides.append(cc_common.CcSharedLibraryHintInfo) + elif hasattr(cc_common, "CcSharedLibraryHintInfo_6_X_getter_do_not_use"): + # This branch can be deleted once 6.X is not supported by upb rules + provides.append(cc_common.CcSharedLibraryHintInfo_6_X_getter_do_not_use) + + return provides + +upb_c_proto_library_aspect = aspect( + attrs = { + "_copts": attr.label( + default = "//upb:upb_proto_library_copts__for_generated_code_only_do_not_use", + ), + "_gen_upb": attr.label( + executable = True, + cfg = "exec", + default = "//upb/upbc:protoc-gen-upb_stage1", + ), + "_protoc": attr.label( + executable = True, + cfg = "exec", + default = "//:protoc", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + "_upb": attr.label_list(default = [ + "//upb:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + ]), + "_fasttable_enabled": attr.label(default = "//upb:fasttable_enabled"), + }, + implementation = _upb_c_proto_library_aspect_impl, + provides = _get_upb_c_proto_library_aspect_provides(), + attr_aspects = ["deps"], + fragments = ["cpp"], + toolchains = upb_use_cpp_toolchain(), + incompatible_use_toolchain_transition = True, +) + +def _upb_c_proto_library_rule_impl(ctx): + return upb_proto_rule_impl(ctx, UpbWrappedCcInfo, _UpbWrappedGeneratedSrcsInfo) + +upb_c_proto_library = rule( + output_to_genfiles = True, + implementation = _upb_c_proto_library_rule_impl, + attrs = { + "deps": attr.label_list( + aspects = [upb_c_proto_library_aspect], + allow_rules = ["proto_library"], + providers = [ProtoInfo], + ), + }, + provides = [CcInfo], +) diff --git a/upb/bazel/upb_proto_library.bzl b/upb/bazel/upb_proto_library.bzl index 036f8d704a..b6c0aff4f9 100644 --- a/upb/bazel/upb_proto_library.bzl +++ b/upb/bazel/upb_proto_library.bzl @@ -10,496 +10,26 @@ - upb_proto_reflection_library() """ -load("@bazel_skylib//lib:paths.bzl", "paths") - -# begin:google_only -# load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain", "use_cpp_toolchain") -# end:google_only - -# begin:github_only -# Compatibility code for Bazel 4.x. Remove this when we drop support for Bazel 4.x. -load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") - -def use_cpp_toolchain(): - return ["@bazel_tools//tools/cpp:toolchain_type"] -# end:github_only - -# Generic support code ######################################################### - -# begin:github_only -_is_google3 = False -# end:github_only - -# begin:google_only -# _is_google3 = True -# end:google_only - -def _get_real_short_path(file): - # For some reason, files from other archives have short paths that look like: - # ../com_google_protobuf/google/protobuf/descriptor.proto - short_path = file.short_path - if short_path.startswith("../"): - second_slash = short_path.index("/", 3) - short_path = short_path[second_slash + 1:] - - # Sometimes it has another few prefixes like: - # _virtual_imports/any_proto/google/protobuf/any.proto - # benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto - # We want just google/protobuf/any.proto. - virtual_imports = "_virtual_imports/" - if virtual_imports in short_path: - short_path = short_path.split(virtual_imports)[1].split("/", 1)[1] - return short_path - -def _get_real_root(ctx, file): - real_short_path = _get_real_short_path(file) - root = file.path[:-len(real_short_path) - 1] - - if not _is_google3 and ctx.rule.attr.strip_import_prefix: - root = paths.join(root, ctx.rule.attr.strip_import_prefix[1:]) - return root - -def _generate_output_file(ctx, src, extension): - package = ctx.label.package - if not _is_google3: - strip_import_prefix = ctx.rule.attr.strip_import_prefix - if strip_import_prefix and strip_import_prefix != "/": - if not package.startswith(strip_import_prefix[1:]): - fail("%s does not begin with prefix %s" % (package, strip_import_prefix)) - package = package[len(strip_import_prefix):] - - real_short_path = _get_real_short_path(src) - real_short_path = paths.relativize(real_short_path, package) - output_filename = paths.replace_extension(real_short_path, extension) - ret = ctx.actions.declare_file(output_filename) - return ret - -def _generate_include_path(src, out, extension): - short_path = _get_real_short_path(src) - short_path = paths.replace_extension(short_path, extension) - if not out.path.endswith(short_path): - fail("%s does not end with %s" % (out.path, short_path)) - - return out.path[:-len(short_path)] - -def _filter_none(elems): - out = [] - for elem in elems: - if elem: - out.append(elem) - return out - -def _cc_library_func(ctx, name, hdrs, srcs, copts, includes, dep_ccinfos): - """Like cc_library(), but callable from rules. - - Args: - ctx: Rule context. - name: Unique name used to generate output files. - hdrs: Public headers that can be #included from other rules. - srcs: C/C++ source files. - copts: Additional options for cc compilation. - includes: Additional include paths. - dep_ccinfos: CcInfo providers of dependencies we should build/link against. - - Returns: - CcInfo provider for this compilation. - """ - - compilation_contexts = [info.compilation_context for info in dep_ccinfos] - linking_contexts = [info.linking_context for info in dep_ccinfos] - toolchain = find_cpp_toolchain(ctx) - feature_configuration = cc_common.configure_features( - ctx = ctx, - cc_toolchain = toolchain, - requested_features = ctx.features, - unsupported_features = ctx.disabled_features, - ) - - (compilation_context, compilation_outputs) = cc_common.compile( - actions = ctx.actions, - feature_configuration = feature_configuration, - cc_toolchain = toolchain, - name = name, - srcs = srcs, - includes = includes, - public_hdrs = hdrs, - user_compile_flags = copts, - compilation_contexts = compilation_contexts, - ) - - # buildifier: disable=unused-variable - (linking_context, linking_outputs) = cc_common.create_linking_context_from_compilation_outputs( - actions = ctx.actions, - name = name, - feature_configuration = feature_configuration, - cc_toolchain = toolchain, - compilation_outputs = compilation_outputs, - linking_contexts = linking_contexts, - disallow_dynamic_library = cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "targets_windows") or not cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "supports_dynamic_linker"), - ) - - return CcInfo( - compilation_context = compilation_context, - linking_context = linking_context, - ) - -# Dummy rule to expose select() copts to aspects ############################## - -UpbProtoLibraryCoptsInfo = provider( - "Provides copts for upb proto targets", - fields = { - "copts": "copts for upb_proto_library()", - }, -) - -def upb_proto_library_copts_impl(ctx): - return UpbProtoLibraryCoptsInfo(copts = ctx.attr.copts) - -upb_proto_library_copts = rule( - implementation = upb_proto_library_copts_impl, - attrs = {"copts": attr.string_list(default = [])}, -) - -# upb_proto_library / upb_proto_reflection_library shared code ################# - -GeneratedSrcsInfo = provider( - "Provides generated headers and sources", - fields = { - "srcs": "list of srcs", - "hdrs": "list of hdrs", - "thunks": "Experimental, do not use. List of srcs defining C API. Incompatible with hdrs.", - "includes": "list of extra includes", - }, -) - -def _concat_lists(lists): - ret = [] - for lst in lists: - ret = ret + lst - return ret - -def _merge_generated_srcs(srcs): - return GeneratedSrcsInfo( - srcs = _concat_lists([s.srcs for s in srcs]), - hdrs = _concat_lists([s.hdrs for s in srcs]), - thunks = _concat_lists([s.thunks for s in srcs]), - includes = _concat_lists([s.includes for s in srcs]), - ) - -UpbWrappedCcInfo = provider("Provider for cc_info for protos", fields = ["cc_info", "cc_info_with_thunks"]) - -_UpbDefsWrappedCcInfo = provider("Provider for cc_info for protos", fields = ["cc_info"]) -_UpbWrappedGeneratedSrcsInfo = provider("Provider for generated sources", fields = ["srcs"]) -_WrappedDefsGeneratedSrcsInfo = provider( - "Provider for generated reflective sources", - fields = ["srcs"], +load( + "//upb/bazel:upb_c_proto_library.bzl", + _UpbWrappedCcInfo = "UpbWrappedCcInfo", + _upb_c_proto_library = "upb_c_proto_library", + _upb_c_proto_library_aspect = "upb_c_proto_library_aspect", ) - -def _generate_upb_protos(ctx, generator, proto_info, proto_sources): - if len(proto_sources) == 0: - return GeneratedSrcsInfo(srcs = [], hdrs = [], thunks = [], includes = []) - - ext = "." + generator - tool = getattr(ctx.executable, "_gen_" + generator) - srcs = [_generate_output_file(ctx, name, ext + ".c") for name in proto_sources] - hdrs = [_generate_output_file(ctx, name, ext + ".h") for name in proto_sources] - thunks = [] - if generator == "upb": - thunks = [_generate_output_file(ctx, name, ext + ".thunks.c") for name in proto_sources] - transitive_sets = proto_info.transitive_descriptor_sets.to_list() - - args = ctx.actions.args() - args.use_param_file(param_file_arg = "@%s") - args.set_param_file_format("multiline") - - args.add("--" + generator + "_out=" + _get_real_root(ctx, srcs[0])) - args.add("--plugin=protoc-gen-" + generator + "=" + tool.path) - args.add("--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets])) - args.add_all(proto_sources, map_each = _get_real_short_path) - - ctx.actions.run( - inputs = depset( - direct = [proto_info.direct_descriptor_set], - transitive = [proto_info.transitive_descriptor_sets], - ), - tools = [tool], - outputs = srcs + hdrs, - executable = ctx.executable._protoc, - arguments = [args], - progress_message = "Generating upb protos for :" + ctx.label.name, - mnemonic = "GenUpbProtos", - ) - if generator == "upb": - ctx.actions.run_shell( - inputs = hdrs, - outputs = thunks, - command = " && ".join([ - "sed 's/UPB_INLINE //' {} > {}".format(hdr.path, thunk.path) - for (hdr, thunk) in zip(hdrs, thunks) - ]), - progress_message = "Generating thunks for upb protos API for: " + ctx.label.name, - mnemonic = "GenUpbProtosThunks", - ) - return GeneratedSrcsInfo( - srcs = srcs, - hdrs = hdrs, - thunks = thunks, - includes = [_generate_include_path(proto_sources[0], hdrs[0], ext + ".h")], - ) - -def _upb_proto_rule_impl(ctx): - if len(ctx.attr.deps) != 1: - fail("only one deps dependency allowed.") - dep = ctx.attr.deps[0] - - if _WrappedDefsGeneratedSrcsInfo in dep: - srcs = dep[_WrappedDefsGeneratedSrcsInfo].srcs - elif _UpbWrappedGeneratedSrcsInfo in dep: - srcs = dep[_UpbWrappedGeneratedSrcsInfo].srcs - else: - fail("proto_library rule must generate _UpbWrappedGeneratedSrcsInfo or " + - "_WrappedDefsGeneratedSrcsInfo (aspect should have handled this).") - - if _UpbDefsWrappedCcInfo in dep: - cc_info = dep[_UpbDefsWrappedCcInfo].cc_info - elif UpbWrappedCcInfo in dep: - cc_info = dep[UpbWrappedCcInfo].cc_info - else: - fail("proto_library rule must generate UpbWrappedCcInfo or " + - "_UpbDefsWrappedCcInfo (aspect should have handled this).") - - lib = cc_info.linking_context.linker_inputs.to_list()[0].libraries[0] - files = _filter_none([ - lib.static_library, - lib.pic_static_library, - lib.dynamic_library, - ]) - return [ - DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs)), - srcs, - cc_info, - ] - -def _generate_name(ctx, generator, thunks = False): - if thunks: - return ctx.rule.attr.name + "." + generator + ".thunks" - return ctx.rule.attr.name + "." + generator - -def _get_dep_cc_infos(target, ctx, generator): - aspect_deps = getattr(ctx.attr, "_" + generator) - rule_deps = ctx.rule.attr.deps - dep_ccinfos = [dep[CcInfo] for dep in aspect_deps] - if generator == "upbdefs": - dep_ccinfos += [dep[_UpbDefsWrappedCcInfo].cc_info for dep in rule_deps] - dep_ccinfos.append(target[UpbWrappedCcInfo].cc_info) - else: - dep_ccinfos += [dep[UpbWrappedCcInfo].cc_info for dep in rule_deps] - return dep_ccinfos - -def _compile_upb_protos(ctx, files, generator, dep_ccinfos, cc_provider): - cc_info = _cc_library_func( - ctx = ctx, - name = _generate_name(ctx, generator), - hdrs = files.hdrs, - srcs = files.srcs, - includes = files.includes, - copts = ctx.attr._copts[UpbProtoLibraryCoptsInfo].copts, - dep_ccinfos = dep_ccinfos, - ) - - if files.thunks: - cc_info_with_thunks = _cc_library_func( - ctx = ctx, - name = _generate_name(ctx, generator, files.thunks), - hdrs = [], - srcs = files.thunks, - includes = files.includes, - copts = ctx.attr._copts[UpbProtoLibraryCoptsInfo].copts, - dep_ccinfos = dep_ccinfos + [cc_info], - ) - return cc_provider( - cc_info = cc_info, - cc_info_with_thunks = cc_info_with_thunks, - ) - else: - return cc_provider( - cc_info = cc_info, - ) - -def _get_hint_providers(ctx, generator): - if generator not in _GENERATORS: - fail("Please add new generator '{}' to _GENERATORS list".format(generator)) - - possible_owners = [] - for generator in _GENERATORS: - possible_owners.append(ctx.label.relative(_generate_name(ctx, generator))) - possible_owners.append(ctx.label.relative(_generate_name(ctx, generator, thunks = True))) - - if hasattr(cc_common, "CcSharedLibraryHintInfo"): - return [cc_common.CcSharedLibraryHintInfo(owners = possible_owners)] - elif hasattr(cc_common, "CcSharedLibraryHintInfo_6_X_constructor_do_not_use"): - # This branch can be deleted once 6.X is not supported by upb rules - return [cc_common.CcSharedLibraryHintInfo_6_X_constructor_do_not_use(owners = possible_owners)] - - return [] - -def _upb_proto_aspect_impl(target, ctx, generator, cc_provider, file_provider, provide_cc_shared_library_hints = True): - dep_ccinfos = _get_dep_cc_infos(target, ctx, generator) - if not getattr(ctx.rule.attr, "srcs", []): - # This target doesn't declare any sources, reexport all its deps instead. - # This is known as an "alias library": - # https://bazel.build/reference/be/protocol-buffer#proto_library.srcs - files = _merge_generated_srcs([dep[file_provider].srcs for dep in ctx.rule.attr.deps]) - wrapped_cc_info = UpbWrappedCcInfo( - cc_info = cc_common.merge_cc_infos(direct_cc_infos = dep_ccinfos), - ) - else: - proto_info = target[ProtoInfo] - files = _generate_upb_protos( - ctx, - generator, - proto_info, - proto_info.direct_sources, - ) - wrapped_cc_info = _compile_upb_protos( - ctx, - files, - generator, - dep_ccinfos, - cc_provider, - ) - - hints = _get_hint_providers(ctx, generator) if provide_cc_shared_library_hints else [] - - return hints + [ - file_provider(srcs = files), - wrapped_cc_info, - ] - -_GENERATORS = ["upb", "upbdefs"] - -def upb_proto_library_aspect_impl(target, ctx): - return _upb_proto_aspect_impl(target, ctx, "upb", UpbWrappedCcInfo, _UpbWrappedGeneratedSrcsInfo) - -def _upb_proto_reflection_library_aspect_impl(target, ctx): - return _upb_proto_aspect_impl(target, ctx, "upbdefs", _UpbDefsWrappedCcInfo, _WrappedDefsGeneratedSrcsInfo, provide_cc_shared_library_hints = False) - -# upb_proto_library() ########################################################## - -def _get_upb_proto_library_aspect_provides(): - provides = [ - UpbWrappedCcInfo, - _UpbWrappedGeneratedSrcsInfo, - ] - - if hasattr(cc_common, "CcSharedLibraryHintInfo"): - provides.append(cc_common.CcSharedLibraryHintInfo) - elif hasattr(cc_common, "CcSharedLibraryHintInfo_6_X_getter_do_not_use"): - # This branch can be deleted once 6.X is not supported by upb rules - provides.append(cc_common.CcSharedLibraryHintInfo_6_X_getter_do_not_use) - - return provides - -upb_proto_library_aspect = aspect( - attrs = { - "_copts": attr.label( - default = "//upb:upb_proto_library_copts__for_generated_code_only_do_not_use", - ), - "_gen_upb": attr.label( - executable = True, - cfg = "exec", - default = "//upb/upbc:protoc-gen-upb_stage1", - ), - "_protoc": attr.label( - executable = True, - cfg = "exec", - default = "//:protoc", - ), - "_cc_toolchain": attr.label( - default = "@bazel_tools//tools/cpp:current_cc_toolchain", - ), - "_upb": attr.label_list(default = [ - "//upb:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", - ]), - "_fasttable_enabled": attr.label(default = "//upb:fasttable_enabled"), - }, - implementation = upb_proto_library_aspect_impl, - provides = _get_upb_proto_library_aspect_provides(), - attr_aspects = ["deps"], - fragments = ["cpp"], - toolchains = use_cpp_toolchain(), - incompatible_use_toolchain_transition = True, +load( + "//upb/bazel:upb_proto_library_internal/aspect.bzl", + _GeneratedSrcsInfo = "GeneratedSrcsInfo", ) - -upb_proto_library = rule( - output_to_genfiles = True, - implementation = _upb_proto_rule_impl, - attrs = { - "deps": attr.label_list( - aspects = [upb_proto_library_aspect], - allow_rules = ["proto_library"], - providers = [ProtoInfo], - ), - }, - provides = [CcInfo], +load( + "//upb/bazel:upb_proto_reflection_library.bzl", + _upb_proto_reflection_library = "upb_proto_reflection_library", ) # Temporary alias, see b/291827469. -upb_c_proto_library = upb_proto_library +upb_proto_library = _upb_c_proto_library -# upb_proto_reflection_library() ############################################### - -_upb_proto_reflection_library_aspect = aspect( - attrs = { - "_copts": attr.label( - default = "//upb:upb_proto_library_copts__for_generated_code_only_do_not_use", - ), - "_gen_upbdefs": attr.label( - executable = True, - cfg = "exec", - default = "//upb/upbc:protoc-gen-upbdefs", - ), - "_protoc": attr.label( - executable = True, - cfg = "exec", - default = "//:protoc", - ), - "_cc_toolchain": attr.label( - default = "@bazel_tools//tools/cpp:current_cc_toolchain", - ), - "_upbdefs": attr.label_list( - default = [ - "//upb:generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", - ], - ), - }, - implementation = _upb_proto_reflection_library_aspect_impl, - provides = [ - _UpbDefsWrappedCcInfo, - _WrappedDefsGeneratedSrcsInfo, - ], - required_aspect_providers = [ - UpbWrappedCcInfo, - _UpbWrappedGeneratedSrcsInfo, - ], - attr_aspects = ["deps"], - fragments = ["cpp"], - toolchains = use_cpp_toolchain(), - incompatible_use_toolchain_transition = True, -) - -upb_proto_reflection_library = rule( - output_to_genfiles = True, - implementation = _upb_proto_rule_impl, - attrs = { - "deps": attr.label_list( - aspects = [ - upb_proto_library_aspect, - _upb_proto_reflection_library_aspect, - ], - allow_rules = ["proto_library"], - providers = [ProtoInfo], - ), - }, - provides = [CcInfo], -) +upb_c_proto_library = _upb_c_proto_library +upb_proto_reflection_library = _upb_proto_reflection_library +GeneratedSrcsInfo = _GeneratedSrcsInfo +UpbWrappedCcInfo = _UpbWrappedCcInfo +upb_proto_library_aspect = _upb_c_proto_library_aspect diff --git a/upb/bazel/upb_proto_library_internal/aspect.bzl b/upb/bazel/upb_proto_library_internal/aspect.bzl new file mode 100644 index 0000000000..006a32eaed --- /dev/null +++ b/upb/bazel/upb_proto_library_internal/aspect.bzl @@ -0,0 +1,261 @@ +"""Implementation of the aspect that powers the upb_*_proto_library() rules.""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("//upb/bazel:upb_proto_library_internal/cc_library_func.bzl", "cc_library_func") +load("//upb/bazel:upb_proto_library_internal/copts.bzl", "UpbProtoLibraryCoptsInfo") + +# begin:github_only +_is_google3 = False +# end:github_only + +# begin:google_only +# _is_google3 = True +# end:google_only + +def _get_real_short_path(file): + # For some reason, files from other archives have short paths that look like: + # ../com_google_protobuf/google/protobuf/descriptor.proto + short_path = file.short_path + if short_path.startswith("../"): + second_slash = short_path.index("/", 3) + short_path = short_path[second_slash + 1:] + + # Sometimes it has another few prefixes like: + # _virtual_imports/any_proto/google/protobuf/any.proto + # benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto + # We want just google/protobuf/any.proto. + virtual_imports = "_virtual_imports/" + if virtual_imports in short_path: + short_path = short_path.split(virtual_imports)[1].split("/", 1)[1] + return short_path + +def _get_real_root(ctx, file): + real_short_path = _get_real_short_path(file) + root = file.path[:-len(real_short_path) - 1] + + if not _is_google3 and ctx.rule.attr.strip_import_prefix: + root = paths.join(root, ctx.rule.attr.strip_import_prefix[1:]) + return root + +def _generate_output_file(ctx, src, extension): + package = ctx.label.package + if not _is_google3: + strip_import_prefix = ctx.rule.attr.strip_import_prefix + if strip_import_prefix and strip_import_prefix != "/": + if not package.startswith(strip_import_prefix[1:]): + fail("%s does not begin with prefix %s" % (package, strip_import_prefix)) + package = package[len(strip_import_prefix):] + + real_short_path = _get_real_short_path(src) + real_short_path = paths.relativize(real_short_path, package) + output_filename = paths.replace_extension(real_short_path, extension) + ret = ctx.actions.declare_file(output_filename) + return ret + +def _generate_include_path(src, out, extension): + short_path = _get_real_short_path(src) + short_path = paths.replace_extension(short_path, extension) + if not out.path.endswith(short_path): + fail("%s does not end with %s" % (out.path, short_path)) + + return out.path[:-len(short_path)] + +GeneratedSrcsInfo = provider( + "Provides generated headers and sources", + fields = { + "srcs": "list of srcs", + "hdrs": "list of hdrs", + "thunks": "Experimental, do not use. List of srcs defining C API. Incompatible with hdrs.", + "includes": "list of extra includes", + }, +) + +def _concat_lists(lists): + ret = [] + for lst in lists: + ret = ret + lst + return ret + +def _merge_generated_srcs(srcs): + return GeneratedSrcsInfo( + srcs = _concat_lists([s.srcs for s in srcs]), + hdrs = _concat_lists([s.hdrs for s in srcs]), + thunks = _concat_lists([s.thunks for s in srcs]), + includes = _concat_lists([s.includes for s in srcs]), + ) + +def _generate_upb_protos(ctx, generator, proto_info, proto_sources): + if len(proto_sources) == 0: + return GeneratedSrcsInfo(srcs = [], hdrs = [], thunks = [], includes = []) + + ext = "." + generator + tool = getattr(ctx.executable, "_gen_" + generator) + srcs = [_generate_output_file(ctx, name, ext + ".c") for name in proto_sources] + hdrs = [_generate_output_file(ctx, name, ext + ".h") for name in proto_sources] + thunks = [] + if generator == "upb": + thunks = [_generate_output_file(ctx, name, ext + ".thunks.c") for name in proto_sources] + transitive_sets = proto_info.transitive_descriptor_sets.to_list() + + args = ctx.actions.args() + args.use_param_file(param_file_arg = "@%s") + args.set_param_file_format("multiline") + + args.add("--" + generator + "_out=" + _get_real_root(ctx, srcs[0])) + args.add("--plugin=protoc-gen-" + generator + "=" + tool.path) + args.add("--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets])) + args.add_all(proto_sources, map_each = _get_real_short_path) + + ctx.actions.run( + inputs = depset( + direct = [proto_info.direct_descriptor_set], + transitive = [proto_info.transitive_descriptor_sets], + ), + tools = [tool], + outputs = srcs + hdrs, + executable = ctx.executable._protoc, + arguments = [args], + progress_message = "Generating upb protos for :" + ctx.label.name, + mnemonic = "GenUpbProtos", + ) + if generator == "upb": + ctx.actions.run_shell( + inputs = hdrs, + outputs = thunks, + command = " && ".join([ + "sed 's/UPB_INLINE //' {} > {}".format(hdr.path, thunk.path) + for (hdr, thunk) in zip(hdrs, thunks) + ]), + progress_message = "Generating thunks for upb protos API for: " + ctx.label.name, + mnemonic = "GenUpbProtosThunks", + ) + return GeneratedSrcsInfo( + srcs = srcs, + hdrs = hdrs, + thunks = thunks, + includes = [_generate_include_path(proto_sources[0], hdrs[0], ext + ".h")], + ) + +def _generate_name(ctx, generator, thunks = False): + if thunks: + return ctx.rule.attr.name + "." + generator + ".thunks" + return ctx.rule.attr.name + "." + generator + +def _get_dep_cc_infos(target, ctx, generator, cc_provider, dep_cc_provider): + aspect_deps = getattr(ctx.attr, "_" + generator) + rule_deps = ctx.rule.attr.deps + dep_ccinfos = [dep[CcInfo] for dep in aspect_deps] + dep_ccinfos += [dep[cc_provider].cc_info for dep in rule_deps] + if dep_cc_provider: + dep_ccinfos.append(target[dep_cc_provider].cc_info) + return dep_ccinfos + +def _compile_upb_protos(ctx, files, generator, dep_ccinfos, cc_provider): + cc_info = cc_library_func( + ctx = ctx, + name = _generate_name(ctx, generator), + hdrs = files.hdrs, + srcs = files.srcs, + includes = files.includes, + copts = ctx.attr._copts[UpbProtoLibraryCoptsInfo].copts, + dep_ccinfos = dep_ccinfos, + ) + + if files.thunks: + cc_info_with_thunks = cc_library_func( + ctx = ctx, + name = _generate_name(ctx, generator, files.thunks), + hdrs = [], + srcs = files.thunks, + includes = files.includes, + copts = ctx.attr._copts[UpbProtoLibraryCoptsInfo].copts, + dep_ccinfos = dep_ccinfos + [cc_info], + ) + return cc_provider( + cc_info = cc_info, + cc_info_with_thunks = cc_info_with_thunks, + ) + else: + return cc_provider( + cc_info = cc_info, + ) + +_GENERATORS = ["upb", "upbdefs"] + +def _get_hint_providers(ctx, generator): + if generator not in _GENERATORS: + fail("Please add new generator '{}' to _GENERATORS list".format(generator)) + + possible_owners = [] + for generator in _GENERATORS: + possible_owners.append(ctx.label.relative(_generate_name(ctx, generator))) + possible_owners.append(ctx.label.relative(_generate_name(ctx, generator, thunks = True))) + + if hasattr(cc_common, "CcSharedLibraryHintInfo"): + return [cc_common.CcSharedLibraryHintInfo(owners = possible_owners)] + elif hasattr(cc_common, "CcSharedLibraryHintInfo_6_X_constructor_do_not_use"): + # This branch can be deleted once 6.X is not supported by upb rules + return [cc_common.CcSharedLibraryHintInfo_6_X_constructor_do_not_use(owners = possible_owners)] + + return [] + +def upb_proto_aspect_impl( + target, + ctx, + generator, + cc_provider, + dep_cc_provider, + file_provider, + provide_cc_shared_library_hints = True): + """A shared aspect implementation for upb_*proto_library() rules. + + Args: + target: The `target` parameter from the aspect function. + ctx: The `ctx` parameter from the aspect function. + generator: A string describing which aspect we are generating. This triggers several special + behaviors, and ideally this will be refactored to be less magical. + cc_provider: The provider that this aspect will attach to the target. Should contain a + `cc_info` field. The aspect will ensure that each compilation action can compile and link + against this provider's cc_info for all proto_library() deps. + dep_cc_provider: For aspects that depend on other aspects, this is the provider of the aspect + that we depend on. The aspect wil be able to include the header files from this provider. + file_provider: A provider that this aspect will attach to the target to expose the source + files generated by this aspect. These files are primarily useful for returning in + DefaultInfo(), so users who build the upb_*proto_library() rule directly can view the + generated sources. + provide_cc_shared_library_hints: Whether shared library hints should be provided. + + Returns: + The `cc_provider` and `file_provider` providers as described above. + """ + dep_ccinfos = _get_dep_cc_infos(target, ctx, generator, cc_provider, dep_cc_provider) + if not getattr(ctx.rule.attr, "srcs", []): + # This target doesn't declare any sources, reexport all its deps instead. + # This is known as an "alias library": + # https://bazel.build/reference/be/protocol-buffer#proto_library.srcs + files = _merge_generated_srcs([dep[file_provider].srcs for dep in ctx.rule.attr.deps]) + wrapped_cc_info = cc_provider( + cc_info = cc_common.merge_cc_infos(direct_cc_infos = dep_ccinfos), + ) + else: + proto_info = target[ProtoInfo] + files = _generate_upb_protos( + ctx, + generator, + proto_info, + proto_info.direct_sources, + ) + wrapped_cc_info = _compile_upb_protos( + ctx, + files, + generator, + dep_ccinfos, + cc_provider, + ) + + hints = _get_hint_providers(ctx, generator) if provide_cc_shared_library_hints else [] + + return hints + [ + file_provider(srcs = files), + wrapped_cc_info, + ] diff --git a/upb/bazel/upb_proto_library_internal/cc_library_func.bzl b/upb/bazel/upb_proto_library_internal/cc_library_func.bzl new file mode 100644 index 0000000000..31ebe9057b --- /dev/null +++ b/upb/bazel/upb_proto_library_internal/cc_library_func.bzl @@ -0,0 +1,71 @@ +"""A function to compile C/C++ code, like cc_library() but from Starlark.""" + +# begin:google_only +# load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain", "use_cpp_toolchain") +# +# def upb_use_cpp_toolchain(): +# return use_cpp_toolchain() +# +# end:google_only + +# begin:github_only +# Compatibility code for Bazel 4.x. Remove this when we drop support for Bazel 4.x. +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +def upb_use_cpp_toolchain(): + return ["@bazel_tools//tools/cpp:toolchain_type"] +# end:github_only + +def cc_library_func(ctx, name, hdrs, srcs, copts, includes, dep_ccinfos): + """Like cc_library(), but callable from rules. + + Args: + ctx: Rule context. + name: Unique name used to generate output files. + hdrs: Public headers that can be #included from other rules. + srcs: C/C++ source files. + copts: Additional options for cc compilation. + includes: Additional include paths. + dep_ccinfos: CcInfo providers of dependencies we should build/link against. + + Returns: + CcInfo provider for this compilation. + """ + + compilation_contexts = [info.compilation_context for info in dep_ccinfos] + linking_contexts = [info.linking_context for info in dep_ccinfos] + toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + + (compilation_context, compilation_outputs) = cc_common.compile( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + name = name, + srcs = srcs, + includes = includes, + public_hdrs = hdrs, + user_compile_flags = copts, + compilation_contexts = compilation_contexts, + ) + + # buildifier: disable=unused-variable + (linking_context, linking_outputs) = cc_common.create_linking_context_from_compilation_outputs( + actions = ctx.actions, + name = name, + feature_configuration = feature_configuration, + cc_toolchain = toolchain, + compilation_outputs = compilation_outputs, + linking_contexts = linking_contexts, + disallow_dynamic_library = cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "targets_windows") or not cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "supports_dynamic_linker"), + ) + + return CcInfo( + compilation_context = compilation_context, + linking_context = linking_context, + ) diff --git a/upb/bazel/upb_proto_library_internal/copts.bzl b/upb/bazel/upb_proto_library_internal/copts.bzl new file mode 100644 index 0000000000..30056259f2 --- /dev/null +++ b/upb/bazel/upb_proto_library_internal/copts.bzl @@ -0,0 +1,16 @@ +"""Dummy rule to expose select() copts to aspects.""" + +UpbProtoLibraryCoptsInfo = provider( + "Provides copts for upb proto targets", + fields = { + "copts": "copts for upb_proto_library()", + }, +) + +def upb_proto_library_copts_impl(ctx): + return UpbProtoLibraryCoptsInfo(copts = ctx.attr.copts) + +upb_proto_library_copts = rule( + implementation = upb_proto_library_copts_impl, + attrs = {"copts": attr.string_list(default = [])}, +) diff --git a/upb/bazel/upb_proto_library_internal/rule.bzl b/upb/bazel/upb_proto_library_internal/rule.bzl new file mode 100644 index 0000000000..feb29292c3 --- /dev/null +++ b/upb/bazel/upb_proto_library_internal/rule.bzl @@ -0,0 +1,39 @@ +"""Internal rule implementation for upb_*_proto_library() rules.""" + +def _filter_none(elems): + out = [] + for elem in elems: + if elem: + out.append(elem) + return out + +def upb_proto_rule_impl(ctx, cc_info_provider, srcs_provider): + """An implementation for upb_*proto_library() rules. + + Args: + ctx: The rule `ctx` argument + cc_info_provider: The provider containing a wrapped CcInfo that will be exposed to users who + depend on this rule. + srcs_provider: The provider containing the generated source files. This will be used to make + the DefaultInfo return the source files. + + Returns: + Providers for this rule. + """ + if len(ctx.attr.deps) != 1: + fail("only one deps dependency allowed.") + dep = ctx.attr.deps[0] + srcs = dep[srcs_provider].srcs + cc_info = dep[cc_info_provider].cc_info + + lib = cc_info.linking_context.linker_inputs.to_list()[0].libraries[0] + files = _filter_none([ + lib.static_library, + lib.pic_static_library, + lib.dynamic_library, + ]) + return [ + DefaultInfo(files = depset(files + srcs.hdrs + srcs.srcs)), + srcs, + cc_info, + ] diff --git a/upb/bazel/upb_proto_reflection_library.bzl b/upb/bazel/upb_proto_reflection_library.bzl new file mode 100644 index 0000000000..3a52fedb6b --- /dev/null +++ b/upb/bazel/upb_proto_reflection_library.bzl @@ -0,0 +1,81 @@ +"""upb_c_proto_reflection_library() exposes upb reflection for protobuf (foo.upbdefs.h)""" + +load("//upb/bazel:upb_c_proto_library.bzl", "UpbWrappedCcInfo", "upb_c_proto_library_aspect") +load("//upb/bazel:upb_proto_library_internal/aspect.bzl", "upb_proto_aspect_impl") +load("//upb/bazel:upb_proto_library_internal/cc_library_func.bzl", "upb_use_cpp_toolchain") +load("//upb/bazel:upb_proto_library_internal/rule.bzl", "upb_proto_rule_impl") + +_UpbDefsWrappedCcInfo = provider("Provider for cc_info for protos", fields = ["cc_info"]) + +_WrappedDefsGeneratedSrcsInfo = provider( + "Provider for generated reflective sources", + fields = ["srcs"], +) + +def _upb_proto_reflection_library_aspect_impl(target, ctx): + return upb_proto_aspect_impl( + target = target, + ctx = ctx, + generator = "upbdefs", + cc_provider = _UpbDefsWrappedCcInfo, + dep_cc_provider = UpbWrappedCcInfo, + file_provider = _WrappedDefsGeneratedSrcsInfo, + provide_cc_shared_library_hints = False, + ) + +_upb_proto_reflection_library_aspect = aspect( + attrs = { + "_copts": attr.label( + default = "//upb:upb_proto_library_copts__for_generated_code_only_do_not_use", + ), + "_gen_upbdefs": attr.label( + executable = True, + cfg = "exec", + default = "//upb/upbc:protoc-gen-upbdefs", + ), + "_protoc": attr.label( + executable = True, + cfg = "exec", + default = "//:protoc", + ), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + "_upbdefs": attr.label_list( + default = [ + "//upb:generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + ], + ), + }, + implementation = _upb_proto_reflection_library_aspect_impl, + provides = [ + _UpbDefsWrappedCcInfo, + _WrappedDefsGeneratedSrcsInfo, + ], + required_aspect_providers = [ + UpbWrappedCcInfo, + ], + attr_aspects = ["deps"], + fragments = ["cpp"], + toolchains = upb_use_cpp_toolchain(), + incompatible_use_toolchain_transition = True, +) + +def _upb_proto_reflection_library_rule_impl(ctx): + return upb_proto_rule_impl(ctx, _UpbDefsWrappedCcInfo, _WrappedDefsGeneratedSrcsInfo) + +upb_proto_reflection_library = rule( + output_to_genfiles = True, + implementation = _upb_proto_reflection_library_rule_impl, + attrs = { + "deps": attr.label_list( + aspects = [ + upb_c_proto_library_aspect, + _upb_proto_reflection_library_aspect, + ], + allow_rules = ["proto_library"], + providers = [ProtoInfo], + ), + }, + provides = [CcInfo], +)