diff --git a/.gitignore b/.gitignore index 0d2647e61b6..7b4c5d36cb4 100644 --- a/.gitignore +++ b/.gitignore @@ -115,6 +115,7 @@ bazel-genfiles bazel-grpc bazel-out bazel-testlogs +bazel_format_virtual_environment/ # Debug output gdb.txt diff --git a/WORKSPACE b/WORKSPACE index fcf5b5d6d10..2db3c5db2ff 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -18,13 +18,6 @@ register_toolchains( "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows", ) -# TODO(https://github.com/grpc/grpc/issues/18331): Move off of this dependency. -git_repository( - name = "org_pubref_rules_protobuf", - remote = "https://github.com/ghostwriternr/rules_protobuf", - tag = "v0.8.2.1-alpha", -) - git_repository( name = "io_bazel_rules_python", commit = "8b5d0683a7d878b28fffe464779c8a53659fc645", diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index 8f30c84f6b9..82f5cbad310 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -4,81 +4,132 @@ This is an internal rule used by cc_grpc_library, and shouldn't be used directly. """ -def generate_cc_impl(ctx): - """Implementation of the generate_cc rule.""" - protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources] - includes = [f for src in ctx.attr.srcs for f in src.proto.transitive_imports] - outs = [] - # label_len is length of the path from WORKSPACE root to the location of this build file - label_len = 0 - # proto_root is the directory relative to which generated include paths should be - proto_root = "" - if ctx.label.package: - # The +1 is for the trailing slash. - label_len += len(ctx.label.package) + 1 - if ctx.label.workspace_root: - label_len += len(ctx.label.workspace_root) + 1 - proto_root = "/" + ctx.label.workspace_root +load( + "//bazel:protobuf.bzl", + "get_include_protoc_args", + "get_plugin_args", + "get_proto_root", + "proto_path_to_generated_filename", +) + +_GRPC_PROTO_HEADER_FMT = "{}.grpc.pb.h" +_GRPC_PROTO_SRC_FMT = "{}.grpc.pb.cc" +_GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h" +_PROTO_HEADER_FMT = "{}.pb.h" +_PROTO_SRC_FMT = "{}.pb.cc" - if ctx.executable.plugin: - outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.h" for proto in protos] - outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.cc" for proto in protos] - if ctx.attr.generate_mocks: - outs += [proto.path[label_len:-len(".proto")] + "_mock.grpc.pb.h" for proto in protos] - else: - outs += [proto.path[label_len:-len(".proto")] + ".pb.h" for proto in protos] - outs += [proto.path[label_len:-len(".proto")] + ".pb.cc" for proto in protos] - out_files = [ctx.actions.declare_file(out) for out in outs] - dir_out = str(ctx.genfiles_dir.path + proto_root) +def _strip_package_from_path(label_package, path): + if len(label_package) == 0: + return path + if not path.startswith(label_package + "/"): + fail("'{}' does not lie within '{}'.".format(path, label_package)) + return path[len(label_package + "/"):] - arguments = [] - if ctx.executable.plugin: - arguments += ["--plugin=protoc-gen-PLUGIN=" + ctx.executable.plugin.path] - flags = list(ctx.attr.flags) - if ctx.attr.generate_mocks: - flags.append("generate_mock_code=true") - arguments += ["--PLUGIN_out=" + ",".join(flags) + ":" + dir_out] - tools = [ctx.executable.plugin] - else: - arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out] - tools = [] +def _join_directories(directories): + massaged_directories = [directory for directory in directories if len(directory) != 0] + return "/".join(massaged_directories) - # Import protos relative to their workspace root so that protoc prints the - # right include paths. - for include in includes: - directory = include.path - if directory.startswith("external"): - external_sep = directory.find("/") - repository_sep = directory.find("/", external_sep + 1) - arguments += ["--proto_path=" + directory[:repository_sep]] +def generate_cc_impl(ctx): + """Implementation of the generate_cc rule.""" + protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources] + includes = [ + f + for src in ctx.attr.srcs + for f in src.proto.transitive_imports + ] + outs = [] + proto_root = get_proto_root( + ctx.label.workspace_root, + ) + + label_package = _join_directories([ctx.label.workspace_root, ctx.label.package]) + if ctx.executable.plugin: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto.path), + _GRPC_PROTO_HEADER_FMT, + ) + for proto in protos + ] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto.path), + _GRPC_PROTO_SRC_FMT, + ) + for proto in protos + ] + if ctx.attr.generate_mocks: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto.path), + _GRPC_PROTO_MOCK_HEADER_FMT, + ) + for proto in protos + ] else: - arguments += ["--proto_path=."] - # Include the output directory so that protoc puts the generated code in the - # right directory. - arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)] - arguments += [proto.path for proto in protos] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto.path), + _PROTO_HEADER_FMT, + ) + for proto in protos + ] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto.path), + _PROTO_SRC_FMT, + ) + for proto in protos + ] + out_files = [ctx.actions.declare_file(out) for out in outs] + dir_out = str(ctx.genfiles_dir.path + proto_root) - # create a list of well known proto files if the argument is non-None - well_known_proto_files = [] - if ctx.attr.well_known_protos: - f = ctx.attr.well_known_protos.files.to_list()[0].dirname - if f != "external/com_google_protobuf/src/google/protobuf": - print("Error: Only @com_google_protobuf//:well_known_protos is supported") + arguments = [] + if ctx.executable.plugin: + arguments += get_plugin_args( + ctx.executable.plugin, + ctx.attr.flags, + dir_out, + ctx.attr.generate_mocks, + ) + tools = [ctx.executable.plugin] else: - # f points to "external/com_google_protobuf/src/google/protobuf" - # add -I argument to protoc so it knows where to look for the proto files. - arguments += ["-I{0}".format(f + "/../..")] - well_known_proto_files = [f for f in ctx.attr.well_known_protos.files] + arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out] + tools = [] + + arguments += get_include_protoc_args(includes) - ctx.actions.run( - inputs = protos + includes + well_known_proto_files, - tools = tools, - outputs = out_files, - executable = ctx.executable._protoc, - arguments = arguments, - ) + # Include the output directory so that protoc puts the generated code in the + # right directory. + arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)] + arguments += [proto.path for proto in protos] - return struct(files=depset(out_files)) + # create a list of well known proto files if the argument is non-None + well_known_proto_files = [] + if ctx.attr.well_known_protos: + f = ctx.attr.well_known_protos.files.to_list()[0].dirname + if f != "external/com_google_protobuf/src/google/protobuf": + print( + "Error: Only @com_google_protobuf//:well_known_protos is supported", + ) + else: + # f points to "external/com_google_protobuf/src/google/protobuf" + # add -I argument to protoc so it knows where to look for the proto files. + arguments += ["-I{0}".format(f + "/../..")] + well_known_proto_files = [ + f + for f in ctx.attr.well_known_protos.files + ] + + ctx.actions.run( + inputs = protos + includes + well_known_proto_files, + tools = tools, + outputs = out_files, + executable = ctx.executable._protoc, + arguments = arguments, + ) + + return struct(files = depset(out_files)) _generate_cc = rule( attrs = { @@ -96,10 +147,8 @@ _generate_cc = rule( mandatory = False, allow_empty = True, ), - "well_known_protos" : attr.label( - mandatory = False, - ), - "generate_mocks" : attr.bool( + "well_known_protos": attr.label(mandatory = False), + "generate_mocks": attr.bool( default = False, mandatory = False, ), @@ -115,7 +164,10 @@ _generate_cc = rule( ) def generate_cc(well_known_protos, **kwargs): - if well_known_protos: - _generate_cc(well_known_protos="@com_google_protobuf//:well_known_protos", **kwargs) - else: - _generate_cc(**kwargs) + if well_known_protos: + _generate_cc( + well_known_protos = "@com_google_protobuf//:well_known_protos", + **kwargs + ) + else: + _generate_cc(**kwargs) diff --git a/bazel/grpc_python_deps.bzl b/bazel/grpc_python_deps.bzl index ec3df19e03a..91438f3927b 100644 --- a/bazel/grpc_python_deps.bzl +++ b/bazel/grpc_python_deps.bzl @@ -1,16 +1,8 @@ load("//third_party/py:python_configure.bzl", "python_configure") load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") load("@grpc_python_dependencies//:requirements.bzl", "pip_install") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories") def grpc_python_deps(): - # TODO(https://github.com/grpc/grpc/issues/18256): Remove conditional. - if hasattr(native, "http_archive"): - python_configure(name = "local_config_python") - pip_repositories() - pip_install() - py_proto_repositories() - else: - print("Building Python gRPC with bazel 23.0+ is disabled pending " + - "resolution of https://github.com/grpc/grpc/issues/18256.") - + python_configure(name = "local_config_python") + pip_repositories() + pip_install() diff --git a/bazel/protobuf.bzl b/bazel/protobuf.bzl new file mode 100644 index 00000000000..bddd0d70c7f --- /dev/null +++ b/bazel/protobuf.bzl @@ -0,0 +1,84 @@ +"""Utility functions for generating protobuf code.""" + +_PROTO_EXTENSION = ".proto" + +def get_proto_root(workspace_root): + """Gets the root protobuf directory. + + Args: + workspace_root: context.label.workspace_root + + Returns: + The directory relative to which generated include paths should be. + """ + if workspace_root: + return "/{}".format(workspace_root) + else: + return "" + +def _strip_proto_extension(proto_filename): + if not proto_filename.endswith(_PROTO_EXTENSION): + fail('"{}" does not end with "{}"'.format( + proto_filename, + _PROTO_EXTENSION, + )) + return proto_filename[:-len(_PROTO_EXTENSION)] + +def proto_path_to_generated_filename(proto_path, fmt_str): + """Calculates the name of a generated file for a protobuf path. + + For example, "examples/protos/helloworld.proto" might map to + "helloworld.pb.h". + + Args: + proto_path: The path to the .proto file. + fmt_str: A format string used to calculate the generated filename. For + example, "{}.pb.h" might be used to calculate a C++ header filename. + + Returns: + The generated filename. + """ + return fmt_str.format(_strip_proto_extension(proto_path)) + +def _get_include_directory(include): + directory = include.path + if directory.startswith("external"): + external_separator = directory.find("/") + repository_separator = directory.find("/", external_separator + 1) + return directory[:repository_separator] + else: + return "." + +def get_include_protoc_args(includes): + """Returns protoc args that imports protos relative to their import root. + + Args: + includes: A list of included proto files. + + Returns: + A list of arguments to be passed to protoc. For example, ["--proto_path=."]. + """ + return [ + "--proto_path={}".format(_get_include_directory(include)) + for include in includes + ] + +def get_plugin_args(plugin, flags, dir_out, generate_mocks): + """Returns arguments configuring protoc to use a plugin for a language. + + Args: + plugin: An executable file to run as the protoc plugin. + flags: The plugin flags to be passed to protoc. + dir_out: The output directory for the plugin. + generate_mocks: A bool indicating whether to generate mocks. + + Returns: + A list of protoc arguments configuring the plugin. + """ + augmented_flags = list(flags) + if generate_mocks: + augmented_flags.append("generate_mock_code=true") + return [ + "--plugin=protoc-gen-PLUGIN=" + plugin.path, + "--PLUGIN_out=" + ",".join(augmented_flags) + ":" + dir_out, + ] diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl new file mode 100644 index 00000000000..bf6b4bec8d2 --- /dev/null +++ b/bazel/python_rules.bzl @@ -0,0 +1,203 @@ +"""Generates and compiles Python gRPC stubs from proto_library rules.""" + +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load( + "//bazel:protobuf.bzl", + "get_include_protoc_args", + "get_plugin_args", + "get_proto_root", + "proto_path_to_generated_filename", +) + +_GENERATED_PROTO_FORMAT = "{}_pb2.py" +_GENERATED_GRPC_PROTO_FORMAT = "{}_pb2_grpc.py" + +def _get_staged_proto_file(context, source_file): + if source_file.dirname == context.label.package: + return source_file + else: + copied_proto = context.actions.declare_file(source_file.basename) + context.actions.run_shell( + inputs = [source_file], + outputs = [copied_proto], + command = "cp {} {}".format(source_file.path, copied_proto.path), + mnemonic = "CopySourceProto", + ) + return copied_proto + +def _generate_py_impl(context): + protos = [] + for src in context.attr.deps: + for file in src.proto.direct_sources: + protos.append(_get_staged_proto_file(context, file)) + includes = [ + file + for src in context.attr.deps + for file in src.proto.transitive_imports + ] + proto_root = get_proto_root(context.label.workspace_root) + format_str = (_GENERATED_GRPC_PROTO_FORMAT if context.executable.plugin else _GENERATED_PROTO_FORMAT) + out_files = [ + context.actions.declare_file( + proto_path_to_generated_filename( + proto.basename, + format_str, + ), + ) + for proto in protos + ] + + arguments = [] + tools = [context.executable._protoc] + if context.executable.plugin: + arguments += get_plugin_args( + context.executable.plugin, + context.attr.flags, + context.genfiles_dir.path, + False, + ) + tools += [context.executable.plugin] + else: + arguments += [ + "--python_out={}:{}".format( + ",".join(context.attr.flags), + context.genfiles_dir.path, + ), + ] + + arguments += get_include_protoc_args(includes) + arguments += [ + "--proto_path={}".format(context.genfiles_dir.path) + for proto in protos + ] + for proto in protos: + massaged_path = proto.path + if massaged_path.startswith(context.genfiles_dir.path): + massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] + arguments.append(massaged_path) + + well_known_proto_files = [] + if context.attr.well_known_protos: + well_known_proto_directory = context.attr.well_known_protos.files.to_list( + )[0].dirname + + arguments += ["-I{}".format(well_known_proto_directory + "/../..")] + well_known_proto_files = context.attr.well_known_protos.files.to_list() + + context.actions.run( + inputs = protos + includes + well_known_proto_files, + tools = tools, + outputs = out_files, + executable = context.executable._protoc, + arguments = arguments, + mnemonic = "ProtocInvocation", + ) + return struct(files = depset(out_files)) + +__generate_py = rule( + attrs = { + "deps": attr.label_list( + mandatory = True, + allow_empty = False, + providers = ["proto"], + ), + "plugin": attr.label( + executable = True, + providers = ["files_to_run"], + cfg = "host", + ), + "flags": attr.string_list( + mandatory = False, + allow_empty = True, + ), + "well_known_protos": attr.label(mandatory = False), + "_protoc": attr.label( + default = Label("//external:protocol_compiler"), + executable = True, + cfg = "host", + ), + }, + output_to_genfiles = True, + implementation = _generate_py_impl, +) + +def _generate_py(well_known_protos, **kwargs): + if well_known_protos: + __generate_py( + well_known_protos = "@com_google_protobuf//:well_known_protos", + **kwargs + ) + else: + __generate_py(**kwargs) + +_WELL_KNOWN_PROTO_LIBS = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:api_proto", + "@com_google_protobuf//:compiler_plugin_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:field_mask_proto", + "@com_google_protobuf//:source_context_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:type_proto", + "@com_google_protobuf//:wrappers_proto", +] + +def py_proto_library( + name, + deps, + well_known_protos = True, + proto_only = False, + **kwargs): + """Generate python code for a protobuf. + + Args: + name: The name of the target. + deps: A list of dependencies. Must contain a single element. + well_known_protos: A bool indicating whether or not to include well-known + protos. + proto_only: A bool indicating whether to generate vanilla protobuf code + or to also generate gRPC code. + """ + if len(deps) > 1: + fail("The supported length of 'deps' is 1.") + + codegen_target = "_{}_codegen".format(name) + codegen_grpc_target = "_{}_grpc_codegen".format(name) + + well_known_proto_rules = _WELL_KNOWN_PROTO_LIBS if well_known_protos else [] + + _generate_py( + name = codegen_target, + deps = deps, + well_known_protos = well_known_protos, + **kwargs + ) + + if not proto_only: + _generate_py( + name = codegen_grpc_target, + deps = deps, + plugin = "//:grpc_python_plugin", + well_known_protos = well_known_protos, + **kwargs + ) + + native.py_library( + name = name, + srcs = [ + ":{}".format(codegen_grpc_target), + ":{}".format(codegen_target), + ], + deps = [requirement("protobuf")], + **kwargs + ) + else: + native.py_library( + name = name, + srcs = [":{}".format(codegen_target), ":{}".format(codegen_target)], + deps = [requirement("protobuf")], + **kwargs + ) diff --git a/examples/BUILD b/examples/BUILD index d2b39b87f4d..d5fb6903d7c 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -16,9 +16,8 @@ licenses(["notice"]) # 3-clause BSD package(default_visibility = ["//visibility:public"]) -load("@grpc_python_dependencies//:requirements.bzl", "requirement") load("//bazel:grpc_build_system.bzl", "grpc_proto_library") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library") grpc_proto_library( name = "auth_sample", @@ -45,11 +44,14 @@ grpc_proto_library( srcs = ["protos/keyvaluestore.proto"], ) +proto_library( + name = "helloworld_proto_descriptor", + srcs = ["protos/helloworld.proto"], +) + py_proto_library( name = "py_helloworld", - protos = ["protos/helloworld.proto"], - with_grpc = True, - deps = [requirement('protobuf'),], + deps = [":helloworld_proto_descriptor"], ) cc_binary( diff --git a/examples/python/errors/client.py b/examples/python/errors/client.py index a79b8fce1bd..c762d798dc2 100644 --- a/examples/python/errors/client.py +++ b/examples/python/errors/client.py @@ -20,8 +20,8 @@ import grpc from grpc_status import rpc_status from google.rpc import error_details_pb2 -from examples.protos import helloworld_pb2 -from examples.protos import helloworld_pb2_grpc +from examples import helloworld_pb2 +from examples import helloworld_pb2_grpc _LOGGER = logging.getLogger(__name__) diff --git a/examples/python/errors/server.py b/examples/python/errors/server.py index f49586b848a..50d4a2ac671 100644 --- a/examples/python/errors/server.py +++ b/examples/python/errors/server.py @@ -24,8 +24,8 @@ from grpc_status import rpc_status from google.protobuf import any_pb2 from google.rpc import code_pb2, status_pb2, error_details_pb2 -from examples.protos import helloworld_pb2 -from examples.protos import helloworld_pb2_grpc +from examples import helloworld_pb2 +from examples import helloworld_pb2_grpc _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/examples/python/errors/test/_error_handling_example_test.py b/examples/python/errors/test/_error_handling_example_test.py index a79ca45e2a1..9eb81ba3742 100644 --- a/examples/python/errors/test/_error_handling_example_test.py +++ b/examples/python/errors/test/_error_handling_example_test.py @@ -26,7 +26,7 @@ import logging import grpc -from examples.protos import helloworld_pb2_grpc +from examples import helloworld_pb2_grpc from examples.python.errors import client as error_handling_client from examples.python.errors import server as error_handling_server diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index 6de1e947d85..0e135f471f2 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -15,12 +15,17 @@ # limitations under the License. load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library") -py_proto_library( +proto_library( name = "prime_proto", - protos = ["prime.proto",], - deps = [requirement("protobuf")], + srcs = ["prime.proto"] +) + +py_proto_library( + name = "prime_proto_pb2", + deps = [":prime_proto"], + well_known_protos = False, ) py_binary( @@ -29,7 +34,7 @@ py_binary( srcs = ["client.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":prime_proto", + ":prime_proto_pb2", ], default_python_version = "PY3", ) @@ -40,7 +45,7 @@ py_binary( srcs = ["server.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":prime_proto" + ":prime_proto_pb2" ] + select({ "//conditions:default": [requirement("futures")], "//:python3": [], diff --git a/examples/python/wait_for_ready/wait_for_ready_example.py b/examples/python/wait_for_ready/wait_for_ready_example.py index 7c16b9bd4a1..a0f076e894a 100644 --- a/examples/python/wait_for_ready/wait_for_ready_example.py +++ b/examples/python/wait_for_ready/wait_for_ready_example.py @@ -22,8 +22,8 @@ import threading import grpc -from examples.protos import helloworld_pb2 -from examples.protos import helloworld_pb2_grpc +from examples import helloworld_pb2 +from examples import helloworld_pb2_grpc _LOGGER = logging.getLogger(__name__) _LOGGER.setLevel(logging.INFO) @@ -33,10 +33,13 @@ _ONE_DAY_IN_SECONDS = 60 * 60 * 24 @contextmanager def get_free_loopback_tcp_port(): - tcp_socket = socket.socket(socket.AF_INET6) + if socket.has_ipv6: + tcp_socket = socket.socket(socket.AF_INET6) + else: + tcp_socket = socket.socket(socket.AF_INET) tcp_socket.bind(('', 0)) address_tuple = tcp_socket.getsockname() - yield "[::1]:%s" % (address_tuple[1]) + yield "localhost:%s" % (address_tuple[1]) tcp_socket.close() diff --git a/src/proto/grpc/channelz/BUILD b/src/proto/grpc/channelz/BUILD index b6b485e3e8d..1d80ec23af1 100644 --- a/src/proto/grpc/channelz/BUILD +++ b/src/proto/grpc/channelz/BUILD @@ -25,6 +25,11 @@ grpc_proto_library( well_known_protos = True, ) +proto_library( + name = "channelz_proto_descriptors", + srcs = ["channelz.proto"], +) + filegroup( name = "channelz_proto_file", srcs = [ diff --git a/src/proto/grpc/health/v1/BUILD b/src/proto/grpc/health/v1/BUILD index 38a7d992e06..fc58e8a1770 100644 --- a/src/proto/grpc/health/v1/BUILD +++ b/src/proto/grpc/health/v1/BUILD @@ -23,6 +23,11 @@ grpc_proto_library( srcs = ["health.proto"], ) +proto_library( + name = "health_proto_descriptor", + srcs = ["health.proto"], +) + filegroup( name = "health_proto_file", srcs = [ diff --git a/src/proto/grpc/reflection/v1alpha/BUILD b/src/proto/grpc/reflection/v1alpha/BUILD index 4d919d029ee..5424c0d867e 100644 --- a/src/proto/grpc/reflection/v1alpha/BUILD +++ b/src/proto/grpc/reflection/v1alpha/BUILD @@ -23,6 +23,11 @@ grpc_proto_library( srcs = ["reflection.proto"], ) +proto_library( + name = "reflection_proto_descriptor", + srcs = ["reflection.proto"], +) + filegroup( name = "reflection_proto_file", srcs = [ diff --git a/src/proto/grpc/testing/BUILD b/src/proto/grpc/testing/BUILD index 9876d5160a1..16e47d81061 100644 --- a/src/proto/grpc/testing/BUILD +++ b/src/proto/grpc/testing/BUILD @@ -16,7 +16,7 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package") load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library") grpc_package(name = "testing", visibility = "public") @@ -61,13 +61,14 @@ grpc_proto_library( has_services = False, ) +proto_library( + name = "empty_proto_descriptor", + srcs = ["empty.proto"], +) + py_proto_library( name = "py_empty_proto", - protos = ["empty.proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], + deps = [":empty_proto_descriptor"], ) grpc_proto_library( @@ -76,13 +77,14 @@ grpc_proto_library( has_services = False, ) +proto_library( + name = "messages_proto_descriptor", + srcs = ["messages.proto"], +) + py_proto_library( name = "py_messages_proto", - protos = ["messages.proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], + deps = [":messages_proto_descriptor"], ) grpc_proto_library( @@ -144,16 +146,19 @@ grpc_proto_library( ], ) +proto_library( + name = "test_proto_descriptor", + srcs = ["test.proto"], + deps = [ + ":empty_proto_descriptor", + ":messages_proto_descriptor", + ], +) + py_proto_library( name = "py_test_proto", - protos = ["test.proto",], - with_grpc = True, deps = [ - requirement('protobuf'), - ], - proto_deps = [ - ":py_empty_proto", - ":py_messages_proto", + ":test_proto_descriptor", ] ) diff --git a/src/proto/grpc/testing/proto2/BUILD.bazel b/src/proto/grpc/testing/proto2/BUILD.bazel index c4c4f004efb..e939c523a64 100644 --- a/src/proto/grpc/testing/proto2/BUILD.bazel +++ b/src/proto/grpc/testing/proto2/BUILD.bazel @@ -1,30 +1,32 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) +load("//bazel:python_rules.bzl", "py_proto_library") + +proto_library( + name = "empty2_proto_descriptor", + srcs = ["empty2.proto"], +) py_proto_library( name = "empty2_proto", - protos = [ - "empty2.proto", - ], - with_grpc = True, deps = [ - requirement('protobuf'), + ":empty2_proto_descriptor", ], ) +proto_library( + name = "empty2_extensions_proto_descriptor", + srcs = ["empty2_extensions.proto"], + deps = [ + ":empty2_proto_descriptor", + ] +) + py_proto_library( name = "empty2_extensions_proto", - protos = [ - "empty2_extensions.proto", - ], - proto_deps = [ - ":empty2_proto", - ], - with_grpc = True, deps = [ - requirement('protobuf'), + ":empty2_extensions_proto_descriptor", ], ) diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel index aae8cedb760..eccc166577d 100644 --- a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel +++ b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel @@ -1,30 +1,10 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") - +load("//bazel:python_rules.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) -genrule( - name = "mv_channelz_proto", - srcs = [ - "//src/proto/grpc/channelz:channelz_proto_file", - ], - outs = ["channelz.proto",], - cmd = "cp $< $@", -) - py_proto_library( name = "py_channelz_proto", - protos = ["mv_channelz_proto",], - imports = [ - "external/com_google_protobuf/src/", - ], - inputs = [ - "@com_google_protobuf//:well_known_protos", - ], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], + well_known_protos = True, + deps = ["//src/proto/grpc/channelz:channelz_proto_descriptors"], ) py_library( diff --git a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel index ce3121fa90e..9e4fff34581 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel +++ b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel @@ -1,24 +1,9 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") - +load("//bazel:python_rules.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) -genrule( - name = "mv_health_proto", - srcs = [ - "//src/proto/grpc/health/v1:health_proto_file", - ], - outs = ["health.proto",], - cmd = "cp $< $@", -) - py_proto_library( name = "py_health_proto", - protos = [":mv_health_proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], + deps = ["//src/proto/grpc/health/v1:health_proto_descriptor",], ) py_library( diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel index 3a2ba263715..6aaa4dd3bdd 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel @@ -1,24 +1,11 @@ +load("//bazel:python_rules.bzl", "py_proto_library") load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) -genrule( - name = "mv_reflection_proto", - srcs = [ - "//src/proto/grpc/reflection/v1alpha:reflection_proto_file", - ], - outs = ["reflection.proto",], - cmd = "cp $< $@", -) - py_proto_library( name = "py_reflection_proto", - protos = [":mv_reflection_proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], + deps = ["//src/proto/grpc/reflection/v1alpha:reflection_proto_descriptor",], ) py_library( diff --git a/src/python/grpcio_tests/tests/reflection/BUILD.bazel b/src/python/grpcio_tests/tests/reflection/BUILD.bazel index c0efb0b7cef..b635b721631 100644 --- a/src/python/grpcio_tests/tests/reflection/BUILD.bazel +++ b/src/python/grpcio_tests/tests/reflection/BUILD.bazel @@ -14,6 +14,7 @@ py_test( "//src/python/grpcio_tests/tests/unit:test_common", "//src/proto/grpc/testing:py_empty_proto", "//src/proto/grpc/testing/proto2:empty2_extensions_proto", + "//src/proto/grpc/testing/proto2:empty2_proto", requirement('protobuf'), ], imports=["../../",], diff --git a/tools/distrib/bazel_style.cfg b/tools/distrib/bazel_style.cfg new file mode 100644 index 00000000000..a5a1fea4aba --- /dev/null +++ b/tools/distrib/bazel_style.cfg @@ -0,0 +1,4 @@ +[style] +based_on_style = google +allow_split_before_dict_value = False +spaces_around_default_or_named_assign = True diff --git a/tools/distrib/format_bazel.sh b/tools/distrib/format_bazel.sh new file mode 100755 index 00000000000..ee230118efb --- /dev/null +++ b/tools/distrib/format_bazel.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2019 The gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set=-ex + +VIRTUAL_ENV=bazel_format_virtual_environment + +CONFIG_PATH="$(dirname ${0})/bazel_style.cfg" + +python -m virtualenv ${VIRTUAL_ENV} +PYTHON=${VIRTUAL_ENV}/bin/python +"$PYTHON" -m pip install --upgrade pip==10.0.1 +"$PYTHON" -m pip install --upgrade futures +"$PYTHON" -m pip install yapf==0.20.0 + +pushd "$(dirname "${0}")/../.." +FILES=$(find . -path ./third_party -prune -o -name '*.bzl' -print) +echo "${FILES}" | xargs "$PYTHON" -m yapf -i --style="${CONFIG_PATH}" + +if ! which buildifier &>/dev/null; then + echo 'buildifer must be installed.' >/dev/stderr + exit 1 +fi + +echo "${FILES}" | xargs buildifier --type=bzl + +popd diff --git a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh index 3dd0167b7c0..24598f43f02 100755 --- a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh +++ b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh @@ -24,5 +24,4 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') cd /var/local/git/grpc -#TODO(yfen): add back examples/... to build targets once python rules issues are resolved -bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... +bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/... diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index 3ca4673ca08..d844cff7f9a 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -23,10 +23,9 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc (cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') -#TODO(yfen): temporarily disabled all python bazel tests due to incompatibility with bazel 0.23.2 -#cd /var/local/git/grpc/test -#bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... -#bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... -#bazel clean --expunge -#bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... -#bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... +cd /var/local/git/grpc/test +bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... +bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... +bazel clean --expunge +bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... +bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/...