From 50b5240d0a9cf0002049982d8a84cce675b0f691 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 29 Apr 2019 17:47:56 -0700 Subject: [PATCH] Revert "Merge pull request #18859 from grpc/internal_py_proto_library" This reverts commit 5b720f19c17d623b1b42f9089108295e25f89d88, reversing changes made to a64ae3c0d54903aa27fcf27296d7bc6c67d1e192. --- .gitignore | 1 - WORKSPACE | 7 + bazel/generate_cc.bzl | 202 +++++++---------- bazel/grpc_python_deps.bzl | 14 +- bazel/protobuf.bzl | 84 -------- bazel/python_rules.bzl | 203 ------------------ examples/BUILD | 12 +- examples/python/errors/client.py | 4 +- examples/python/errors/server.py | 4 +- .../test/_error_handling_example_test.py | 2 +- examples/python/multiprocessing/BUILD | 17 +- .../wait_for_ready/wait_for_ready_example.py | 11 +- src/proto/grpc/channelz/BUILD | 5 - src/proto/grpc/health/v1/BUILD | 5 - src/proto/grpc/reflection/v1alpha/BUILD | 5 - src/proto/grpc/testing/BUILD | 41 ++-- src/proto/grpc/testing/proto2/BUILD.bazel | 30 ++- .../grpc_channelz/v1/BUILD.bazel | 26 ++- .../grpc_health/v1/BUILD.bazel | 19 +- .../grpc_reflection/v1alpha/BUILD.bazel | 17 +- .../grpcio_tests/tests/reflection/BUILD.bazel | 1 - tools/distrib/bazel_style.cfg | 4 - tools/distrib/format_bazel.sh | 39 ---- .../linux/grpc_bazel_build_in_docker.sh | 3 +- .../linux/grpc_python_bazel_test_in_docker.sh | 13 +- 25 files changed, 209 insertions(+), 560 deletions(-) delete mode 100644 bazel/protobuf.bzl delete mode 100644 bazel/python_rules.bzl delete mode 100644 tools/distrib/bazel_style.cfg delete mode 100755 tools/distrib/format_bazel.sh diff --git a/.gitignore b/.gitignore index 7b4c5d36cb4..0d2647e61b6 100644 --- a/.gitignore +++ b/.gitignore @@ -115,7 +115,6 @@ bazel-genfiles bazel-grpc bazel-out bazel-testlogs -bazel_format_virtual_environment/ # Debug output gdb.txt diff --git a/WORKSPACE b/WORKSPACE index 2db3c5db2ff..fcf5b5d6d10 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -18,6 +18,13 @@ register_toolchains( "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows", ) +# TODO(https://github.com/grpc/grpc/issues/18331): Move off of this dependency. +git_repository( + name = "org_pubref_rules_protobuf", + remote = "https://github.com/ghostwriternr/rules_protobuf", + tag = "v0.8.2.1-alpha", +) + git_repository( name = "io_bazel_rules_python", commit = "8b5d0683a7d878b28fffe464779c8a53659fc645", diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index 82f5cbad310..8f30c84f6b9 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -4,132 +4,81 @@ This is an internal rule used by cc_grpc_library, and shouldn't be used directly. """ -load( - "//bazel:protobuf.bzl", - "get_include_protoc_args", - "get_plugin_args", - "get_proto_root", - "proto_path_to_generated_filename", -) - -_GRPC_PROTO_HEADER_FMT = "{}.grpc.pb.h" -_GRPC_PROTO_SRC_FMT = "{}.grpc.pb.cc" -_GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h" -_PROTO_HEADER_FMT = "{}.pb.h" -_PROTO_SRC_FMT = "{}.pb.cc" - -def _strip_package_from_path(label_package, path): - if len(label_package) == 0: - return path - if not path.startswith(label_package + "/"): - fail("'{}' does not lie within '{}'.".format(path, label_package)) - return path[len(label_package + "/"):] +def generate_cc_impl(ctx): + """Implementation of the generate_cc rule.""" + protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources] + includes = [f for src in ctx.attr.srcs for f in src.proto.transitive_imports] + outs = [] + # label_len is length of the path from WORKSPACE root to the location of this build file + label_len = 0 + # proto_root is the directory relative to which generated include paths should be + proto_root = "" + if ctx.label.package: + # The +1 is for the trailing slash. + label_len += len(ctx.label.package) + 1 + if ctx.label.workspace_root: + label_len += len(ctx.label.workspace_root) + 1 + proto_root = "/" + ctx.label.workspace_root -def _join_directories(directories): - massaged_directories = [directory for directory in directories if len(directory) != 0] - return "/".join(massaged_directories) + if ctx.executable.plugin: + outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.h" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + ".grpc.pb.cc" for proto in protos] + if ctx.attr.generate_mocks: + outs += [proto.path[label_len:-len(".proto")] + "_mock.grpc.pb.h" for proto in protos] + else: + outs += [proto.path[label_len:-len(".proto")] + ".pb.h" for proto in protos] + outs += [proto.path[label_len:-len(".proto")] + ".pb.cc" for proto in protos] + out_files = [ctx.actions.declare_file(out) for out in outs] + dir_out = str(ctx.genfiles_dir.path + proto_root) -def generate_cc_impl(ctx): - """Implementation of the generate_cc rule.""" - protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources] - includes = [ - f - for src in ctx.attr.srcs - for f in src.proto.transitive_imports - ] - outs = [] - proto_root = get_proto_root( - ctx.label.workspace_root, - ) + arguments = [] + if ctx.executable.plugin: + arguments += ["--plugin=protoc-gen-PLUGIN=" + ctx.executable.plugin.path] + flags = list(ctx.attr.flags) + if ctx.attr.generate_mocks: + flags.append("generate_mock_code=true") + arguments += ["--PLUGIN_out=" + ",".join(flags) + ":" + dir_out] + tools = [ctx.executable.plugin] + else: + arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out] + tools = [] - label_package = _join_directories([ctx.label.workspace_root, ctx.label.package]) - if ctx.executable.plugin: - outs += [ - proto_path_to_generated_filename( - _strip_package_from_path(label_package, proto.path), - _GRPC_PROTO_HEADER_FMT, - ) - for proto in protos - ] - outs += [ - proto_path_to_generated_filename( - _strip_package_from_path(label_package, proto.path), - _GRPC_PROTO_SRC_FMT, - ) - for proto in protos - ] - if ctx.attr.generate_mocks: - outs += [ - proto_path_to_generated_filename( - _strip_package_from_path(label_package, proto.path), - _GRPC_PROTO_MOCK_HEADER_FMT, - ) - for proto in protos - ] + # Import protos relative to their workspace root so that protoc prints the + # right include paths. + for include in includes: + directory = include.path + if directory.startswith("external"): + external_sep = directory.find("/") + repository_sep = directory.find("/", external_sep + 1) + arguments += ["--proto_path=" + directory[:repository_sep]] else: - outs += [ - proto_path_to_generated_filename( - _strip_package_from_path(label_package, proto.path), - _PROTO_HEADER_FMT, - ) - for proto in protos - ] - outs += [ - proto_path_to_generated_filename( - _strip_package_from_path(label_package, proto.path), - _PROTO_SRC_FMT, - ) - for proto in protos - ] - out_files = [ctx.actions.declare_file(out) for out in outs] - dir_out = str(ctx.genfiles_dir.path + proto_root) + arguments += ["--proto_path=."] + # Include the output directory so that protoc puts the generated code in the + # right directory. + arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)] + arguments += [proto.path for proto in protos] - arguments = [] - if ctx.executable.plugin: - arguments += get_plugin_args( - ctx.executable.plugin, - ctx.attr.flags, - dir_out, - ctx.attr.generate_mocks, - ) - tools = [ctx.executable.plugin] + # create a list of well known proto files if the argument is non-None + well_known_proto_files = [] + if ctx.attr.well_known_protos: + f = ctx.attr.well_known_protos.files.to_list()[0].dirname + if f != "external/com_google_protobuf/src/google/protobuf": + print("Error: Only @com_google_protobuf//:well_known_protos is supported") else: - arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out] - tools = [] - - arguments += get_include_protoc_args(includes) + # f points to "external/com_google_protobuf/src/google/protobuf" + # add -I argument to protoc so it knows where to look for the proto files. + arguments += ["-I{0}".format(f + "/../..")] + well_known_proto_files = [f for f in ctx.attr.well_known_protos.files] - # Include the output directory so that protoc puts the generated code in the - # right directory. - arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)] - arguments += [proto.path for proto in protos] + ctx.actions.run( + inputs = protos + includes + well_known_proto_files, + tools = tools, + outputs = out_files, + executable = ctx.executable._protoc, + arguments = arguments, + ) - # create a list of well known proto files if the argument is non-None - well_known_proto_files = [] - if ctx.attr.well_known_protos: - f = ctx.attr.well_known_protos.files.to_list()[0].dirname - if f != "external/com_google_protobuf/src/google/protobuf": - print( - "Error: Only @com_google_protobuf//:well_known_protos is supported", - ) - else: - # f points to "external/com_google_protobuf/src/google/protobuf" - # add -I argument to protoc so it knows where to look for the proto files. - arguments += ["-I{0}".format(f + "/../..")] - well_known_proto_files = [ - f - for f in ctx.attr.well_known_protos.files - ] - - ctx.actions.run( - inputs = protos + includes + well_known_proto_files, - tools = tools, - outputs = out_files, - executable = ctx.executable._protoc, - arguments = arguments, - ) - - return struct(files = depset(out_files)) + return struct(files=depset(out_files)) _generate_cc = rule( attrs = { @@ -147,8 +96,10 @@ _generate_cc = rule( mandatory = False, allow_empty = True, ), - "well_known_protos": attr.label(mandatory = False), - "generate_mocks": attr.bool( + "well_known_protos" : attr.label( + mandatory = False, + ), + "generate_mocks" : attr.bool( default = False, mandatory = False, ), @@ -164,10 +115,7 @@ _generate_cc = rule( ) def generate_cc(well_known_protos, **kwargs): - if well_known_protos: - _generate_cc( - well_known_protos = "@com_google_protobuf//:well_known_protos", - **kwargs - ) - else: - _generate_cc(**kwargs) + if well_known_protos: + _generate_cc(well_known_protos="@com_google_protobuf//:well_known_protos", **kwargs) + else: + _generate_cc(**kwargs) diff --git a/bazel/grpc_python_deps.bzl b/bazel/grpc_python_deps.bzl index 91438f3927b..ec3df19e03a 100644 --- a/bazel/grpc_python_deps.bzl +++ b/bazel/grpc_python_deps.bzl @@ -1,8 +1,16 @@ load("//third_party/py:python_configure.bzl", "python_configure") load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") load("@grpc_python_dependencies//:requirements.bzl", "pip_install") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories") def grpc_python_deps(): - python_configure(name = "local_config_python") - pip_repositories() - pip_install() + # TODO(https://github.com/grpc/grpc/issues/18256): Remove conditional. + if hasattr(native, "http_archive"): + python_configure(name = "local_config_python") + pip_repositories() + pip_install() + py_proto_repositories() + else: + print("Building Python gRPC with bazel 23.0+ is disabled pending " + + "resolution of https://github.com/grpc/grpc/issues/18256.") + diff --git a/bazel/protobuf.bzl b/bazel/protobuf.bzl deleted file mode 100644 index bddd0d70c7f..00000000000 --- a/bazel/protobuf.bzl +++ /dev/null @@ -1,84 +0,0 @@ -"""Utility functions for generating protobuf code.""" - -_PROTO_EXTENSION = ".proto" - -def get_proto_root(workspace_root): - """Gets the root protobuf directory. - - Args: - workspace_root: context.label.workspace_root - - Returns: - The directory relative to which generated include paths should be. - """ - if workspace_root: - return "/{}".format(workspace_root) - else: - return "" - -def _strip_proto_extension(proto_filename): - if not proto_filename.endswith(_PROTO_EXTENSION): - fail('"{}" does not end with "{}"'.format( - proto_filename, - _PROTO_EXTENSION, - )) - return proto_filename[:-len(_PROTO_EXTENSION)] - -def proto_path_to_generated_filename(proto_path, fmt_str): - """Calculates the name of a generated file for a protobuf path. - - For example, "examples/protos/helloworld.proto" might map to - "helloworld.pb.h". - - Args: - proto_path: The path to the .proto file. - fmt_str: A format string used to calculate the generated filename. For - example, "{}.pb.h" might be used to calculate a C++ header filename. - - Returns: - The generated filename. - """ - return fmt_str.format(_strip_proto_extension(proto_path)) - -def _get_include_directory(include): - directory = include.path - if directory.startswith("external"): - external_separator = directory.find("/") - repository_separator = directory.find("/", external_separator + 1) - return directory[:repository_separator] - else: - return "." - -def get_include_protoc_args(includes): - """Returns protoc args that imports protos relative to their import root. - - Args: - includes: A list of included proto files. - - Returns: - A list of arguments to be passed to protoc. For example, ["--proto_path=."]. - """ - return [ - "--proto_path={}".format(_get_include_directory(include)) - for include in includes - ] - -def get_plugin_args(plugin, flags, dir_out, generate_mocks): - """Returns arguments configuring protoc to use a plugin for a language. - - Args: - plugin: An executable file to run as the protoc plugin. - flags: The plugin flags to be passed to protoc. - dir_out: The output directory for the plugin. - generate_mocks: A bool indicating whether to generate mocks. - - Returns: - A list of protoc arguments configuring the plugin. - """ - augmented_flags = list(flags) - if generate_mocks: - augmented_flags.append("generate_mock_code=true") - return [ - "--plugin=protoc-gen-PLUGIN=" + plugin.path, - "--PLUGIN_out=" + ",".join(augmented_flags) + ":" + dir_out, - ] diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl deleted file mode 100644 index bf6b4bec8d2..00000000000 --- a/bazel/python_rules.bzl +++ /dev/null @@ -1,203 +0,0 @@ -"""Generates and compiles Python gRPC stubs from proto_library rules.""" - -load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load( - "//bazel:protobuf.bzl", - "get_include_protoc_args", - "get_plugin_args", - "get_proto_root", - "proto_path_to_generated_filename", -) - -_GENERATED_PROTO_FORMAT = "{}_pb2.py" -_GENERATED_GRPC_PROTO_FORMAT = "{}_pb2_grpc.py" - -def _get_staged_proto_file(context, source_file): - if source_file.dirname == context.label.package: - return source_file - else: - copied_proto = context.actions.declare_file(source_file.basename) - context.actions.run_shell( - inputs = [source_file], - outputs = [copied_proto], - command = "cp {} {}".format(source_file.path, copied_proto.path), - mnemonic = "CopySourceProto", - ) - return copied_proto - -def _generate_py_impl(context): - protos = [] - for src in context.attr.deps: - for file in src.proto.direct_sources: - protos.append(_get_staged_proto_file(context, file)) - includes = [ - file - for src in context.attr.deps - for file in src.proto.transitive_imports - ] - proto_root = get_proto_root(context.label.workspace_root) - format_str = (_GENERATED_GRPC_PROTO_FORMAT if context.executable.plugin else _GENERATED_PROTO_FORMAT) - out_files = [ - context.actions.declare_file( - proto_path_to_generated_filename( - proto.basename, - format_str, - ), - ) - for proto in protos - ] - - arguments = [] - tools = [context.executable._protoc] - if context.executable.plugin: - arguments += get_plugin_args( - context.executable.plugin, - context.attr.flags, - context.genfiles_dir.path, - False, - ) - tools += [context.executable.plugin] - else: - arguments += [ - "--python_out={}:{}".format( - ",".join(context.attr.flags), - context.genfiles_dir.path, - ), - ] - - arguments += get_include_protoc_args(includes) - arguments += [ - "--proto_path={}".format(context.genfiles_dir.path) - for proto in protos - ] - for proto in protos: - massaged_path = proto.path - if massaged_path.startswith(context.genfiles_dir.path): - massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] - arguments.append(massaged_path) - - well_known_proto_files = [] - if context.attr.well_known_protos: - well_known_proto_directory = context.attr.well_known_protos.files.to_list( - )[0].dirname - - arguments += ["-I{}".format(well_known_proto_directory + "/../..")] - well_known_proto_files = context.attr.well_known_protos.files.to_list() - - context.actions.run( - inputs = protos + includes + well_known_proto_files, - tools = tools, - outputs = out_files, - executable = context.executable._protoc, - arguments = arguments, - mnemonic = "ProtocInvocation", - ) - return struct(files = depset(out_files)) - -__generate_py = rule( - attrs = { - "deps": attr.label_list( - mandatory = True, - allow_empty = False, - providers = ["proto"], - ), - "plugin": attr.label( - executable = True, - providers = ["files_to_run"], - cfg = "host", - ), - "flags": attr.string_list( - mandatory = False, - allow_empty = True, - ), - "well_known_protos": attr.label(mandatory = False), - "_protoc": attr.label( - default = Label("//external:protocol_compiler"), - executable = True, - cfg = "host", - ), - }, - output_to_genfiles = True, - implementation = _generate_py_impl, -) - -def _generate_py(well_known_protos, **kwargs): - if well_known_protos: - __generate_py( - well_known_protos = "@com_google_protobuf//:well_known_protos", - **kwargs - ) - else: - __generate_py(**kwargs) - -_WELL_KNOWN_PROTO_LIBS = [ - "@com_google_protobuf//:any_proto", - "@com_google_protobuf//:api_proto", - "@com_google_protobuf//:compiler_plugin_proto", - "@com_google_protobuf//:descriptor_proto", - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:empty_proto", - "@com_google_protobuf//:field_mask_proto", - "@com_google_protobuf//:source_context_proto", - "@com_google_protobuf//:struct_proto", - "@com_google_protobuf//:timestamp_proto", - "@com_google_protobuf//:type_proto", - "@com_google_protobuf//:wrappers_proto", -] - -def py_proto_library( - name, - deps, - well_known_protos = True, - proto_only = False, - **kwargs): - """Generate python code for a protobuf. - - Args: - name: The name of the target. - deps: A list of dependencies. Must contain a single element. - well_known_protos: A bool indicating whether or not to include well-known - protos. - proto_only: A bool indicating whether to generate vanilla protobuf code - or to also generate gRPC code. - """ - if len(deps) > 1: - fail("The supported length of 'deps' is 1.") - - codegen_target = "_{}_codegen".format(name) - codegen_grpc_target = "_{}_grpc_codegen".format(name) - - well_known_proto_rules = _WELL_KNOWN_PROTO_LIBS if well_known_protos else [] - - _generate_py( - name = codegen_target, - deps = deps, - well_known_protos = well_known_protos, - **kwargs - ) - - if not proto_only: - _generate_py( - name = codegen_grpc_target, - deps = deps, - plugin = "//:grpc_python_plugin", - well_known_protos = well_known_protos, - **kwargs - ) - - native.py_library( - name = name, - srcs = [ - ":{}".format(codegen_grpc_target), - ":{}".format(codegen_target), - ], - deps = [requirement("protobuf")], - **kwargs - ) - else: - native.py_library( - name = name, - srcs = [":{}".format(codegen_target), ":{}".format(codegen_target)], - deps = [requirement("protobuf")], - **kwargs - ) diff --git a/examples/BUILD b/examples/BUILD index d5fb6903d7c..d2b39b87f4d 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -16,8 +16,9 @@ licenses(["notice"]) # 3-clause BSD package(default_visibility = ["//visibility:public"]) +load("@grpc_python_dependencies//:requirements.bzl", "requirement") load("//bazel:grpc_build_system.bzl", "grpc_proto_library") -load("//bazel:python_rules.bzl", "py_proto_library") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") grpc_proto_library( name = "auth_sample", @@ -44,14 +45,11 @@ grpc_proto_library( srcs = ["protos/keyvaluestore.proto"], ) -proto_library( - name = "helloworld_proto_descriptor", - srcs = ["protos/helloworld.proto"], -) - py_proto_library( name = "py_helloworld", - deps = [":helloworld_proto_descriptor"], + protos = ["protos/helloworld.proto"], + with_grpc = True, + deps = [requirement('protobuf'),], ) cc_binary( diff --git a/examples/python/errors/client.py b/examples/python/errors/client.py index c762d798dc2..a79b8fce1bd 100644 --- a/examples/python/errors/client.py +++ b/examples/python/errors/client.py @@ -20,8 +20,8 @@ import grpc from grpc_status import rpc_status from google.rpc import error_details_pb2 -from examples import helloworld_pb2 -from examples import helloworld_pb2_grpc +from examples.protos import helloworld_pb2 +from examples.protos import helloworld_pb2_grpc _LOGGER = logging.getLogger(__name__) diff --git a/examples/python/errors/server.py b/examples/python/errors/server.py index 50d4a2ac671..f49586b848a 100644 --- a/examples/python/errors/server.py +++ b/examples/python/errors/server.py @@ -24,8 +24,8 @@ from grpc_status import rpc_status from google.protobuf import any_pb2 from google.rpc import code_pb2, status_pb2, error_details_pb2 -from examples import helloworld_pb2 -from examples import helloworld_pb2_grpc +from examples.protos import helloworld_pb2 +from examples.protos import helloworld_pb2_grpc _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/examples/python/errors/test/_error_handling_example_test.py b/examples/python/errors/test/_error_handling_example_test.py index 9eb81ba3742..a79ca45e2a1 100644 --- a/examples/python/errors/test/_error_handling_example_test.py +++ b/examples/python/errors/test/_error_handling_example_test.py @@ -26,7 +26,7 @@ import logging import grpc -from examples import helloworld_pb2_grpc +from examples.protos import helloworld_pb2_grpc from examples.python.errors import client as error_handling_client from examples.python.errors import server as error_handling_server diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index 0e135f471f2..6de1e947d85 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -15,17 +15,12 @@ # limitations under the License. load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py_proto_library") - -proto_library( - name = "prime_proto", - srcs = ["prime.proto"] -) +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") py_proto_library( - name = "prime_proto_pb2", - deps = [":prime_proto"], - well_known_protos = False, + name = "prime_proto", + protos = ["prime.proto",], + deps = [requirement("protobuf")], ) py_binary( @@ -34,7 +29,7 @@ py_binary( srcs = ["client.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":prime_proto_pb2", + ":prime_proto", ], default_python_version = "PY3", ) @@ -45,7 +40,7 @@ py_binary( srcs = ["server.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":prime_proto_pb2" + ":prime_proto" ] + select({ "//conditions:default": [requirement("futures")], "//:python3": [], diff --git a/examples/python/wait_for_ready/wait_for_ready_example.py b/examples/python/wait_for_ready/wait_for_ready_example.py index a0f076e894a..7c16b9bd4a1 100644 --- a/examples/python/wait_for_ready/wait_for_ready_example.py +++ b/examples/python/wait_for_ready/wait_for_ready_example.py @@ -22,8 +22,8 @@ import threading import grpc -from examples import helloworld_pb2 -from examples import helloworld_pb2_grpc +from examples.protos import helloworld_pb2 +from examples.protos import helloworld_pb2_grpc _LOGGER = logging.getLogger(__name__) _LOGGER.setLevel(logging.INFO) @@ -33,13 +33,10 @@ _ONE_DAY_IN_SECONDS = 60 * 60 * 24 @contextmanager def get_free_loopback_tcp_port(): - if socket.has_ipv6: - tcp_socket = socket.socket(socket.AF_INET6) - else: - tcp_socket = socket.socket(socket.AF_INET) + tcp_socket = socket.socket(socket.AF_INET6) tcp_socket.bind(('', 0)) address_tuple = tcp_socket.getsockname() - yield "localhost:%s" % (address_tuple[1]) + yield "[::1]:%s" % (address_tuple[1]) tcp_socket.close() diff --git a/src/proto/grpc/channelz/BUILD b/src/proto/grpc/channelz/BUILD index 1d80ec23af1..b6b485e3e8d 100644 --- a/src/proto/grpc/channelz/BUILD +++ b/src/proto/grpc/channelz/BUILD @@ -25,11 +25,6 @@ grpc_proto_library( well_known_protos = True, ) -proto_library( - name = "channelz_proto_descriptors", - srcs = ["channelz.proto"], -) - filegroup( name = "channelz_proto_file", srcs = [ diff --git a/src/proto/grpc/health/v1/BUILD b/src/proto/grpc/health/v1/BUILD index fc58e8a1770..38a7d992e06 100644 --- a/src/proto/grpc/health/v1/BUILD +++ b/src/proto/grpc/health/v1/BUILD @@ -23,11 +23,6 @@ grpc_proto_library( srcs = ["health.proto"], ) -proto_library( - name = "health_proto_descriptor", - srcs = ["health.proto"], -) - filegroup( name = "health_proto_file", srcs = [ diff --git a/src/proto/grpc/reflection/v1alpha/BUILD b/src/proto/grpc/reflection/v1alpha/BUILD index 5424c0d867e..4d919d029ee 100644 --- a/src/proto/grpc/reflection/v1alpha/BUILD +++ b/src/proto/grpc/reflection/v1alpha/BUILD @@ -23,11 +23,6 @@ grpc_proto_library( srcs = ["reflection.proto"], ) -proto_library( - name = "reflection_proto_descriptor", - srcs = ["reflection.proto"], -) - filegroup( name = "reflection_proto_file", srcs = [ diff --git a/src/proto/grpc/testing/BUILD b/src/proto/grpc/testing/BUILD index 16e47d81061..9876d5160a1 100644 --- a/src/proto/grpc/testing/BUILD +++ b/src/proto/grpc/testing/BUILD @@ -16,7 +16,7 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package") load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py_proto_library") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") grpc_package(name = "testing", visibility = "public") @@ -61,14 +61,13 @@ grpc_proto_library( has_services = False, ) -proto_library( - name = "empty_proto_descriptor", - srcs = ["empty.proto"], -) - py_proto_library( name = "py_empty_proto", - deps = [":empty_proto_descriptor"], + protos = ["empty.proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], ) grpc_proto_library( @@ -77,14 +76,13 @@ grpc_proto_library( has_services = False, ) -proto_library( - name = "messages_proto_descriptor", - srcs = ["messages.proto"], -) - py_proto_library( name = "py_messages_proto", - deps = [":messages_proto_descriptor"], + protos = ["messages.proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], ) grpc_proto_library( @@ -146,19 +144,16 @@ grpc_proto_library( ], ) -proto_library( - name = "test_proto_descriptor", - srcs = ["test.proto"], - deps = [ - ":empty_proto_descriptor", - ":messages_proto_descriptor", - ], -) - py_proto_library( name = "py_test_proto", + protos = ["test.proto",], + with_grpc = True, deps = [ - ":test_proto_descriptor", + requirement('protobuf'), + ], + proto_deps = [ + ":py_empty_proto", + ":py_messages_proto", ] ) diff --git a/src/proto/grpc/testing/proto2/BUILD.bazel b/src/proto/grpc/testing/proto2/BUILD.bazel index e939c523a64..c4c4f004efb 100644 --- a/src/proto/grpc/testing/proto2/BUILD.bazel +++ b/src/proto/grpc/testing/proto2/BUILD.bazel @@ -1,32 +1,30 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) -load("//bazel:python_rules.bzl", "py_proto_library") - -proto_library( - name = "empty2_proto_descriptor", - srcs = ["empty2.proto"], -) py_proto_library( name = "empty2_proto", - deps = [ - ":empty2_proto_descriptor", + protos = [ + "empty2.proto", ], -) - -proto_library( - name = "empty2_extensions_proto_descriptor", - srcs = ["empty2_extensions.proto"], + with_grpc = True, deps = [ - ":empty2_proto_descriptor", - ] + requirement('protobuf'), + ], ) py_proto_library( name = "empty2_extensions_proto", + protos = [ + "empty2_extensions.proto", + ], + proto_deps = [ + ":empty2_proto", + ], + with_grpc = True, deps = [ - ":empty2_extensions_proto_descriptor", + requirement('protobuf'), ], ) diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel index eccc166577d..aae8cedb760 100644 --- a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel +++ b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel @@ -1,10 +1,30 @@ -load("//bazel:python_rules.bzl", "py_proto_library") +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") + package(default_visibility = ["//visibility:public"]) +genrule( + name = "mv_channelz_proto", + srcs = [ + "//src/proto/grpc/channelz:channelz_proto_file", + ], + outs = ["channelz.proto",], + cmd = "cp $< $@", +) + py_proto_library( name = "py_channelz_proto", - well_known_protos = True, - deps = ["//src/proto/grpc/channelz:channelz_proto_descriptors"], + protos = ["mv_channelz_proto",], + imports = [ + "external/com_google_protobuf/src/", + ], + inputs = [ + "@com_google_protobuf//:well_known_protos", + ], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], ) py_library( diff --git a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel index 9e4fff34581..ce3121fa90e 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel +++ b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel @@ -1,9 +1,24 @@ -load("//bazel:python_rules.bzl", "py_proto_library") +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") + package(default_visibility = ["//visibility:public"]) +genrule( + name = "mv_health_proto", + srcs = [ + "//src/proto/grpc/health/v1:health_proto_file", + ], + outs = ["health.proto",], + cmd = "cp $< $@", +) + py_proto_library( name = "py_health_proto", - deps = ["//src/proto/grpc/health/v1:health_proto_descriptor",], + protos = [":mv_health_proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], ) py_library( diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel index 6aaa4dd3bdd..3a2ba263715 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel @@ -1,11 +1,24 @@ -load("//bazel:python_rules.bzl", "py_proto_library") load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) +genrule( + name = "mv_reflection_proto", + srcs = [ + "//src/proto/grpc/reflection/v1alpha:reflection_proto_file", + ], + outs = ["reflection.proto",], + cmd = "cp $< $@", +) + py_proto_library( name = "py_reflection_proto", - deps = ["//src/proto/grpc/reflection/v1alpha:reflection_proto_descriptor",], + protos = [":mv_reflection_proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], ) py_library( diff --git a/src/python/grpcio_tests/tests/reflection/BUILD.bazel b/src/python/grpcio_tests/tests/reflection/BUILD.bazel index b635b721631..c0efb0b7cef 100644 --- a/src/python/grpcio_tests/tests/reflection/BUILD.bazel +++ b/src/python/grpcio_tests/tests/reflection/BUILD.bazel @@ -14,7 +14,6 @@ py_test( "//src/python/grpcio_tests/tests/unit:test_common", "//src/proto/grpc/testing:py_empty_proto", "//src/proto/grpc/testing/proto2:empty2_extensions_proto", - "//src/proto/grpc/testing/proto2:empty2_proto", requirement('protobuf'), ], imports=["../../",], diff --git a/tools/distrib/bazel_style.cfg b/tools/distrib/bazel_style.cfg deleted file mode 100644 index a5a1fea4aba..00000000000 --- a/tools/distrib/bazel_style.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[style] -based_on_style = google -allow_split_before_dict_value = False -spaces_around_default_or_named_assign = True diff --git a/tools/distrib/format_bazel.sh b/tools/distrib/format_bazel.sh deleted file mode 100755 index ee230118efb..00000000000 --- a/tools/distrib/format_bazel.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright 2019 The gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set=-ex - -VIRTUAL_ENV=bazel_format_virtual_environment - -CONFIG_PATH="$(dirname ${0})/bazel_style.cfg" - -python -m virtualenv ${VIRTUAL_ENV} -PYTHON=${VIRTUAL_ENV}/bin/python -"$PYTHON" -m pip install --upgrade pip==10.0.1 -"$PYTHON" -m pip install --upgrade futures -"$PYTHON" -m pip install yapf==0.20.0 - -pushd "$(dirname "${0}")/../.." -FILES=$(find . -path ./third_party -prune -o -name '*.bzl' -print) -echo "${FILES}" | xargs "$PYTHON" -m yapf -i --style="${CONFIG_PATH}" - -if ! which buildifier &>/dev/null; then - echo 'buildifer must be installed.' >/dev/stderr - exit 1 -fi - -echo "${FILES}" | xargs buildifier --type=bzl - -popd diff --git a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh index 24598f43f02..3dd0167b7c0 100755 --- a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh +++ b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh @@ -24,4 +24,5 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') cd /var/local/git/grpc -bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... examples/... +#TODO(yfen): add back examples/... to build targets once python rules issues are resolved +bazel build --spawn_strategy=standalone --genrule_strategy=standalone :all test/... diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index d844cff7f9a..3ca4673ca08 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -23,9 +23,10 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc (cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') -cd /var/local/git/grpc/test -bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... -bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... -bazel clean --expunge -bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... -bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... +#TODO(yfen): temporarily disabled all python bazel tests due to incompatibility with bazel 0.23.2 +#cd /var/local/git/grpc/test +#bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... +#bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... +#bazel clean --expunge +#bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... +#bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/...