From 7b2c8c27b06f92b7d3594ec4f2863e5d759f1760 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 16 Jul 2019 15:20:56 -0700 Subject: [PATCH 01/18] Separate py_grpc_library and py_proto_library. By popular demand, we'll now be offering separate py_grpc_library and py_proto_library targets sharing the same interface as within google3. This change necessitated some modifications to how we pull in our own Python-level dependencies and how we make those available to those pulling in our project via Bazel. There is now a grpc_python_deps() Bazel workspace rule that pulls in the appropriate dependencies, which should be called from the client project's WORKSPACE file. A test has been added to the bazel/test/ directory to verify that this behavior works as intended. It's worth noting that the protobuf repository's usage of Starlark bind() caused a great deal of trouble in ensuring that we could also pull in six. This change also required a change in the way generated proto code is imported in the channelz and health-check modules, as well as in their associated tests. We were importing them two different ways, each relative. This resulted in two different module objects being imported into the process, which were incompatible. I am not sure exactly what caused this behavior to begin, as this should have been possible before this PR. As a workaround, I am simply trying two different absolute imports and using the one that works. This should function both inside and outside of Bazel environments. --- WORKSPACE | 43 ++-- bazel/grpc_python_deps.bzl | 69 +++++- bazel/python_rules.bzl | 218 ++++++++++-------- bazel/test/python_test_repo/.gitignore | 2 + bazel/test/python_test_repo/BUILD | 46 ++++ bazel/test/python_test_repo/WORKSPACE | 10 + bazel/test/python_test_repo/helloworld.proto | 43 ++++ bazel/test/python_test_repo/helloworld.py | 73 ++++++ bazel/test/python_test_repo/tools/bazel | 1 + examples/BUILD | 16 +- examples/python/multiprocessing/BUILD | 18 +- examples/python/wait_for_ready/BUILD.bazel | 3 +- src/proto/grpc/channelz/BUILD | 10 +- src/proto/grpc/testing/BUILD | 31 ++- src/proto/grpc/testing/proto2/BUILD.bazel | 8 +- src/python/grpcio/grpc/BUILD.bazel | 6 +- .../grpcio/grpc/experimental/BUILD.bazel | 1 - .../grpcio/grpc/framework/common/BUILD.bazel | 5 +- .../grpc/framework/foundation/BUILD.bazel | 11 +- .../framework/interfaces/base/BUILD.bazel | 7 +- .../framework/interfaces/face/BUILD.bazel | 5 +- .../grpc_channelz/v1/BUILD.bazel | 16 +- .../grpc_channelz/v1/channelz.py | 8 +- .../grpc_health/v1/BUILD.bazel | 15 +- .../grpc_health/v1/health.py | 9 +- .../grpc_reflection/v1alpha/BUILD.bazel | 17 +- .../grpc_reflection/v1alpha/reflection.py | 8 +- .../grpcio_status/grpc_status/BUILD.bazel | 2 +- .../tests/channelz/_channelz_servicer_test.py | 11 +- .../health_check/_health_servicer_test.py | 15 +- .../grpcio_tests/tests/interop/BUILD.bazel | 7 +- .../grpcio_tests/tests/reflection/BUILD.bazel | 2 +- .../reflection/_reflection_servicer_test.py | 11 +- .../grpcio_tests/tests/unit/BUILD.bazel | 4 +- third_party/BUILD | 3 + third_party/enum34.BUILD | 6 + third_party/futures.BUILD | 6 + third_party/six.BUILD | 6 + .../linux/grpc_python_bazel_test_in_docker.sh | 3 + 39 files changed, 571 insertions(+), 204 deletions(-) create mode 100644 bazel/test/python_test_repo/.gitignore create mode 100644 bazel/test/python_test_repo/BUILD create mode 100644 bazel/test/python_test_repo/WORKSPACE create mode 100644 bazel/test/python_test_repo/helloworld.proto create mode 100644 bazel/test/python_test_repo/helloworld.py create mode 120000 bazel/test/python_test_repo/tools/bazel create mode 100644 third_party/enum34.BUILD create mode 100644 third_party/futures.BUILD create mode 100644 third_party/six.BUILD diff --git a/WORKSPACE b/WORKSPACE index 60582d1a0f5..a9a8cfbba1b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -18,33 +18,6 @@ register_toolchains( "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows", ) -git_repository( - name = "io_bazel_rules_python", - commit = "fdbb17a4118a1728d19e638a5291b4c4266ea5b8", - remote = "https://github.com/bazelbuild/rules_python.git", -) - -load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip_import") - -pip_import( - name = "grpc_python_dependencies", - requirements = "//:requirements.bazel.txt", -) - -http_archive( - name = "cython", - build_file = "//third_party:cython.BUILD", - sha256 = "d68138a2381afbdd0876c3cb2a22389043fa01c4badede1228ee073032b07a27", - strip_prefix = "cython-c2b80d87658a8525ce091cbe146cb7eaa29fed5c", - urls = [ - "https://github.com/cython/cython/archive/c2b80d87658a8525ce091cbe146cb7eaa29fed5c.tar.gz", - ], -) - -load("//bazel:grpc_python_deps.bzl", "grpc_python_deps") - -grpc_python_deps() - load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") # Create toolchain configuration for remote execution. @@ -65,3 +38,19 @@ rbe_autoconfig( }, ), ) + +load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") +grpc_python_deps() + + +load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip_import") + +pip_import( + name = "grpc_python_dependencies", + requirements = "@com_github_grpc_grpc//:requirements.bazel.txt", +) + +load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") +load("@grpc_python_dependencies//:requirements.bzl", "pip_install") +pip_repositories() +pip_install() diff --git a/bazel/grpc_python_deps.bzl b/bazel/grpc_python_deps.bzl index 91438f3927b..4e7cc1537fa 100644 --- a/bazel/grpc_python_deps.bzl +++ b/bazel/grpc_python_deps.bzl @@ -1,8 +1,67 @@ -load("//third_party/py:python_configure.bzl", "python_configure") -load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") -load("@grpc_python_dependencies//:requirements.bzl", "pip_install") +"""Load dependencies needed to compile and test the grpc python library as a 3rd-party consumer.""" + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure") def grpc_python_deps(): + native.bind( + name = "six", + actual = "@six_archive//:six", + ) + + # protobuf binds to the name "six", so we can't use it here. + # See https://github.com/bazelbuild/bazel/issues/1952 for why bind is + # horrible. + if "six_archive" not in native.existing_rules(): + http_archive( + name = "six_archive", + strip_prefix = "six-1.12.0", + build_file = "@com_github_grpc_grpc//third_party:six.BUILD", + sha256 = "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73", + urls = ["https://files.pythonhosted.org/packages/dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca/six-1.12.0.tar.gz"], + ) + + if "enum34" not in native.existing_rules(): + http_archive( + name = "enum34", + build_file = "@com_github_grpc_grpc//third_party:enum34.BUILD", + strip_prefix = "enum34-1.1.6", + sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1", + urls = ["https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"], + ) + + if "futures" not in native.existing_rules(): + http_archive( + name = "futures", + build_file = "@com_github_grpc_grpc//third_party:futures.BUILD", + strip_prefix = "futures-3.3.0", + sha256 = "7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794", + urls = ["https://files.pythonhosted.org/packages/47/04/5fc6c74ad114032cd2c544c575bffc17582295e9cd6a851d6026ab4b2c00/futures-3.3.0.tar.gz"], + ) + + if "io_bazel_rules_python" not in native.existing_rules(): + git_repository( + name = "io_bazel_rules_python", + commit = "fdbb17a4118a1728d19e638a5291b4c4266ea5b8", + remote = "https://github.com/bazelbuild/rules_python.git", + ) + python_configure(name = "local_config_python") - pip_repositories() - pip_install() + + native.bind( + name = "python_headers", + actual = "@local_config_python//:python_headers", + ) + + if "cython" not in native.existing_rules(): + http_archive( + name = "cython", + build_file = "@com_github_grpc_grpc//third_party:cython.BUILD", + sha256 = "d68138a2381afbdd0876c3cb2a22389043fa01c4badede1228ee073032b07a27", + strip_prefix = "cython-c2b80d87658a8525ce091cbe146cb7eaa29fed5c", + urls = [ + "https://github.com/cython/cython/archive/c2b80d87658a8525ce091cbe146cb7eaa29fed5c.tar.gz", + ], + ) + diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index 14550852a4a..eb4ac9c9721 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -1,6 +1,5 @@ """Generates and compiles Python gRPC stubs from proto_library rules.""" -load("@grpc_python_dependencies//:requirements.bzl", "requirement") load( "//bazel:protobuf.bzl", "get_include_protoc_args", @@ -36,56 +35,33 @@ def _generate_py_impl(context): for file in src[ProtoInfo].transitive_imports.to_list() ] proto_root = get_proto_root(context.label.workspace_root) - format_str = (_GENERATED_GRPC_PROTO_FORMAT if context.executable.plugin else _GENERATED_PROTO_FORMAT) out_files = [ context.actions.declare_file( proto_path_to_generated_filename( proto.basename, - format_str, + _GENERATED_PROTO_FORMAT, ), ) for proto in protos ] - arguments = [] tools = [context.executable._protoc] - if context.executable.plugin: - arguments += get_plugin_args( - context.executable.plugin, - context.attr.flags, + arguments = ([ + "--python_out={}".format( context.genfiles_dir.path, - False, - ) - tools += [context.executable.plugin] - else: - arguments += [ - "--python_out={}:{}".format( - ",".join(context.attr.flags), - context.genfiles_dir.path, - ), - ] - - arguments += get_include_protoc_args(includes) - arguments += [ + ), + ] + get_include_protoc_args(includes) + [ "--proto_path={}".format(context.genfiles_dir.path) for proto in protos - ] + ]) for proto in protos: massaged_path = proto.path if massaged_path.startswith(context.genfiles_dir.path): massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] arguments.append(massaged_path) - well_known_proto_files = [] - if context.attr.well_known_protos: - well_known_proto_directory = context.attr.well_known_protos.files.to_list( - )[0].dirname - - arguments += ["-I{}".format(well_known_proto_directory + "/../..")] - well_known_proto_files = context.attr.well_known_protos.files.to_list() - context.actions.run( - inputs = protos + includes + well_known_proto_files, + inputs = protos + includes, tools = tools, outputs = out_files, executable = context.executable._protoc, @@ -94,93 +70,155 @@ def _generate_py_impl(context): ) return struct(files = depset(out_files)) -__generate_py = rule( +_generate_pb2_src = rule( attrs = { "deps": attr.label_list( mandatory = True, allow_empty = False, providers = [ProtoInfo], ), - "plugin": attr.label( - executable = True, - providers = ["files_to_run"], - cfg = "host", - ), - "flags": attr.string_list( - mandatory = False, - allow_empty = True, - ), - "well_known_protos": attr.label(mandatory = False), "_protoc": attr.label( default = Label("//external:protocol_compiler"), + providers = ["files_to_run"], executable = True, cfg = "host", ), }, - output_to_genfiles = True, implementation = _generate_py_impl, ) -def _generate_py(well_known_protos, **kwargs): - if well_known_protos: - __generate_py( - well_known_protos = "@com_google_protobuf//:well_known_protos", - **kwargs - ) - else: - __generate_py(**kwargs) - def py_proto_library( name, - deps, - well_known_protos = True, - proto_only = False, + srcs, **kwargs): """Generate python code for a protobuf. Args: name: The name of the target. - deps: A list of dependencies. Must contain a single element. - well_known_protos: A bool indicating whether or not to include well-known - protos. - proto_only: A bool indicating whether to generate vanilla protobuf code - or to also generate gRPC code. + srcs: A list of proto_library dependencies. Must contain a single element. """ - if len(deps) > 1: - fail("The supported length of 'deps' is 1.") - codegen_target = "_{}_codegen".format(name) - codegen_grpc_target = "_{}_grpc_codegen".format(name) + if len(srcs) > 1: + fail("Can only compile a single proto at a time.") - _generate_py( + + _generate_pb2_src( name = codegen_target, - deps = deps, - well_known_protos = well_known_protos, + deps = srcs, **kwargs ) - if not proto_only: - _generate_py( - name = codegen_grpc_target, - deps = deps, - plugin = "//src/compiler:grpc_python_plugin", - well_known_protos = well_known_protos, - **kwargs - ) + native.py_library( + name = name, + srcs = [":{}".format(codegen_target)], + deps = ["@com_google_protobuf//:protobuf_python"], + **kwargs + ) - native.py_library( - name = name, - srcs = [ - ":{}".format(codegen_grpc_target), - ":{}".format(codegen_target), - ], - deps = [requirement("protobuf")], - **kwargs - ) - else: - native.py_library( - name = name, - srcs = [":{}".format(codegen_target), ":{}".format(codegen_target)], - deps = [requirement("protobuf")], - **kwargs +def _generate_pb2_grpc_src_impl(context): + protos = [] + for src in context.attr.deps: + for file in src[ProtoInfo].direct_sources: + protos.append(_get_staged_proto_file(context, file)) + includes = [ + file + for src in context.attr.deps + for file in src[ProtoInfo].transitive_imports.to_list() + ] + proto_root = get_proto_root(context.label.workspace_root) + out_files = [ + context.actions.declare_file( + proto_path_to_generated_filename( + proto.basename, + _GENERATED_GRPC_PROTO_FORMAT, + ), ) + for proto in protos + ] + + arguments = [] + tools = [context.executable._protoc, context.executable._plugin] + arguments += get_plugin_args( + context.executable._plugin, + [], + context.genfiles_dir.path, + False, + ) + + arguments += get_include_protoc_args(includes) + arguments += [ + "--proto_path={}".format(context.genfiles_dir.path) + for proto in protos + ] + for proto in protos: + massaged_path = proto.path + if massaged_path.startswith(context.genfiles_dir.path): + massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] + arguments.append(massaged_path) + + context.actions.run( + inputs = protos + includes, + tools = tools, + outputs = out_files, + executable = context.executable._protoc, + arguments = arguments, + mnemonic = "ProtocInvocation", + ) + return struct(files = depset(out_files)) + + +_generate_pb2_grpc_src = rule( + attrs = { + "deps": attr.label_list( + mandatory = True, + allow_empty = False, + providers = [ProtoInfo], + ), + "_plugin": attr.label( + executable = True, + providers = ["files_to_run"], + cfg = "host", + default = Label("//src/compiler:grpc_python_plugin"), + ), + "_protoc": attr.label( + executable = True, + providers = ["files_to_run"], + cfg = "host", + default = Label("//external:protocol_compiler"), + ), + }, + implementation = _generate_pb2_grpc_src_impl, +) + +def py_grpc_library( + name, + srcs, + deps, + **kwargs): + """Generate python code for gRPC services defined in a protobuf. + + Args: + name: The name of the target. + srcs: (List of `labels`) a single proto_library target containing the + schema of the service. + deps: (List of `labels`) a single py_proto_library target for the + proto_library in `srcs`. + """ + codegen_grpc_target = "_{}_grpc_codegen".format(name) + if len(srcs) > 1: + fail("Can only compile a single proto at a time.") + + _generate_pb2_grpc_src( + name = codegen_grpc_target, + deps = srcs, + **kwargs + ) + + native.py_library( + name = name, + srcs = [ + ":{}".format(codegen_grpc_target), + ], + deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps, + **kwargs + ) diff --git a/bazel/test/python_test_repo/.gitignore b/bazel/test/python_test_repo/.gitignore new file mode 100644 index 00000000000..7146a250705 --- /dev/null +++ b/bazel/test/python_test_repo/.gitignore @@ -0,0 +1,2 @@ +bazel-* +tools/bazel-* diff --git a/bazel/test/python_test_repo/BUILD b/bazel/test/python_test_repo/BUILD new file mode 100644 index 00000000000..ed4ffd3b74f --- /dev/null +++ b/bazel/test/python_test_repo/BUILD @@ -0,0 +1,46 @@ +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") + +package(default_testonly = 1) + +proto_library( + name = "helloworld_proto", + srcs = ["helloworld.proto"], + deps = [ + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +py_proto_library( + name = "helloworld_py_pb2", + srcs = [":helloworld_proto"], +) + +py_grpc_library( + name = "helloworld_py_pb2_grpc", + srcs = [":helloworld_proto"], + deps = [":helloworld_py_pb2"], +) + +py_proto_library( + name = "duration_py_pb2", + srcs = ["@com_google_protobuf//:duration_proto"], +) + +py_proto_library( + name = "timestamp_py_pb2", + srcs = ["@com_google_protobuf//:timestamp_proto"], +) + +py_test( + name = "import_test", + main = "helloworld.py", + srcs = ["helloworld.py"], + deps = [ + ":helloworld_py_pb2", + ":helloworld_py_pb2_grpc", + ":duration_py_pb2", + ":timestamp_py_pb2", + ], + python_version = "PY3", +) diff --git a/bazel/test/python_test_repo/WORKSPACE b/bazel/test/python_test_repo/WORKSPACE new file mode 100644 index 00000000000..69d34ac933c --- /dev/null +++ b/bazel/test/python_test_repo/WORKSPACE @@ -0,0 +1,10 @@ +local_repository( + name = "com_github_grpc_grpc", + path = "../../..", +) + +load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") +grpc_deps() + +load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") +grpc_python_deps() diff --git a/bazel/test/python_test_repo/helloworld.proto b/bazel/test/python_test_repo/helloworld.proto new file mode 100644 index 00000000000..b333a7409b8 --- /dev/null +++ b/bazel/test/python_test_repo/helloworld.proto @@ -0,0 +1,43 @@ +// Copyright 2019 The gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.helloworld"; +option java_outer_classname = "HelloWorldProto"; +option objc_class_prefix = "HLW"; + +package helloworld; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; + google.protobuf.Timestamp request_initiation = 2; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; + google.protobuf.Duration request_duration = 2; +} diff --git a/bazel/test/python_test_repo/helloworld.py b/bazel/test/python_test_repo/helloworld.py new file mode 100644 index 00000000000..deee36a8f71 --- /dev/null +++ b/bazel/test/python_test_repo/helloworld.py @@ -0,0 +1,73 @@ +# Copyright 2019 the gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Python implementation of the GRPC helloworld.Greeter client.""" + +import contextlib +import datetime +import logging +import unittest + +import grpc + +import duration_pb2 +import helloworld_pb2 +import helloworld_pb2_grpc + +_HOST = 'localhost' +_SERVER_ADDRESS = '{}:0'.format(_HOST) + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, context): + request_in_flight = datetime.now() - request.request_initation.ToDatetime() + request_duration = duration_pb2.Duration() + request_duration.FromTimedelta(request_in_flight) + return helloworld_pb2.HelloReply( + message='Hello, %s!' % request.name, + request_duration=request_duration, + ) + + +@contextlib.contextmanager +def _listening_server(): + server = grpc.server(futures.ThreadPoolExecutor()) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + port = server.add_insecure_port(_SERVER_ADDRESS) + server.start() + try: + yield port + finally: + server.stop(0) + + +class ImportTest(unittest.TestCase): + def run(): + with _listening_server() as port: + with grpc.insecure_channel('{}:{}'.format(_HOST, port)) as channel: + stub = helloworld_pb2_grpc.GreeterStub(channel) + request_timestamp = timestamp_pb2.Timestamp() + request_timestamp.GetCurrentTime() + response = stub.SayHello(helloworld_pb2.HelloRequest( + name='you', + request_initiation=request_timestamp, + ), + wait_for_ready=True) + self.assertEqual(response.message, "Hello, you!") + self.assertGreater(response.request_duration.microseconds, 0) + + +if __name__ == '__main__': + logging.basicConfig() + unittest.main() diff --git a/bazel/test/python_test_repo/tools/bazel b/bazel/test/python_test_repo/tools/bazel new file mode 120000 index 00000000000..ad4a1267bcd --- /dev/null +++ b/bazel/test/python_test_repo/tools/bazel @@ -0,0 +1 @@ +../../../../tools/bazel \ No newline at end of file diff --git a/examples/BUILD b/examples/BUILD index a9dd94902a4..b3c5d0a825c 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -18,7 +18,8 @@ package(default_visibility = ["//visibility:public"]) load("//bazel:grpc_build_system.bzl", "grpc_proto_library") load("//bazel:cc_grpc_library.bzl", "cc_grpc_library") -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") +load("@grpc_python_dependencies//:requirements.bzl", "requirement") grpc_proto_library( name = "auth_sample", @@ -60,13 +61,20 @@ grpc_proto_library( ) proto_library( - name = "helloworld_proto_descriptor", + name = "protos/helloworld_proto", srcs = ["protos/helloworld.proto"], ) py_proto_library( - name = "py_helloworld", - deps = [":helloworld_proto_descriptor"], + name = "helloworld_py_pb2", + srcs = [":protos/helloworld_proto"], + deps = [requirement('protobuf')], +) + +py_grpc_library( + name = "helloworld_py_pb2_grpc", + srcs = [":protos/helloworld_proto"], + deps = [":helloworld_py_pb2"], ) cc_binary( diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index f9fa6598068..257523f14bb 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") proto_library( name = "prime_proto", @@ -24,8 +23,13 @@ proto_library( py_proto_library( name = "prime_proto_pb2", - deps = [":prime_proto"], - well_known_protos = False, + srcs = [":prime_proto"], +) + +py_grpc_library( + name = "prime_proto_pb2_grpc", + srcs = [":prime_proto"], + deps = [":prime_proto_pb2"], ) py_binary( @@ -35,6 +39,7 @@ py_binary( deps = [ "//src/python/grpcio/grpc:grpcio", ":prime_proto_pb2", + ":prime_proto_pb2_grpc", ], srcs_version = "PY3", ) @@ -45,9 +50,10 @@ py_binary( srcs = ["server.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":prime_proto_pb2" + ":prime_proto_pb2", + ":prime_proto_pb2_grpc", ] + select({ - "//conditions:default": [requirement("futures")], + "//conditions:default": ["@futures//:futures"], "//:python3": [], }), srcs_version = "PY3", diff --git a/examples/python/wait_for_ready/BUILD.bazel b/examples/python/wait_for_ready/BUILD.bazel index 70daf83d334..f074ae7bb7f 100644 --- a/examples/python/wait_for_ready/BUILD.bazel +++ b/examples/python/wait_for_ready/BUILD.bazel @@ -20,7 +20,8 @@ py_library( srcs = ["wait_for_ready_example.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], ) diff --git a/src/proto/grpc/channelz/BUILD b/src/proto/grpc/channelz/BUILD index 1d80ec23af1..5d1ec5a60cb 100644 --- a/src/proto/grpc/channelz/BUILD +++ b/src/proto/grpc/channelz/BUILD @@ -27,7 +27,15 @@ grpc_proto_library( proto_library( name = "channelz_proto_descriptors", - srcs = ["channelz.proto"], + srcs = [ + "channelz.proto", + ], + deps = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:wrappers_proto", + ], ) filegroup( diff --git a/src/proto/grpc/testing/BUILD b/src/proto/grpc/testing/BUILD index 212f0f3cca7..15be8227de1 100644 --- a/src/proto/grpc/testing/BUILD +++ b/src/proto/grpc/testing/BUILD @@ -16,7 +16,7 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package") load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") grpc_package( name = "testing", @@ -75,8 +75,14 @@ proto_library( ) py_proto_library( - name = "py_empty_proto", - deps = [":empty_proto_descriptor"], + name = "empty_py_pb2", + srcs = [":empty_proto_descriptor"], +) + +py_grpc_library( + name = "empty_py_pb2_grpc", + srcs = [":empty_proto_descriptor"], + deps = [":empty_py_pb2"], ) grpc_proto_library( @@ -92,7 +98,13 @@ proto_library( py_proto_library( name = "py_messages_proto", - deps = [":messages_proto_descriptor"], + srcs = [":messages_proto_descriptor"], +) + +py_grpc_library( + name = "messages_py_pb2_grpc", + srcs = [":messages_proto_descriptor"], + deps = [":py_messages_proto"], ) grpc_proto_library( @@ -196,7 +208,12 @@ proto_library( py_proto_library( name = "py_test_proto", - deps = [ - ":test_proto_descriptor", - ], + srcs = [":test_proto_descriptor"], +) + +py_grpc_library( + name = "test_py_pb2_grpc", + srcs = [":test_proto_descriptor"], + deps = [":py_test_proto"], ) + diff --git a/src/proto/grpc/testing/proto2/BUILD.bazel b/src/proto/grpc/testing/proto2/BUILD.bazel index e939c523a64..668d0a484ea 100644 --- a/src/proto/grpc/testing/proto2/BUILD.bazel +++ b/src/proto/grpc/testing/proto2/BUILD.bazel @@ -10,9 +10,7 @@ proto_library( py_proto_library( name = "empty2_proto", - deps = [ - ":empty2_proto_descriptor", - ], + srcs = [":empty2_proto_descriptor"], ) proto_library( @@ -25,8 +23,6 @@ proto_library( py_proto_library( name = "empty2_extensions_proto", - deps = [ - ":empty2_extensions_proto_descriptor", - ], + srcs = [":empty2_extensions_proto_descriptor"], ) diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel index a2bedae4bea..1b5a018249a 100644 --- a/src/python/grpcio/grpc/BUILD.bazel +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -1,5 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") - package(default_visibility = ["//visibility:public"]) py_library( @@ -16,9 +14,9 @@ py_library( "//src/python/grpcio/grpc/_cython:cygrpc", "//src/python/grpcio/grpc/experimental", "//src/python/grpcio/grpc/framework", - requirement('six'), + "@six_archive//:six", ] + select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), data = [ diff --git a/src/python/grpcio/grpc/experimental/BUILD.bazel b/src/python/grpcio/grpc/experimental/BUILD.bazel index 6598d02747b..cd8afe533b6 100644 --- a/src/python/grpcio/grpc/experimental/BUILD.bazel +++ b/src/python/grpcio/grpc/experimental/BUILD.bazel @@ -1,4 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) py_library( diff --git a/src/python/grpcio/grpc/framework/common/BUILD.bazel b/src/python/grpcio/grpc/framework/common/BUILD.bazel index 52fbb2b516c..c0d1486d537 100644 --- a/src/python/grpcio/grpc/framework/common/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/common/BUILD.bazel @@ -1,4 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) py_library( @@ -14,7 +13,7 @@ py_library( name = "cardinality", srcs = ["cardinality.py"], deps = select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), ) @@ -23,7 +22,7 @@ py_library( name = "style", srcs = ["style.py"], deps = select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), ) diff --git a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel index a447ecded49..b539fa46da7 100644 --- a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel @@ -1,4 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) py_library( @@ -23,9 +22,9 @@ py_library( name = "callable_util", srcs = ["callable_util.py"], deps = [ - requirement("six"), + "//external:six", ] + select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), ) @@ -34,7 +33,7 @@ py_library( name = "future", srcs = ["future.py"], deps = [ - requirement("six"), + "//external:six", ], ) @@ -42,7 +41,7 @@ py_library( name = "logging_pool", srcs = ["logging_pool.py"], deps = select({ - "//conditions:default": [requirement('futures'),], + "//conditions:default": ["@futures//:futures",], "//:python3": [], }), ) @@ -59,6 +58,6 @@ py_library( name = "stream", srcs = ["stream.py"], deps = [ - requirement("six"), + "//external:six", ], ) diff --git a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel index 35cfe877f34..5d0c06950a8 100644 --- a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel @@ -1,4 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) py_library( @@ -15,9 +14,9 @@ py_library( srcs = ["base.py"], deps = [ "//src/python/grpcio/grpc/framework/foundation:abandonment", - requirement("six"), + "//external:six", ] + select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), ) @@ -26,7 +25,7 @@ py_library( name = "utilities", srcs = ["utilities.py"], deps = select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), ) diff --git a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel index 83fadb6372e..3af1404eade 100644 --- a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel @@ -1,4 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) py_library( @@ -16,9 +15,9 @@ py_library( deps = [ "//src/python/grpcio/grpc/framework/foundation", "//src/python/grpcio/grpc/framework/common", - requirement("six"), + "//external:six", ] + select({ - "//conditions:default": [requirement('enum34'),], + "//conditions:default": ["@enum34//:enum34",], "//:python3": [], }), ) diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel index eccc166577d..c0a7e3d3026 100644 --- a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel +++ b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel @@ -1,17 +1,23 @@ -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") package(default_visibility = ["//visibility:public"]) py_proto_library( - name = "py_channelz_proto", - well_known_protos = True, - deps = ["//src/proto/grpc/channelz:channelz_proto_descriptors"], + name = "channelz_py_pb2", + srcs = ["//src/proto/grpc/channelz:channelz_proto_descriptors"], +) + +py_grpc_library( + name = "channelz_py_pb2_grpc", + srcs = ["//src/proto/grpc/channelz:channelz_proto_descriptors"], + deps = [":channelz_py_pb2"], ) py_library( name = "grpc_channelz", srcs = ["channelz.py",], deps = [ - ":py_channelz_proto", + ":channelz_py_pb2", + ":channelz_py_pb2_grpc", "//src/python/grpcio/grpc:grpcio", ], imports=["../../",], diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py b/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py index 00eca311dc1..18810cab290 100644 --- a/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py +++ b/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py @@ -16,8 +16,12 @@ import grpc from grpc._cython import cygrpc -import grpc_channelz.v1.channelz_pb2 as _channelz_pb2 -import grpc_channelz.v1.channelz_pb2_grpc as _channelz_pb2_grpc +try: + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2 as _channelz_pb2 + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2_grpc as _channelz_pb2_grpc +except ImportError: + import grpc_channelz.v1.channelz_pb2 as _channelz_pb2 + import grpc_channelz.v1.channelz_pb2_grpc as _channelz_pb2_grpc from google.protobuf import json_format diff --git a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel index 9e4fff34581..0ee176f0506 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel +++ b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel @@ -1,16 +1,23 @@ -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") package(default_visibility = ["//visibility:public"]) py_proto_library( - name = "py_health_proto", - deps = ["//src/proto/grpc/health/v1:health_proto_descriptor",], + name = "health_py_pb2", + srcs = ["//src/proto/grpc/health/v1:health_proto_descriptor",], +) + +py_grpc_library( + name = "health_py_pb2_grpc", + srcs = ["//src/proto/grpc/health/v1:health_proto_descriptor",], + deps = [":health_py_pb2"], ) py_library( name = "grpc_health", srcs = ["health.py",], deps = [ - ":py_health_proto", + ":health_py_pb2", + ":health_py_pb2_grpc", "//src/python/grpcio/grpc:grpcio", ], imports=["../../",], diff --git a/src/python/grpcio_health_checking/grpc_health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/v1/health.py index 15494fafdbc..93c1895df4c 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/health.py +++ b/src/python/grpcio_health_checking/grpc_health/v1/health.py @@ -18,8 +18,13 @@ import threading import grpc -from grpc_health.v1 import health_pb2 as _health_pb2 -from grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc +# Import using an absolute path to ensure no duplicate loaded modules. +try: + from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2 as _health_pb2 + from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc +except ImportError: + from grpc_health.v1 import health_pb2 as _health_pb2 + from grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc SERVICE_NAME = _health_pb2.DESCRIPTOR.services_by_name['Health'].full_name diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel index 6aaa4dd3bdd..49bc517caf1 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel @@ -1,20 +1,27 @@ -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) py_proto_library( - name = "py_reflection_proto", - deps = ["//src/proto/grpc/reflection/v1alpha:reflection_proto_descriptor",], + name = "reflection_py_pb2", + srcs = ["//src/proto/grpc/reflection/v1alpha:reflection_proto_descriptor",], +) + +py_grpc_library( + name = "reflection_py_pb2_grpc", + srcs = ["//src/proto/grpc/reflection/v1alpha:reflection_proto_descriptor",], + deps = ["reflection_py_pb2"], ) py_library( name = "grpc_reflection", srcs = ["reflection.py",], deps = [ - ":py_reflection_proto", + ":reflection_py_pb2", + ":reflection_py_pb2_grpc", "//src/python/grpcio/grpc:grpcio", - requirement('protobuf'), + "@com_google_protobuf//:protobuf_python", ], imports=["../../",], ) diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index 6df1a364269..70c506c1c7d 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -17,8 +17,12 @@ import grpc from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool -from grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 -from grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc +try: + from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 + from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc +except ImportError: + from grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 + from grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc _POOL = descriptor_pool.Default() SERVICE_NAME = _reflection_pb2.DESCRIPTOR.services_by_name[ diff --git a/src/python/grpcio_status/grpc_status/BUILD.bazel b/src/python/grpcio_status/grpc_status/BUILD.bazel index 223a077c3f2..a7d47c9f34e 100644 --- a/src/python/grpcio_status/grpc_status/BUILD.bazel +++ b/src/python/grpcio_status/grpc_status/BUILD.bazel @@ -7,7 +7,7 @@ py_library( srcs = ["rpc_status.py",], deps = [ "//src/python/grpcio/grpc:grpcio", - requirement('protobuf'), + "@com_google_protobuf//:protobuf_python", requirement('googleapis-common-protos'), ], imports=["../",], diff --git a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py index 5265911a0be..bfe184107ec 100644 --- a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py +++ b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py @@ -18,9 +18,14 @@ import unittest from concurrent import futures import grpc -from grpc_channelz.v1 import channelz -from grpc_channelz.v1 import channelz_pb2 -from grpc_channelz.v1 import channelz_pb2_grpc +try: + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2 + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2_grpc +except ImportError: + from grpc_channelz.v1 import channelz + from grpc_channelz.v1 import channelz_pb2 + from grpc_channelz.v1 import channelz_pb2_grpc from tests.unit import test_common from tests.unit.framework.common import test_constants diff --git a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py index 7a332b8390f..91e841a4d60 100644 --- a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py +++ b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py @@ -13,14 +13,22 @@ # limitations under the License. """Tests of grpc_health.v1.health.""" +import logging import threading import time import unittest import grpc -from grpc_health.v1 import health -from grpc_health.v1 import health_pb2 -from grpc_health.v1 import health_pb2_grpc + +# Import using an absolute path to ensure no duplicate loaded modules. +try: + from src.python.grpcio_health_checking.grpc_health.v1 import health + from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2 + from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2_grpc +except ImportError: + from grpc_health.v1 import health + from grpc_health.v1 import health_pb2 + from grpc_health.v1 import health_pb2_grpc from tests.unit import test_common from tests.unit import thread_pool @@ -276,4 +284,5 @@ class HealthServicerBackwardsCompatibleWatchTest(BaseWatchTests.WatchTests): if __name__ == '__main__': + logging.basicConfig() unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel index fd636556074..43fdc7ab462 100644 --- a/src/python/grpcio_tests/tests/interop/BUILD.bazel +++ b/src/python/grpcio_tests/tests/interop/BUILD.bazel @@ -31,9 +31,12 @@ py_library( deps = [ "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_tests/tests:bazel_namespace_package_hack", - "//src/proto/grpc/testing:py_empty_proto", + "//src/proto/grpc/testing:empty_py_pb2", + "//src/proto/grpc/testing:empty_py_pb2_grpc", "//src/proto/grpc/testing:py_messages_proto", + "//src/proto/grpc/testing:messages_py_pb2_grpc", "//src/proto/grpc/testing:py_test_proto", + "//src/proto/grpc/testing:test_py_pb2_grpc", requirement("google-auth"), requirement("requests"), requirement("urllib3"), @@ -59,7 +62,7 @@ py_library( srcs = ["service.py"], imports = ["../../"], deps = [ - "//src/proto/grpc/testing:py_empty_proto", + "//src/proto/grpc/testing:empty_py_pb2", "//src/proto/grpc/testing:py_messages_proto", "//src/proto/grpc/testing:py_test_proto", "//src/python/grpcio/grpc:grpcio", diff --git a/src/python/grpcio_tests/tests/reflection/BUILD.bazel b/src/python/grpcio_tests/tests/reflection/BUILD.bazel index b635b721631..e9b56191df8 100644 --- a/src/python/grpcio_tests/tests/reflection/BUILD.bazel +++ b/src/python/grpcio_tests/tests/reflection/BUILD.bazel @@ -12,7 +12,7 @@ py_test( "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_reflection/grpc_reflection/v1alpha:grpc_reflection", "//src/python/grpcio_tests/tests/unit:test_common", - "//src/proto/grpc/testing:py_empty_proto", + "//src/proto/grpc/testing:empty_py_pb2", "//src/proto/grpc/testing/proto2:empty2_extensions_proto", "//src/proto/grpc/testing/proto2:empty2_proto", requirement('protobuf'), diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 29bb292c913..1a1fdc83163 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -16,9 +16,14 @@ import unittest import grpc -from grpc_reflection.v1alpha import reflection -from grpc_reflection.v1alpha import reflection_pb2 -from grpc_reflection.v1alpha import reflection_pb2_grpc +try: + from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection + from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2 + from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2_grpc +except ImportError: + from grpc_reflection.v1alpha import reflection + from grpc_reflection.v1alpha import reflection_pb2 + from grpc_reflection.v1alpha import reflection_pb2_grpc from google.protobuf import descriptor_pool from google.protobuf import descriptor_pb2 diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index a161794f8be..cf4350a6beb 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -1,5 +1,3 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") - package(default_visibility = ["//visibility:public"]) GRPCIO_TESTS_UNIT = [ @@ -89,7 +87,7 @@ py_library( ":_tcp_proxy", "//src/python/grpcio_tests/tests/unit/framework/common", "//src/python/grpcio_tests/tests/testing", - requirement('six'), + "//external:six" ], imports=["../../",], data=[ diff --git a/third_party/BUILD b/third_party/BUILD index 8b43d6b8300..cd4a6dad72b 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -8,4 +8,7 @@ exports_files([ "zope_interface.BUILD", "constantly.BUILD", "cython.BUILD", + "six.BUILD", + "enum34.BUILD", + "futures.BUILD", ]) diff --git a/third_party/enum34.BUILD b/third_party/enum34.BUILD new file mode 100644 index 00000000000..3fc2b3de571 --- /dev/null +++ b/third_party/enum34.BUILD @@ -0,0 +1,6 @@ +py_library( + name = "enum34", + srcs = ["enum/__init__.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/futures.BUILD b/third_party/futures.BUILD new file mode 100644 index 00000000000..c280e1edfd7 --- /dev/null +++ b/third_party/futures.BUILD @@ -0,0 +1,6 @@ +py_library( + name = "futures", + srcs = glob(["concurrent/**/*.py"]), + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/six.BUILD b/third_party/six.BUILD new file mode 100644 index 00000000000..60dc7811492 --- /dev/null +++ b/third_party/six.BUILD @@ -0,0 +1,6 @@ +py_library( + name = "six", + srcs = ["six.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index d844cff7f9a..f33fd0f4f55 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -29,3 +29,6 @@ bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_outp bazel clean --expunge bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... +(cd bazel/test/python_test_repo; + bazel test --test_output=errors //... +) From 2b7ec3ad2300266c05a50786486003eb5afba9c7 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 1 Aug 2019 16:09:25 -0700 Subject: [PATCH 02/18] Fix up examples. --- examples/BUILD | 1 - examples/python/auth/BUILD.bazel | 9 +++++---- examples/python/compression/BUILD.bazel | 6 ++++-- examples/python/debug/BUILD.bazel | 11 +++++++---- examples/python/debug/get_stats.py | 9 +++++++-- examples/python/errors/BUILD.bazel | 6 ++++-- 6 files changed, 27 insertions(+), 15 deletions(-) diff --git a/examples/BUILD b/examples/BUILD index b3c5d0a825c..aa9ed945849 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -68,7 +68,6 @@ proto_library( py_proto_library( name = "helloworld_py_pb2", srcs = [":protos/helloworld_proto"], - deps = [requirement('protobuf')], ) py_grpc_library( diff --git a/examples/python/auth/BUILD.bazel b/examples/python/auth/BUILD.bazel index 7bb4203f93f..cc454fdfdfe 100644 --- a/examples/python/auth/BUILD.bazel +++ b/examples/python/auth/BUILD.bazel @@ -36,7 +36,8 @@ py_binary( deps = [ ":_credentials", "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], ) @@ -47,8 +48,8 @@ py_binary( deps = [ ":_credentials", "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", - + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], ) @@ -57,7 +58,7 @@ py_test( srcs = ["test/_auth_example_test.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", ":customized_auth_client", ":customized_auth_server", ":_credentials", diff --git a/examples/python/compression/BUILD.bazel b/examples/python/compression/BUILD.bazel index b95d7cccc77..9d5f6bb83ed 100644 --- a/examples/python/compression/BUILD.bazel +++ b/examples/python/compression/BUILD.bazel @@ -17,7 +17,8 @@ py_binary( srcs = ["server.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], srcs_version = "PY2AND3", ) @@ -27,7 +28,8 @@ py_binary( srcs = ["client.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], srcs_version = "PY2AND3", ) diff --git a/examples/python/debug/BUILD.bazel b/examples/python/debug/BUILD.bazel index 0134d8675ed..657ae860ae3 100644 --- a/examples/python/debug/BUILD.bazel +++ b/examples/python/debug/BUILD.bazel @@ -21,7 +21,8 @@ py_binary( deps = [ "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], ) @@ -31,7 +32,8 @@ py_binary( srcs = ["send_message.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ], ) @@ -51,9 +53,10 @@ py_test( deps = [ "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ":debug_server", ":send_message", ":get_stats", ], -) \ No newline at end of file +) diff --git a/examples/python/debug/get_stats.py b/examples/python/debug/get_stats.py index d0b7061c54f..6eb593deb22 100644 --- a/examples/python/debug/get_stats.py +++ b/examples/python/debug/get_stats.py @@ -20,8 +20,13 @@ from __future__ import print_function import logging import argparse import grpc -from grpc_channelz.v1 import channelz_pb2 -from grpc_channelz.v1 import channelz_pb2_grpc + +try: + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2 + from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2_grpc +except ImportError: + from grpc_channelz.v1 import channelz_pb2 + from grpc_channelz.v1 import channelz_pb2_grpc def run(addr): diff --git a/examples/python/errors/BUILD.bazel b/examples/python/errors/BUILD.bazel index b07dd12ebd3..4b779ddfcf1 100644 --- a/examples/python/errors/BUILD.bazel +++ b/examples/python/errors/BUILD.bazel @@ -21,7 +21,8 @@ py_library( deps = [ "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_status/grpc_status:grpc_status", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", requirement('googleapis-common-protos'), ], ) @@ -33,7 +34,8 @@ py_library( deps = [ "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_status/grpc_status:grpc_status", - "//examples:py_helloworld", + "//examples:helloworld_py_pb2", + "//examples:helloworld_py_pb2_grpc", ] + select({ "//conditions:default": [requirement("futures")], "//:python3": [], From e64961d658d9e0de870ce37fed7154d35d57f5c1 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 1 Aug 2019 16:22:02 -0700 Subject: [PATCH 03/18] Shorten long lines. --- .../grpcio_reflection/grpc_reflection/v1alpha/reflection.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index 70c506c1c7d..bc76136cc5a 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -18,8 +18,10 @@ from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool try: - from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 - from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc + from src.python.grpcio_reflection.grpc_reflection.v1alpha \ + import reflection_pb2 as _reflection_pb2 + from src.python.grpcio_reflection.grpc_reflection.v1alpha \ + import reflection_pb2_grpc as _reflection_pb2_grpc except ImportError: from grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 from grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc From 6954030175eab57951b721ffbf2bebaf94dbc9b1 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 1 Aug 2019 16:40:26 -0700 Subject: [PATCH 04/18] Cd to an absolute path. --- tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index f33fd0f4f55..eda730d3a54 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -29,6 +29,6 @@ bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_outp bazel clean --expunge bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... -(cd bazel/test/python_test_repo; +(cd /var/local/git/grpc/bazel/test/python_test_repo; bazel test --test_output=errors //... ) From 835b3d32930cd6e179b8c244837d2b49477cce01 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 1 Aug 2019 17:11:56 -0700 Subject: [PATCH 05/18] Copyright. --- bazel/test/python_test_repo/BUILD | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bazel/test/python_test_repo/BUILD b/bazel/test/python_test_repo/BUILD index ed4ffd3b74f..0df3700ec93 100644 --- a/bazel/test/python_test_repo/BUILD +++ b/bazel/test/python_test_repo/BUILD @@ -1,3 +1,19 @@ +# gRPC Bazel BUILD file. +# +# Copyright 2019 The gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") package(default_testonly = 1) From f054bd73f48111536be316631cc7ef3d57b8c837 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 2 Aug 2019 14:47:18 -0700 Subject: [PATCH 06/18] Compensate for missing upb dependencies --- bazel/test/python_test_repo/WORKSPACE | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bazel/test/python_test_repo/WORKSPACE b/bazel/test/python_test_repo/WORKSPACE index 69d34ac933c..29bec643f96 100644 --- a/bazel/test/python_test_repo/WORKSPACE +++ b/bazel/test/python_test_repo/WORKSPACE @@ -8,3 +8,9 @@ grpc_deps() load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") grpc_python_deps() + + +# TODO(https://github.com/grpc/grpc/issues/19835): Remove. +load("@upb//bazel:workspace_deps.bzl", "upb_deps") +upb_deps() + From 66114201b45f805278c632906c7bc0366482b07f Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 2 Aug 2019 15:11:06 -0700 Subject: [PATCH 07/18] Modernize cancellation example --- examples/python/cancellation/BUILD.bazel | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/python/cancellation/BUILD.bazel b/examples/python/cancellation/BUILD.bazel index 81cd3a881b8..41b394a07a2 100644 --- a/examples/python/cancellation/BUILD.bazel +++ b/examples/python/cancellation/BUILD.bazel @@ -15,7 +15,7 @@ # limitations under the License. load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py_proto_library") +load("//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") package(default_testonly = 1) @@ -25,9 +25,14 @@ proto_library( ) py_proto_library( - name = "hash_name_proto_pb2", - deps = [":hash_name_proto"], - well_known_protos = False, + name = "hash_name_py_pb2", + srcs = [":hash_name_proto"], +) + +py_grpc_library( + name = "hash_name_py_pb2_grpc", + srcs = [":hash_name_proto"], + deps = [":hash_name_py_pb2"], ) py_binary( @@ -35,8 +40,9 @@ py_binary( srcs = ["client.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":hash_name_proto_pb2", - requirement("six"), + ":hash_name_py_pb2", + ":hash_name_py_pb2_grpc", + "//external:six" ], srcs_version = "PY2AND3", ) @@ -46,7 +52,7 @@ py_library( srcs = ["search.py"], srcs_version = "PY2AND3", deps = [ - ":hash_name_proto_pb2", + ":hash_name_py_pb2", ], ) @@ -55,10 +61,10 @@ py_binary( srcs = ["server.py"], deps = [ "//src/python/grpcio/grpc:grpcio", - ":hash_name_proto_pb2", + ":hash_name_py_pb2", ":search", ] + select({ - "//conditions:default": [requirement("futures")], + "//conditions:default": ["@futures//:futures"], "//:python3": [], }), srcs_version = "PY2AND3", From e89096cdf2b6dab56f5c1e910ac3c38361886dcd Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 2 Aug 2019 16:04:59 -0700 Subject: [PATCH 08/18] AAAAAAAHHHHHH --- bazel/test/python_test_repo/WORKSPACE | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bazel/test/python_test_repo/WORKSPACE b/bazel/test/python_test_repo/WORKSPACE index 29bec643f96..0bc58045693 100644 --- a/bazel/test/python_test_repo/WORKSPACE +++ b/bazel/test/python_test_repo/WORKSPACE @@ -9,8 +9,12 @@ grpc_deps() load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") grpc_python_deps() - # TODO(https://github.com/grpc/grpc/issues/19835): Remove. load("@upb//bazel:workspace_deps.bzl", "upb_deps") upb_deps() +load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") +apple_rules_dependencies() + +load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") +apple_support_dependencies() From 3638167e4092cb72b45c07888fa81a6a2016ce90 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 5 Aug 2019 12:57:50 -0700 Subject: [PATCH 09/18] Call grpc_python_deps() from grpc_deps(). --- WORKSPACE | 6 +----- bazel/grpc_deps.bzl | 8 +++++++- bazel/test/python_test_repo/WORKSPACE | 3 --- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 3f7911b0a22..523b0153d7b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -39,10 +39,6 @@ rbe_autoconfig( ), ) -load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") -grpc_python_deps() - - load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip_import") pip_import( @@ -70,4 +66,4 @@ load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependenci apple_rules_dependencies() load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") -apple_support_dependencies() \ No newline at end of file +apple_support_dependencies() diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index cce2f88fe87..f9e84065b73 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -2,6 +2,8 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") + def grpc_deps(): """Loads dependencies need to compile and test the grpc library.""" @@ -231,7 +233,10 @@ def grpc_deps(): remote = "https://github.com/bazelbuild/rules_apple.git", tag = "0.17.2", ) - + + grpc_python_deps() + + # TODO: move some dependencies from "grpc_deps" here? def grpc_test_only_deps(): """Internal, not intended for use by packages that are consuming grpc. @@ -290,3 +295,4 @@ def grpc_test_only_deps(): url = "https://github.com/twisted/constantly/archive/15.1.0.zip", build_file = "@com_github_grpc_grpc//third_party:constantly.BUILD", ) + diff --git a/bazel/test/python_test_repo/WORKSPACE b/bazel/test/python_test_repo/WORKSPACE index 0bc58045693..f8a00c88283 100644 --- a/bazel/test/python_test_repo/WORKSPACE +++ b/bazel/test/python_test_repo/WORKSPACE @@ -6,9 +6,6 @@ local_repository( load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") grpc_deps() -load("@com_github_grpc_grpc//bazel:grpc_python_deps.bzl", "grpc_python_deps") -grpc_python_deps() - # TODO(https://github.com/grpc/grpc/issues/19835): Remove. load("@upb//bazel:workspace_deps.bzl", "upb_deps") upb_deps() From 37b6c8cefd1e59fa700d7986456aa40d04e79677 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 5 Aug 2019 14:18:52 -0700 Subject: [PATCH 10/18] Fix sanity check --- tools/run_tests/sanity/check_bazel_workspace.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/run_tests/sanity/check_bazel_workspace.py b/tools/run_tests/sanity/check_bazel_workspace.py index 4ddbc74646f..40a4f5b4fd2 100755 --- a/tools/run_tests/sanity/check_bazel_workspace.py +++ b/tools/run_tests/sanity/check_bazel_workspace.py @@ -115,6 +115,9 @@ class BazelEvalState(object): return self.names_and_urls[args['name']] = args['remote'] + def grpc_python_deps(self): + pass + # Parse git hashes from bazel/grpc_deps.bzl {new_}http_archive rules with open(os.path.join('bazel', 'grpc_deps.bzl'), 'r') as f: @@ -131,6 +134,7 @@ build_rules = { 'http_archive': lambda **args: eval_state.http_archive(**args), 'load': lambda a, b: None, 'git_repository': lambda **args: eval_state.git_repository(**args), + 'grpc_python_deps': lambda: None, } exec bazel_file in build_rules for name in _GRPC_DEP_NAMES: @@ -173,6 +177,7 @@ for name in _GRPC_DEP_NAMES: 'http_archive': lambda **args: state.http_archive(**args), 'load': lambda a, b: None, 'git_repository': lambda **args: state.git_repository(**args), + 'grpc_python_deps': lambda *args, **kwargs: None, } exec bazel_file in rules assert name not in names_and_urls_with_overridden_name.keys() From 0aa30c6f239ded57cb191c1c18038dd5b293d091 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 5 Aug 2019 16:34:47 -0700 Subject: [PATCH 11/18] Remove unintentional dependencies --- src/python/grpcio_tests/tests/interop/BUILD.bazel | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel index 43fdc7ab462..8ac3f7d52d5 100644 --- a/src/python/grpcio_tests/tests/interop/BUILD.bazel +++ b/src/python/grpcio_tests/tests/interop/BUILD.bazel @@ -32,9 +32,7 @@ py_library( "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_tests/tests:bazel_namespace_package_hack", "//src/proto/grpc/testing:empty_py_pb2", - "//src/proto/grpc/testing:empty_py_pb2_grpc", "//src/proto/grpc/testing:py_messages_proto", - "//src/proto/grpc/testing:messages_py_pb2_grpc", "//src/proto/grpc/testing:py_test_proto", "//src/proto/grpc/testing:test_py_pb2_grpc", requirement("google-auth"), From a77404ae6b8e286ccf0e5b61f7e3b1d3d3fafd3c Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 5 Aug 2019 16:38:41 -0700 Subject: [PATCH 12/18] Add README to bazel test --- bazel/test/python_test_repo/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 bazel/test/python_test_repo/README.md diff --git a/bazel/test/python_test_repo/README.md b/bazel/test/python_test_repo/README.md new file mode 100644 index 00000000000..d9dfd1d99b5 --- /dev/null +++ b/bazel/test/python_test_repo/README.md @@ -0,0 +1,5 @@ +## Bazel Workspace Test + +This directory houses a test ensuring that downstream projects can use +`@com_github_grpc_grpc//src/python/grpcio:grpcio`, `py_proto_library`, and +`py_grpc_library`. From b7eaa36a8cb9264cf8cc7dbc7248086805b066ec Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 5 Aug 2019 16:43:31 -0700 Subject: [PATCH 13/18] Add todo to create new kokoro job. --- tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index eda730d3a54..00bf971a3e2 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -29,6 +29,9 @@ bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_outp bazel clean --expunge bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... + +# TODO(https://github.com/grpc/grpc/issues/19854): Move this to a new kokoro +# job. (cd /var/local/git/grpc/bazel/test/python_test_repo; bazel test --test_output=errors //... ) From 4f13303ec460b3550cac11e71a09cafa45920f86 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 5 Aug 2019 16:46:29 -0700 Subject: [PATCH 14/18] Validate length of deps in py_grpc_library --- bazel/python_rules.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index eb4ac9c9721..8f696b1022d 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -208,6 +208,9 @@ def py_grpc_library( if len(srcs) > 1: fail("Can only compile a single proto at a time.") + if len(deps) > 1: + fail("Deps must have length 1.") + _generate_pb2_grpc_src( name = codegen_grpc_target, deps = srcs, From 5fd25f3c7c04ef4153fe26fcb1e42a2a0556b829 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 6 Aug 2019 10:41:54 -0700 Subject: [PATCH 15/18] Refactor rule. --- bazel/protobuf.bzl | 60 +++++++++++++++++++++++++++++++++++ bazel/python_rules.bzl | 71 +++++++----------------------------------- 2 files changed, 72 insertions(+), 59 deletions(-) diff --git a/bazel/protobuf.bzl b/bazel/protobuf.bzl index f2df7bd87b2..39a8cabcdc9 100644 --- a/bazel/protobuf.bzl +++ b/bazel/protobuf.bzl @@ -102,3 +102,63 @@ def get_plugin_args(plugin, flags, dir_out, generate_mocks): "--plugin=protoc-gen-PLUGIN=" + plugin.path, "--PLUGIN_out=" + ",".join(augmented_flags) + ":" + dir_out, ] + +def _get_staged_proto_file(context, source_file): + if source_file.dirname == context.label.package: + return source_file + else: + copied_proto = context.actions.declare_file(source_file.basename) + context.actions.run_shell( + inputs = [source_file], + outputs = [copied_proto], + command = "cp {} {}".format(source_file.path, copied_proto.path), + mnemonic = "CopySourceProto", + ) + return copied_proto + + +def protos_from_context(context): + """Copies proto files to the appropriate location. + + Args: + context: The ctx object for the rule. + + Returns: + A list of the protos. + """ + protos = [] + for src in context.attr.deps: + for file in src[ProtoInfo].direct_sources: + protos.append(_get_staged_proto_file(context, file)) + return protos + + +def includes_from_deps(deps): + """Get includes from rule dependencies.""" + return [ + file + for src in deps + for file in src[ProtoInfo].transitive_imports.to_list() + ] + +def get_proto_arguments(protos, genfiles_dir_path): + """Get the protoc arguments specifying which protos to compile.""" + arguments = [] + for proto in protos: + massaged_path = proto.path + if massaged_path.startswith(genfiles_dir_path): + massaged_path = proto.path[len(genfiles_dir_path) + 1:] + arguments.append(massaged_path) + return arguments + +def declare_out_files(protos, context, generated_file_format): + """Declares and returns the files to be generated.""" + return [ + context.actions.declare_file( + proto_path_to_generated_filename( + proto.basename, + generated_file_format, + ), + ) + for proto in protos + ] diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index 8f696b1022d..c8fdd1ed3c0 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -6,44 +6,20 @@ load( "get_plugin_args", "get_proto_root", "proto_path_to_generated_filename", + "protos_from_context", + "includes_from_deps", + "get_proto_arguments", + "declare_out_files", ) _GENERATED_PROTO_FORMAT = "{}_pb2.py" _GENERATED_GRPC_PROTO_FORMAT = "{}_pb2_grpc.py" -def _get_staged_proto_file(context, source_file): - if source_file.dirname == context.label.package: - return source_file - else: - copied_proto = context.actions.declare_file(source_file.basename) - context.actions.run_shell( - inputs = [source_file], - outputs = [copied_proto], - command = "cp {} {}".format(source_file.path, copied_proto.path), - mnemonic = "CopySourceProto", - ) - return copied_proto - def _generate_py_impl(context): - protos = [] - for src in context.attr.deps: - for file in src[ProtoInfo].direct_sources: - protos.append(_get_staged_proto_file(context, file)) - includes = [ - file - for src in context.attr.deps - for file in src[ProtoInfo].transitive_imports.to_list() - ] + protos = protos_from_context(context) + includes = includes_from_deps(context.attr.deps) proto_root = get_proto_root(context.label.workspace_root) - out_files = [ - context.actions.declare_file( - proto_path_to_generated_filename( - proto.basename, - _GENERATED_PROTO_FORMAT, - ), - ) - for proto in protos - ] + out_files = declare_out_files(protos, context, _GENERATED_PROTO_FORMAT) tools = [context.executable._protoc] arguments = ([ @@ -54,11 +30,7 @@ def _generate_py_impl(context): "--proto_path={}".format(context.genfiles_dir.path) for proto in protos ]) - for proto in protos: - massaged_path = proto.path - if massaged_path.startswith(context.genfiles_dir.path): - massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] - arguments.append(massaged_path) + arguments += get_proto_arguments(protos, context.genfiles_dir.path) context.actions.run( inputs = protos + includes, @@ -116,25 +88,10 @@ def py_proto_library( ) def _generate_pb2_grpc_src_impl(context): - protos = [] - for src in context.attr.deps: - for file in src[ProtoInfo].direct_sources: - protos.append(_get_staged_proto_file(context, file)) - includes = [ - file - for src in context.attr.deps - for file in src[ProtoInfo].transitive_imports.to_list() - ] + protos = protos_from_context(context) + includes = includes_from_deps(context.attr.deps) proto_root = get_proto_root(context.label.workspace_root) - out_files = [ - context.actions.declare_file( - proto_path_to_generated_filename( - proto.basename, - _GENERATED_GRPC_PROTO_FORMAT, - ), - ) - for proto in protos - ] + out_files = declare_out_files(protos, context, _GENERATED_GRPC_PROTO_FORMAT) arguments = [] tools = [context.executable._protoc, context.executable._plugin] @@ -150,11 +107,7 @@ def _generate_pb2_grpc_src_impl(context): "--proto_path={}".format(context.genfiles_dir.path) for proto in protos ] - for proto in protos: - massaged_path = proto.path - if massaged_path.startswith(context.genfiles_dir.path): - massaged_path = proto.path[len(context.genfiles_dir.path) + 1:] - arguments.append(massaged_path) + arguments += get_proto_arguments(protos, context.genfiles_dir.path) context.actions.run( inputs = protos + includes, From 3db88288767e903db35130c03c920fca709047be Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 6 Aug 2019 15:24:09 -0700 Subject: [PATCH 16/18] Add TODOs for prefix stripping --- examples/python/debug/get_stats.py | 1 + src/python/grpcio_channelz/grpc_channelz/v1/channelz.py | 1 + src/python/grpcio_health_checking/grpc_health/v1/health.py | 2 +- .../grpcio_reflection/grpc_reflection/v1alpha/reflection.py | 1 + .../grpcio_tests/tests/channelz/_channelz_servicer_test.py | 2 ++ .../grpcio_tests/tests/health_check/_health_servicer_test.py | 2 +- .../grpcio_tests/tests/reflection/_reflection_servicer_test.py | 2 ++ 7 files changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/python/debug/get_stats.py b/examples/python/debug/get_stats.py index 6eb593deb22..2da51d0efcf 100644 --- a/examples/python/debug/get_stats.py +++ b/examples/python/debug/get_stats.py @@ -21,6 +21,7 @@ import logging import argparse import grpc +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2 from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2_grpc diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py b/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py index 18810cab290..965797f89ec 100644 --- a/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py +++ b/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py @@ -16,6 +16,7 @@ import grpc from grpc._cython import cygrpc +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2 as _channelz_pb2 from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2_grpc as _channelz_pb2_grpc diff --git a/src/python/grpcio_health_checking/grpc_health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/v1/health.py index 93c1895df4c..a0d55570990 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/health.py +++ b/src/python/grpcio_health_checking/grpc_health/v1/health.py @@ -18,7 +18,7 @@ import threading import grpc -# Import using an absolute path to ensure no duplicate loaded modules. +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2 as _health_pb2 from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index bc76136cc5a..61153d9d625 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -17,6 +17,7 @@ import grpc from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_reflection.grpc_reflection.v1alpha \ import reflection_pb2 as _reflection_pb2 diff --git a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py index bfe184107ec..f2357a4e4a3 100644 --- a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py +++ b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py @@ -18,6 +18,8 @@ import unittest from concurrent import futures import grpc + +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_channelz.grpc_channelz.v1 import channelz from src.python.grpcio_channelz.grpc_channelz.v1 import channelz_pb2 diff --git a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py index 91e841a4d60..428806c71b0 100644 --- a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py +++ b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py @@ -20,7 +20,7 @@ import unittest import grpc -# Import using an absolute path to ensure no duplicate loaded modules. +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_health_checking.grpc_health.v1 import health from src.python.grpcio_health_checking.grpc_health.v1 import health_pb2 diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 1a1fdc83163..e45f98abec6 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -16,6 +16,8 @@ import unittest import grpc + +# TODO(https://github.com/grpc/grpc/issues/19863): Remove. try: from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection from src.python.grpcio_reflection.grpc_reflection.v1alpha import reflection_pb2 From b26b25113f2fd09046613a2956c7d84971259e7b Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 7 Aug 2019 11:22:01 -0700 Subject: [PATCH 17/18] Make soruce file validation exact --- bazel/python_rules.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index c8fdd1ed3c0..0f00d53b6fb 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -70,7 +70,7 @@ def py_proto_library( srcs: A list of proto_library dependencies. Must contain a single element. """ codegen_target = "_{}_codegen".format(name) - if len(srcs) > 1: + if len(srcs) != 1: fail("Can only compile a single proto at a time.") @@ -158,10 +158,10 @@ def py_grpc_library( proto_library in `srcs`. """ codegen_grpc_target = "_{}_grpc_codegen".format(name) - if len(srcs) > 1: + if len(srcs) != 1: fail("Can only compile a single proto at a time.") - if len(deps) > 1: + if len(deps) != 1: fail("Deps must have length 1.") _generate_pb2_grpc_src( From b1021ae64816c1da0ec01a5aa390267651bf0d63 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 7 Aug 2019 11:22:56 -0700 Subject: [PATCH 18/18] Capitalize name. --- tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index 00bf971a3e2..f725eb7a3d7 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -30,7 +30,7 @@ bazel clean --expunge bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... -# TODO(https://github.com/grpc/grpc/issues/19854): Move this to a new kokoro +# TODO(https://github.com/grpc/grpc/issues/19854): Move this to a new Kokoro # job. (cd /var/local/git/grpc/bazel/test/python_test_repo; bazel test --test_output=errors //...