From 1444cd1dd31c81daa63477be445a941722a42f5d Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 29 Aug 2019 10:51:36 -0700 Subject: [PATCH 1/2] Revert "Revert "Merge pull request #20097 from gnossen/dual_version_python_tests"" This reverts commit 24c562dbaa95539031c45ab4bdce5070ca6c2af5. --- BUILD | 2 +- bazel/grpc_build_system.bzl | 11 +- bazel/grpc_deps.bzl | 15 +-- bazel/grpc_python_deps.bzl | 9 ++ bazel/python_rules.bzl | 26 +++++ examples/python/auth/BUILD.bazel | 3 + examples/python/cancellation/BUILD.bazel | 3 + examples/python/compression/BUILD.bazel | 3 + examples/python/debug/BUILD.bazel | 3 + examples/python/errors/BUILD.bazel | 1 + examples/python/multiprocessing/BUILD | 3 + examples/python/wait_for_ready/BUILD.bazel | 1 + .../grpcio_tests/tests/channelz/BUILD.bazel | 4 +- .../tests/health_check/BUILD.bazel | 3 +- .../grpcio_tests/tests/interop/BUILD.bazel | 5 +- .../grpcio_tests/tests/reflection/BUILD.bazel | 3 +- .../grpcio_tests/tests/status/BUILD.bazel | 3 +- .../grpcio_tests/tests/unit/BUILD.bazel | 4 +- .../tests/unit/_cython/BUILD.bazel | 3 +- .../unit/framework/foundation/BUILD.bazel | 3 +- templates/tools/dockerfile/bazel.include | 2 +- third_party/py/BUILD.tpl | 49 +++++---- third_party/py/python_configure.bzl | 104 +++++++++++------- third_party/py/remote.BUILD.tpl | 10 -- third_party/py/variety.tpl | 26 +++++ tools/bazel | 2 +- tools/bazel.rc | 4 - tools/dockerfile/test/bazel/Dockerfile | 2 +- tools/dockerfile/test/sanity/Dockerfile | 2 +- .../linux/grpc_python_bazel_test_in_docker.sh | 3 - tools/remote_build/kokoro.bazelrc | 7 +- tools/remote_build/manual.bazelrc | 7 +- tools/remote_build/rbe_common.bazelrc | 2 +- 33 files changed, 214 insertions(+), 114 deletions(-) delete mode 100644 third_party/py/remote.BUILD.tpl create mode 100644 third_party/py/variety.tpl diff --git a/BUILD b/BUILD index 9198dd61282..e489bc2584a 100644 --- a/BUILD +++ b/BUILD @@ -65,7 +65,7 @@ config_setting( config_setting( name = "python3", - values = {"python_path": "python3"}, + flag_values = {"@bazel_tools//tools/python:python_version": "PY3"}, ) config_setting( diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index f386b87b583..f4777e50bc1 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -269,13 +269,22 @@ def grpc_sh_binary(name, srcs, data = []): data = data, ) -def grpc_py_binary(name, srcs, data = [], deps = [], external_deps = [], testonly = False): +def grpc_py_binary(name, + srcs, + data = [], + deps = [], + external_deps = [], + testonly = False, + python_version = "PY2", + **kwargs): native.py_binary( name = name, srcs = srcs, testonly = testonly, data = data, deps = deps + _get_external_deps(external_deps), + python_version = python_version, + **kwargs ) def grpc_package(name, visibility = "private", features = []): diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 06323ff3f24..6bbb960b6c3 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -176,11 +176,11 @@ def grpc_deps(): if "bazel_toolchains" not in native.existing_rules(): http_archive( name = "bazel_toolchains", - sha256 = "d968b414b32aa99c86977e1171645d31da2b52ac88060de3ac1e49932d5dcbf1", - strip_prefix = "bazel-toolchains-4bd5df80d77aa7f4fb943dfdfad5c9056a62fb47", + sha256 = "872955b658113924eb1a3594b04d43238da47f4f90c17b76e8785709490dc041", + strip_prefix = "bazel-toolchains-1083686fde6032378d52b4c98044922cebde364e", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/4bd5df80d77aa7f4fb943dfdfad5c9056a62fb47.tar.gz", - "https://github.com/bazelbuild/bazel-toolchains/archive/4bd5df80d77aa7f4fb943dfdfad5c9056a62fb47.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/1083686fde6032378d52b4c98044922cebde364e.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/1083686fde6032378d52b4c98044922cebde364e.tar.gz", ], ) @@ -221,10 +221,11 @@ def grpc_deps(): ) if "build_bazel_rules_apple" not in native.existing_rules(): - git_repository( + http_archive( name = "build_bazel_rules_apple", - remote = "https://github.com/bazelbuild/rules_apple.git", - tag = "0.17.2", + url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", + strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", + sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", ) grpc_python_deps() diff --git a/bazel/grpc_python_deps.bzl b/bazel/grpc_python_deps.bzl index 4e7cc1537fa..2a439bdf226 100644 --- a/bazel/grpc_python_deps.bzl +++ b/bazel/grpc_python_deps.bzl @@ -47,6 +47,15 @@ def grpc_python_deps(): remote = "https://github.com/bazelbuild/rules_python.git", ) + + if "rules_python" not in native.existing_rules(): + http_archive( + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/archive/9d68f24659e8ce8b736590ba1e4418af06ec2552.zip", + sha256 = "f7402f11691d657161f871e11968a984e5b48b023321935f5a55d7e56cf4758a", + strip_prefix = "rules_python-9d68f24659e8ce8b736590ba1e4418af06ec2552", + ) + python_configure(name = "local_config_python") native.bind( diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index 12f51f8b172..e7e9e597b05 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -178,3 +178,29 @@ def py_grpc_library( deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps, **kwargs ) + + +def py2and3_test(name, + py_test = native.py_test, + **kwargs): + if "python_version" in kwargs: + fail("Cannot specify 'python_version' in py2and3_test.") + + names = [name + suffix for suffix in (".python2", ".python3")] + python_versions = ["PY2", "PY3"] + for case_name, python_version in zip(names, python_versions): + py_test( + name = case_name, + python_version = python_version, + **kwargs + ) + + suite_kwargs = {} + if "visibility" in kwargs: + suite_kwargs["visibility"] = kwargs["visibility"] + + native.test_suite( + name = name, + tests = names, + **suite_kwargs + ) diff --git a/examples/python/auth/BUILD.bazel b/examples/python/auth/BUILD.bazel index cc454fdfdfe..72620ee46c5 100644 --- a/examples/python/auth/BUILD.bazel +++ b/examples/python/auth/BUILD.bazel @@ -39,6 +39,7 @@ py_binary( "//examples:helloworld_py_pb2", "//examples:helloworld_py_pb2_grpc", ], + python_version = "PY3", ) py_binary( @@ -51,6 +52,7 @@ py_binary( "//examples:helloworld_py_pb2", "//examples:helloworld_py_pb2_grpc", ], + python_version = "PY3", ) py_test( @@ -63,4 +65,5 @@ py_test( ":customized_auth_server", ":_credentials", ], + python_version = "PY3", ) diff --git a/examples/python/cancellation/BUILD.bazel b/examples/python/cancellation/BUILD.bazel index 17b1b20168e..b4451f60711 100644 --- a/examples/python/cancellation/BUILD.bazel +++ b/examples/python/cancellation/BUILD.bazel @@ -45,6 +45,7 @@ py_binary( "//external:six" ], srcs_version = "PY2AND3", + python_version = "PY3", ) py_library( @@ -68,6 +69,7 @@ py_binary( "//:python3": [], }), srcs_version = "PY2AND3", + python_version = "PY3", ) py_test( @@ -78,4 +80,5 @@ py_test( ":server" ], size = "small", + python_version = "PY3", ) diff --git a/examples/python/compression/BUILD.bazel b/examples/python/compression/BUILD.bazel index 9d5f6bb83ed..4141eda2ffd 100644 --- a/examples/python/compression/BUILD.bazel +++ b/examples/python/compression/BUILD.bazel @@ -21,6 +21,7 @@ py_binary( "//examples:helloworld_py_pb2_grpc", ], srcs_version = "PY2AND3", + python_version = "PY3", ) py_binary( @@ -32,6 +33,7 @@ py_binary( "//examples:helloworld_py_pb2_grpc", ], srcs_version = "PY2AND3", + python_version = "PY3", ) py_test( @@ -43,4 +45,5 @@ py_test( ":server", ], size = "small", + python_version = "PY3", ) diff --git a/examples/python/debug/BUILD.bazel b/examples/python/debug/BUILD.bazel index 657ae860ae3..332991332f8 100644 --- a/examples/python/debug/BUILD.bazel +++ b/examples/python/debug/BUILD.bazel @@ -35,6 +35,7 @@ py_binary( "//examples:helloworld_py_pb2", "//examples:helloworld_py_pb2_grpc", ], + python_version = "PY3", ) py_binary( @@ -45,6 +46,7 @@ py_binary( "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz", ], + python_version = "PY3", ) py_test( @@ -59,4 +61,5 @@ py_test( ":send_message", ":get_stats", ], + python_version = "PY3", ) diff --git a/examples/python/errors/BUILD.bazel b/examples/python/errors/BUILD.bazel index 4b779ddfcf1..367bd81925f 100644 --- a/examples/python/errors/BUILD.bazel +++ b/examples/python/errors/BUILD.bazel @@ -55,4 +55,5 @@ py_test( "../../../src/python/grpcio_status", "../../../src/python/grpcio_tests", ], + python_version = "PY3", ) diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index 490aea0c1e6..2503970bc80 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -42,6 +42,7 @@ py_binary( ":prime_proto_pb2_grpc", ], srcs_version = "PY3", + python_version = "PY3", ) py_binary( @@ -57,6 +58,7 @@ py_binary( "//:python3": [], }), srcs_version = "PY3", + python_version = "PY3", ) py_test( @@ -67,4 +69,5 @@ py_test( ":server" ], size = "small", + python_version = "PY3", ) diff --git a/examples/python/wait_for_ready/BUILD.bazel b/examples/python/wait_for_ready/BUILD.bazel index f074ae7bb7f..9cbddd1a6e3 100644 --- a/examples/python/wait_for_ready/BUILD.bazel +++ b/examples/python/wait_for_ready/BUILD.bazel @@ -30,4 +30,5 @@ py_test( srcs = ["test/_wait_for_ready_example_test.py"], deps = [":wait_for_ready_example",], size = "small", + python_version = "PY3", ) diff --git a/src/python/grpcio_tests/tests/channelz/BUILD.bazel b/src/python/grpcio_tests/tests/channelz/BUILD.bazel index 63513616e77..f4d246847f7 100644 --- a/src/python/grpcio_tests/tests/channelz/BUILD.bazel +++ b/src/python/grpcio_tests/tests/channelz/BUILD.bazel @@ -1,6 +1,8 @@ package(default_visibility = ["//visibility:public"]) -py_test( +load("//bazel:python_rules.bzl", "py2and3_test") + +py2and3_test( name = "channelz_servicer_test", srcs = ["_channelz_servicer_test.py"], main = "_channelz_servicer_test.py", diff --git a/src/python/grpcio_tests/tests/health_check/BUILD.bazel b/src/python/grpcio_tests/tests/health_check/BUILD.bazel index 49f076be9a1..797b6a48e91 100644 --- a/src/python/grpcio_tests/tests/health_check/BUILD.bazel +++ b/src/python/grpcio_tests/tests/health_check/BUILD.bazel @@ -1,6 +1,7 @@ package(default_visibility = ["//visibility:public"]) +load("//bazel:python_rules.bzl", "py2and3_test") -py_test( +py2and3_test( name = "health_servicer_test", srcs = ["_health_servicer_test.py"], main = "_health_servicer_test.py", diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel index 8ac3f7d52d5..4685852162b 100644 --- a/src/python/grpcio_tests/tests/interop/BUILD.bazel +++ b/src/python/grpcio_tests/tests/interop/BUILD.bazel @@ -1,4 +1,5 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) @@ -80,7 +81,7 @@ py_library( ], ) -py_test( +py2and3_test( name = "_insecure_intraop_test", size = "small", srcs = ["_insecure_intraop_test.py"], @@ -99,7 +100,7 @@ py_test( ], ) -py_test( +py2and3_test( name = "_secure_intraop_test", size = "small", srcs = ["_secure_intraop_test.py"], diff --git a/src/python/grpcio_tests/tests/reflection/BUILD.bazel b/src/python/grpcio_tests/tests/reflection/BUILD.bazel index e9b56191df8..65a08c2a435 100644 --- a/src/python/grpcio_tests/tests/reflection/BUILD.bazel +++ b/src/python/grpcio_tests/tests/reflection/BUILD.bazel @@ -1,8 +1,9 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) -py_test( +py2and3_test( name="_reflection_servicer_test", size="small", timeout="moderate", diff --git a/src/python/grpcio_tests/tests/status/BUILD.bazel b/src/python/grpcio_tests/tests/status/BUILD.bazel index b163fe3975e..0868de01acf 100644 --- a/src/python/grpcio_tests/tests/status/BUILD.bazel +++ b/src/python/grpcio_tests/tests/status/BUILD.bazel @@ -1,8 +1,9 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) -py_test( +py2and3_test( name = "grpc_status_test", srcs = ["_grpc_status_test.py"], main = "_grpc_status_test.py", diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index 49203b7fa16..587d8cb246f 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -1,3 +1,5 @@ +load("//bazel:python_rules.bzl", "py2and3_test") + package(default_visibility = ["//visibility:public"]) GRPCIO_TESTS_UNIT = [ @@ -80,7 +82,7 @@ py_library( ) [ - py_test( + py2and3_test( name=test_file_name[:-3], size="small", srcs=[test_file_name], diff --git a/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel b/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel index 458a6b1fb8a..867649a6a50 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel @@ -1,4 +1,5 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) @@ -23,7 +24,7 @@ py_library( ) [ - py_test( + py2and3_test( name=test_file_name[:-3], size="small", srcs=[test_file_name], diff --git a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel index d69186e1fde..a93249301c4 100644 --- a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel @@ -1,11 +1,12 @@ package(default_visibility = ["//visibility:public"]) +load("//bazel:python_rules.bzl", "py2and3_test") py_library( name = "stream_testing", srcs = ["stream_testing.py"], ) -py_test( +py2and3_test( name = "logging_pool_test", srcs = ["_logging_pool_test.py"], main = "_logging_pool_test.py", diff --git a/templates/tools/dockerfile/bazel.include b/templates/tools/dockerfile/bazel.include index adde6ed4787..12a22785623 100644 --- a/templates/tools/dockerfile/bazel.include +++ b/templates/tools/dockerfile/bazel.include @@ -2,7 +2,7 @@ # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 0.26.0 +ENV BAZEL_VERSION 0.28.1 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl index 2283c573bc3..8f010f85a03 100644 --- a/third_party/py/BUILD.tpl +++ b/third_party/py/BUILD.tpl @@ -2,35 +2,36 @@ package(default_visibility=["//visibility:public"]) -# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib -# See https://docs.python.org/3/extending/windows.html -cc_import( - name="python_lib", - interface_library=select({ - ":windows": ":python_import_lib", - # A placeholder for Unix platforms which makes --no_build happy. - "//conditions:default": "not-existing.lib", - }), - system_provided=1, -) - -cc_library( - name="python_headers", - hdrs=[":python_include"], - deps=select({ - ":windows": [":python_lib"], - "//conditions:default": [], - }), - includes=["python_include"], -) - config_setting( name="windows", values={"cpu": "x64_windows"}, visibility=["//visibility:public"], ) -%{PYTHON_INCLUDE_GENRULE} -%{PYTHON_IMPORT_LIB_GENRULE} +config_setting( + name="python2", + flag_values = {"@rules_python//python:python_version": "PY2"} +) + +config_setting( + name="python3", + flag_values = {"@rules_python//python:python_version": "PY3"} +) +cc_library( + name = "python_lib", + deps = select({ + ":python2": ["//_python2:_python2_lib"], + ":python3": ["//_python3:_python3_lib"], + "//conditions:default": ["not-existing.lib"], + }) +) +cc_library( + name = "python_headers", + deps = select({ + ":python2": ["//_python2:_python2_headers"], + ":python3": ["//_python3:_python3_headers"], + "//conditions:default": ["not-existing.headers"], + }) +) diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index e6fa5ed10e9..6f9a178a057 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -3,14 +3,15 @@ `python_configure` depends on the following environment variables: - * `PYTHON_BIN_PATH`: location of python binary. - * `PYTHON_LIB_PATH`: Location of python libraries. + * `PYTHON2_BIN_PATH`: location of python binary. + * `PYTHON2_LIB_PATH`: Location of python libraries. """ _BAZEL_SH = "BAZEL_SH" -_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" -_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" -_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" +_PYTHON2_BIN_PATH = "PYTHON2_BIN_PATH" +_PYTHON2_LIB_PATH = "PYTHON2_LIB_PATH" +_PYTHON3_BIN_PATH = "PYTHON3_BIN_PATH" +_PYTHON3_LIB_PATH = "PYTHON3_LIB_PATH" def _tpl(repository_ctx, tpl, substitutions={}, out=None): @@ -136,9 +137,9 @@ def _symlink_genrule_for_dir(repository_ctx, "\n".join(outs)) -def _get_python_bin(repository_ctx): +def _get_python_bin(repository_ctx, bin_path_key, default_bin_path): """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH, 'python') + python_bin = repository_ctx.os.environ.get(bin_path_key, default_bin_path) if not repository_ctx.path(python_bin).exists: # It's a command, use 'which' to find its path. python_bin_path = repository_ctx.which(python_bin) @@ -150,7 +151,7 @@ def _get_python_bin(repository_ctx): _fail("Cannot find python in PATH, please make sure " + "python is installed and add its directory in PATH, or --define " + "%s='/something/else'.\nPATH=%s" % - (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) + (bin_path_key, repository_ctx.os.environ.get("PATH", ""))) def _get_bash_bin(repository_ctx): @@ -170,9 +171,9 @@ def _get_bash_bin(repository_ctx): (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) -def _get_python_lib(repository_ctx, python_bin): +def _get_python_lib(repository_ctx, python_bin, lib_path_key): """Gets the python lib path.""" - python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + python_lib = repository_ctx.os.environ.get(lib_path_key) if python_lib != None: return python_lib print_lib = ( @@ -202,13 +203,13 @@ def _check_python_lib(repository_ctx, python_lib): _fail("Invalid python library path: %s" % python_lib) -def _check_python_bin(repository_ctx, python_bin): +def _check_python_bin(repository_ctx, python_bin, bin_path_key): """Checks the python bin path.""" cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) if result.return_code == 1: _fail("--define %s='%s' is not executable. Is it the python binary?" % - (_PYTHON_BIN_PATH, python_bin)) + (bin_path_key, python_bin)) def _get_python_include(repository_ctx, python_bin): @@ -222,11 +223,11 @@ def _get_python_include(repository_ctx, python_bin): error_msg="Problem getting python include path.", error_details=( "Is the Python binary path set up right? " + "(See ./configure or " - + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) + + _PYTHON2_BIN_PATH + ".) " + "Is distutils installed?")) return result.stdout.splitlines()[0] -def _get_python_import_lib_name(repository_ctx, python_bin): +def _get_python_import_lib_name(repository_ctx, python_bin, bin_path_key): """Get Python import library name (pythonXY.lib) on Windows.""" result = _execute( repository_ctx, [ @@ -236,66 +237,85 @@ def _get_python_import_lib_name(repository_ctx, python_bin): ], error_msg="Problem getting python import library.", error_details=("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) + "(See ./configure or " + bin_path_key + ".) ")) return result.stdout.splitlines()[0] -def _create_local_python_repository(repository_ctx): +def _create_single_version_package(repository_ctx, + variety_name, + bin_path_key, + default_bin_path, + lib_path_key): """Creates the repository containing files set up to build with Python.""" - python_bin = _get_python_bin(repository_ctx) - _check_python_bin(repository_ctx, python_bin) - python_lib = _get_python_lib(repository_ctx, python_bin) + python_bin = _get_python_bin(repository_ctx, bin_path_key, default_bin_path) + _check_python_bin(repository_ctx, python_bin, bin_path_key) + python_lib = _get_python_lib(repository_ctx, python_bin, lib_path_key) _check_python_lib(repository_ctx, python_lib) python_include = _get_python_include(repository_ctx, python_bin) python_include_rule = _symlink_genrule_for_dir( - repository_ctx, python_include, 'python_include', 'python_include') + repository_ctx, python_include, '{}_include'.format(variety_name), + '{}_include'.format(variety_name)) python_import_lib_genrule = "" # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib # See https://docs.python.org/3/extending/windows.html if _is_windows(repository_ctx): python_include = _normalize_path(python_include) - python_import_lib_name = _get_python_import_lib_name( + python_import_lib_name = _get_python_import_lib_name, bin_path_key( repository_ctx, python_bin) python_import_lib_src = python_include.rsplit( '/', 1)[0] + "/libs/" + python_import_lib_name python_import_lib_genrule = _symlink_genrule_for_dir( - repository_ctx, None, '', 'python_import_lib', + repository_ctx, None, '', '{}_import_lib'.format(variety_name), [python_import_lib_src], [python_import_lib_name]) _tpl( - repository_ctx, "BUILD", { + repository_ctx, "variety", { "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, - }) - - -def _create_remote_python_repository(repository_ctx, remote_config_repo): - """Creates pointers to a remotely configured repo set up to build with Python. - """ - _tpl(repository_ctx, "remote.BUILD", { - "%{REMOTE_PYTHON_REPO}": remote_config_repo, - }, "BUILD") + "%{VARIETY_NAME}": variety_name, + }, + out="{}/BUILD".format(variety_name)) def _python_autoconf_impl(repository_ctx): """Implementation of the python_autoconf repository rule.""" - if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: - _create_remote_python_repository( - repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) - else: - _create_local_python_repository(repository_ctx) + _create_single_version_package(repository_ctx, + "_python2", + _PYTHON2_BIN_PATH, + "python", + _PYTHON2_LIB_PATH) + _create_single_version_package(repository_ctx, + "_python3", + _PYTHON3_BIN_PATH, + "python3", + _PYTHON3_LIB_PATH) + _tpl(repository_ctx, "BUILD") python_configure = repository_rule( - implementation=_python_autoconf_impl, - environ=[ + implementation = _python_autoconf_impl, + environ = [ _BAZEL_SH, - _PYTHON_BIN_PATH, - _PYTHON_LIB_PATH, - _PYTHON_CONFIG_REPO, + _PYTHON2_BIN_PATH, + _PYTHON2_LIB_PATH, + _PYTHON3_BIN_PATH, + _PYTHON3_LIB_PATH, ], + attrs={ + "_build_tpl": attr.label( + default = Label("//third_party/py:BUILD.tpl"), + allow_single_file = True, + ), + "_variety_tpl": attr.label( + default = Label("//third_party/py:variety.tpl"), + allow_single_file = True, + ), + }, ) """Detects and configures the local Python. +It is expected that the system have both a working Python 2 and python 3 +installation + Add the following to your WORKSPACE FILE: ```python diff --git a/third_party/py/remote.BUILD.tpl b/third_party/py/remote.BUILD.tpl deleted file mode 100644 index 1bfe1f0bf65..00000000000 --- a/third_party/py/remote.BUILD.tpl +++ /dev/null @@ -1,10 +0,0 @@ -# Adapted with modifications from tensorflow/third_party/py/ - -package(default_visibility=["//visibility:public"]) - -alias( - name="python_headers", - actual="%{REMOTE_PYTHON_REPO}:python_headers", -) - - diff --git a/third_party/py/variety.tpl b/third_party/py/variety.tpl new file mode 100644 index 00000000000..0c466d6d8f0 --- /dev/null +++ b/third_party/py/variety.tpl @@ -0,0 +1,26 @@ +package(default_visibility=["//visibility:public"]) + +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name="%{VARIETY_NAME}_lib", + interface_library=select({ + "//:windows": ":%{VARIETY_NAME}_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided=1, +) + +cc_library( + name="%{VARIETY_NAME}_headers", + hdrs=[":%{VARIETY_NAME}_include"], + deps=select({ + "//:windows": [":%{VARIETY_NAME}_lib"], + "//conditions:default": [], + }), + includes=["%{VARIETY_NAME}_include"], +) + +%{PYTHON_INCLUDE_GENRULE} +%{PYTHON_IMPORT_LIB_GENRULE} diff --git a/tools/bazel b/tools/bazel index 4f08d18c656..4d1d57f60d9 100755 --- a/tools/bazel +++ b/tools/bazel @@ -32,7 +32,7 @@ then exec -a "$0" "${BAZEL_REAL}" "$@" fi -VERSION=0.26.0 +VERSION=0.28.1 echo "INFO: Running bazel wrapper (see //tools/bazel for details), bazel version $VERSION will be used instead of system-wide bazel installation." diff --git a/tools/bazel.rc b/tools/bazel.rc index b24f603ddda..fcbe9337b9f 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -82,7 +82,3 @@ build:basicprof --copt=-DNDEBUG build:basicprof --copt=-O2 build:basicprof --copt=-DGRPC_BASIC_PROFILER build:basicprof --copt=-DGRPC_TIMERS_RDTSC - -build:python3 --python_path=python3 -build:python3 --python_version=PY3 -build:python3 --action_env=PYTHON_BIN_PATH=python3 diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index 7e7903359e7..7141fb9c5f1 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -52,7 +52,7 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 0.26.0 +ENV BAZEL_VERSION 0.28.1 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 675378b305b..badff52de34 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -97,7 +97,7 @@ ENV CLANG_TIDY=clang-tidy # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 0.26.0 +ENV BAZEL_VERSION 0.28.1 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index f725eb7a3d7..4a48760aab1 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -26,9 +26,6 @@ ${name}') cd /var/local/git/grpc/test bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... -bazel clean --expunge -bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... -bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... # TODO(https://github.com/grpc/grpc/issues/19854): Move this to a new Kokoro # job. diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/kokoro.bazelrc index 064e94b2e15..5c1b061bce3 100644 --- a/tools/remote_build/kokoro.bazelrc +++ b/tools/remote_build/kokoro.bazelrc @@ -16,13 +16,12 @@ import %workspace%/tools/remote_build/rbe_common.bazelrc -build --remote_cache=remotebuildexecution.googleapis.com -build --remote_executor=remotebuildexecution.googleapis.com -build --tls_enabled=true +build --remote_cache=grpcs://remotebuildexecution.googleapis.com +build --remote_executor=grpcs://remotebuildexecution.googleapis.com build --auth_enabled=true -build --bes_backend=buildeventservice.googleapis.com +build --bes_backend=grpcs://buildeventservice.googleapis.com build --bes_timeout=600s build --project_id=grpc-testing diff --git a/tools/remote_build/manual.bazelrc b/tools/remote_build/manual.bazelrc index fcd41f57521..c3c6af42877 100644 --- a/tools/remote_build/manual.bazelrc +++ b/tools/remote_build/manual.bazelrc @@ -17,9 +17,8 @@ import %workspace%/tools/remote_build/rbe_common.bazelrc -build --remote_cache=remotebuildexecution.googleapis.com -build --remote_executor=remotebuildexecution.googleapis.com -build --tls_enabled=true +build --remote_cache=grpcs://remotebuildexecution.googleapis.com +build --remote_executor=grpcs://remotebuildexecution.googleapis.com # Enable authentication. This will pick up application default credentials by # default. You can use --auth_credentials=some_file.json to use a service @@ -30,7 +29,7 @@ build --auth_enabled=true # Set flags for uploading to BES in order to view results in the Bazel Build # Results UI. -build --bes_backend="buildeventservice.googleapis.com" +build --bes_backend=grpcs://buildeventservice.googleapis.com build --bes_timeout=60s build --bes_results_url="https://source.cloud.google.com/results/invocations/" build --project_id=grpc-testing diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc index 6baaf24732c..6236003fe17 100644 --- a/tools/remote_build/rbe_common.bazelrc +++ b/tools/remote_build/rbe_common.bazelrc @@ -87,4 +87,4 @@ build:ubsan --test_timeout=3600 # how to update the bazel toolchain for ubsan: # - check for the latest released version in https://github.com/bazelbuild/bazel-toolchains/tree/master/configs/experimental/ubuntu16_04_clang # - you might need to update the bazel_toolchains dependency in grpc_deps.bzl -build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.23.0/ubsan:toolchain +build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.28.0/ubsan:toolchain From 9c4de3a983f54a23cbf574356c55be3704ce19f6 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 29 Aug 2019 12:16:36 -0700 Subject: [PATCH 2/2] Move reference to external workspace to OSS-specific file --- BUILD | 8 +++----- bazel/grpc_build_system.bzl | 7 +++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/BUILD b/BUILD index e489bc2584a..d911050b1be 100644 --- a/BUILD +++ b/BUILD @@ -31,6 +31,7 @@ load( "grpc_cc_library", "grpc_generate_one_off_targets", "grpc_upb_proto_library", + "python_config_settings", ) config_setting( @@ -63,11 +64,6 @@ config_setting( values = {"cpu": "x64_windows_msvc"}, ) -config_setting( - name = "python3", - flag_values = {"@bazel_tools//tools/python:python_version": "PY3"}, -) - config_setting( name = "mac_x86_64", values = {"cpu": "darwin"}, @@ -78,6 +74,8 @@ config_setting( values = {"define": "GRPC_USE_CPP_STD_LIB=1"}, ) +python_config_settings() + # This should be updated along with build.yaml g_stands_for = "ganges" diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index f4777e50bc1..74a9c66ff67 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -342,3 +342,10 @@ def grpc_objc_library( def grpc_upb_proto_library(name, deps): upb_proto_library(name = name, deps = deps) + +def python_config_settings(): + native.config_setting( + name = "python3", + flag_values = {"@bazel_tools//tools/python:python_version": "PY3"}, + ) +