From 24c562dbaa95539031c45ab4bdce5070ca6c2af5 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 29 Aug 2019 10:47:44 -0700 Subject: [PATCH] Revert "Merge pull request #20097 from gnossen/dual_version_python_tests" This reverts commit c9c847f334d9310f01456c5f69ec983cfcb99496, reversing changes made to 07ba4de3927dbb2c71b92b4ca38b2fcd1b52e306. --- BUILD | 2 +- bazel/grpc_build_system.bzl | 11 +- bazel/grpc_deps.bzl | 15 ++- bazel/grpc_python_deps.bzl | 9 -- bazel/python_rules.bzl | 26 ----- examples/python/auth/BUILD.bazel | 3 - examples/python/cancellation/BUILD.bazel | 3 - examples/python/compression/BUILD.bazel | 3 - examples/python/debug/BUILD.bazel | 3 - examples/python/errors/BUILD.bazel | 1 - examples/python/multiprocessing/BUILD | 3 - examples/python/wait_for_ready/BUILD.bazel | 1 - .../grpcio_tests/tests/channelz/BUILD.bazel | 4 +- .../tests/health_check/BUILD.bazel | 3 +- .../grpcio_tests/tests/interop/BUILD.bazel | 5 +- .../grpcio_tests/tests/reflection/BUILD.bazel | 3 +- .../grpcio_tests/tests/status/BUILD.bazel | 3 +- .../grpcio_tests/tests/unit/BUILD.bazel | 4 +- .../tests/unit/_cython/BUILD.bazel | 3 +- .../unit/framework/foundation/BUILD.bazel | 3 +- templates/tools/dockerfile/bazel.include | 2 +- third_party/py/BUILD.tpl | 49 ++++----- third_party/py/python_configure.bzl | 104 +++++++----------- third_party/py/remote.BUILD.tpl | 10 ++ third_party/py/variety.tpl | 26 ----- tools/bazel | 2 +- tools/bazel.rc | 4 + tools/dockerfile/test/bazel/Dockerfile | 2 +- tools/dockerfile/test/sanity/Dockerfile | 2 +- .../linux/grpc_python_bazel_test_in_docker.sh | 3 + tools/remote_build/kokoro.bazelrc | 7 +- tools/remote_build/manual.bazelrc | 7 +- tools/remote_build/rbe_common.bazelrc | 2 +- 33 files changed, 114 insertions(+), 214 deletions(-) create mode 100644 third_party/py/remote.BUILD.tpl delete mode 100644 third_party/py/variety.tpl diff --git a/BUILD b/BUILD index e489bc2584a..9198dd61282 100644 --- a/BUILD +++ b/BUILD @@ -65,7 +65,7 @@ config_setting( config_setting( name = "python3", - flag_values = {"@bazel_tools//tools/python:python_version": "PY3"}, + values = {"python_path": "python3"}, ) config_setting( diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index f4777e50bc1..f386b87b583 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -269,22 +269,13 @@ def grpc_sh_binary(name, srcs, data = []): data = data, ) -def grpc_py_binary(name, - srcs, - data = [], - deps = [], - external_deps = [], - testonly = False, - python_version = "PY2", - **kwargs): +def grpc_py_binary(name, srcs, data = [], deps = [], external_deps = [], testonly = False): native.py_binary( name = name, srcs = srcs, testonly = testonly, data = data, deps = deps + _get_external_deps(external_deps), - python_version = python_version, - **kwargs ) def grpc_package(name, visibility = "private", features = []): diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 6bbb960b6c3..06323ff3f24 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -176,11 +176,11 @@ def grpc_deps(): if "bazel_toolchains" not in native.existing_rules(): http_archive( name = "bazel_toolchains", - sha256 = "872955b658113924eb1a3594b04d43238da47f4f90c17b76e8785709490dc041", - strip_prefix = "bazel-toolchains-1083686fde6032378d52b4c98044922cebde364e", + sha256 = "d968b414b32aa99c86977e1171645d31da2b52ac88060de3ac1e49932d5dcbf1", + strip_prefix = "bazel-toolchains-4bd5df80d77aa7f4fb943dfdfad5c9056a62fb47", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/1083686fde6032378d52b4c98044922cebde364e.tar.gz", - "https://github.com/bazelbuild/bazel-toolchains/archive/1083686fde6032378d52b4c98044922cebde364e.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/4bd5df80d77aa7f4fb943dfdfad5c9056a62fb47.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/4bd5df80d77aa7f4fb943dfdfad5c9056a62fb47.tar.gz", ], ) @@ -221,11 +221,10 @@ def grpc_deps(): ) if "build_bazel_rules_apple" not in native.existing_rules(): - http_archive( + git_repository( name = "build_bazel_rules_apple", - url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", - strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", - sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", + remote = "https://github.com/bazelbuild/rules_apple.git", + tag = "0.17.2", ) grpc_python_deps() diff --git a/bazel/grpc_python_deps.bzl b/bazel/grpc_python_deps.bzl index 2a439bdf226..4e7cc1537fa 100644 --- a/bazel/grpc_python_deps.bzl +++ b/bazel/grpc_python_deps.bzl @@ -47,15 +47,6 @@ def grpc_python_deps(): remote = "https://github.com/bazelbuild/rules_python.git", ) - - if "rules_python" not in native.existing_rules(): - http_archive( - name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/archive/9d68f24659e8ce8b736590ba1e4418af06ec2552.zip", - sha256 = "f7402f11691d657161f871e11968a984e5b48b023321935f5a55d7e56cf4758a", - strip_prefix = "rules_python-9d68f24659e8ce8b736590ba1e4418af06ec2552", - ) - python_configure(name = "local_config_python") native.bind( diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index e7e9e597b05..12f51f8b172 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -178,29 +178,3 @@ def py_grpc_library( deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps, **kwargs ) - - -def py2and3_test(name, - py_test = native.py_test, - **kwargs): - if "python_version" in kwargs: - fail("Cannot specify 'python_version' in py2and3_test.") - - names = [name + suffix for suffix in (".python2", ".python3")] - python_versions = ["PY2", "PY3"] - for case_name, python_version in zip(names, python_versions): - py_test( - name = case_name, - python_version = python_version, - **kwargs - ) - - suite_kwargs = {} - if "visibility" in kwargs: - suite_kwargs["visibility"] = kwargs["visibility"] - - native.test_suite( - name = name, - tests = names, - **suite_kwargs - ) diff --git a/examples/python/auth/BUILD.bazel b/examples/python/auth/BUILD.bazel index 72620ee46c5..cc454fdfdfe 100644 --- a/examples/python/auth/BUILD.bazel +++ b/examples/python/auth/BUILD.bazel @@ -39,7 +39,6 @@ py_binary( "//examples:helloworld_py_pb2", "//examples:helloworld_py_pb2_grpc", ], - python_version = "PY3", ) py_binary( @@ -52,7 +51,6 @@ py_binary( "//examples:helloworld_py_pb2", "//examples:helloworld_py_pb2_grpc", ], - python_version = "PY3", ) py_test( @@ -65,5 +63,4 @@ py_test( ":customized_auth_server", ":_credentials", ], - python_version = "PY3", ) diff --git a/examples/python/cancellation/BUILD.bazel b/examples/python/cancellation/BUILD.bazel index b4451f60711..17b1b20168e 100644 --- a/examples/python/cancellation/BUILD.bazel +++ b/examples/python/cancellation/BUILD.bazel @@ -45,7 +45,6 @@ py_binary( "//external:six" ], srcs_version = "PY2AND3", - python_version = "PY3", ) py_library( @@ -69,7 +68,6 @@ py_binary( "//:python3": [], }), srcs_version = "PY2AND3", - python_version = "PY3", ) py_test( @@ -80,5 +78,4 @@ py_test( ":server" ], size = "small", - python_version = "PY3", ) diff --git a/examples/python/compression/BUILD.bazel b/examples/python/compression/BUILD.bazel index 4141eda2ffd..9d5f6bb83ed 100644 --- a/examples/python/compression/BUILD.bazel +++ b/examples/python/compression/BUILD.bazel @@ -21,7 +21,6 @@ py_binary( "//examples:helloworld_py_pb2_grpc", ], srcs_version = "PY2AND3", - python_version = "PY3", ) py_binary( @@ -33,7 +32,6 @@ py_binary( "//examples:helloworld_py_pb2_grpc", ], srcs_version = "PY2AND3", - python_version = "PY3", ) py_test( @@ -45,5 +43,4 @@ py_test( ":server", ], size = "small", - python_version = "PY3", ) diff --git a/examples/python/debug/BUILD.bazel b/examples/python/debug/BUILD.bazel index 332991332f8..657ae860ae3 100644 --- a/examples/python/debug/BUILD.bazel +++ b/examples/python/debug/BUILD.bazel @@ -35,7 +35,6 @@ py_binary( "//examples:helloworld_py_pb2", "//examples:helloworld_py_pb2_grpc", ], - python_version = "PY3", ) py_binary( @@ -46,7 +45,6 @@ py_binary( "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz", ], - python_version = "PY3", ) py_test( @@ -61,5 +59,4 @@ py_test( ":send_message", ":get_stats", ], - python_version = "PY3", ) diff --git a/examples/python/errors/BUILD.bazel b/examples/python/errors/BUILD.bazel index 367bd81925f..4b779ddfcf1 100644 --- a/examples/python/errors/BUILD.bazel +++ b/examples/python/errors/BUILD.bazel @@ -55,5 +55,4 @@ py_test( "../../../src/python/grpcio_status", "../../../src/python/grpcio_tests", ], - python_version = "PY3", ) diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index 2503970bc80..490aea0c1e6 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -42,7 +42,6 @@ py_binary( ":prime_proto_pb2_grpc", ], srcs_version = "PY3", - python_version = "PY3", ) py_binary( @@ -58,7 +57,6 @@ py_binary( "//:python3": [], }), srcs_version = "PY3", - python_version = "PY3", ) py_test( @@ -69,5 +67,4 @@ py_test( ":server" ], size = "small", - python_version = "PY3", ) diff --git a/examples/python/wait_for_ready/BUILD.bazel b/examples/python/wait_for_ready/BUILD.bazel index 9cbddd1a6e3..f074ae7bb7f 100644 --- a/examples/python/wait_for_ready/BUILD.bazel +++ b/examples/python/wait_for_ready/BUILD.bazel @@ -30,5 +30,4 @@ py_test( srcs = ["test/_wait_for_ready_example_test.py"], deps = [":wait_for_ready_example",], size = "small", - python_version = "PY3", ) diff --git a/src/python/grpcio_tests/tests/channelz/BUILD.bazel b/src/python/grpcio_tests/tests/channelz/BUILD.bazel index f4d246847f7..63513616e77 100644 --- a/src/python/grpcio_tests/tests/channelz/BUILD.bazel +++ b/src/python/grpcio_tests/tests/channelz/BUILD.bazel @@ -1,8 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load("//bazel:python_rules.bzl", "py2and3_test") - -py2and3_test( +py_test( name = "channelz_servicer_test", srcs = ["_channelz_servicer_test.py"], main = "_channelz_servicer_test.py", diff --git a/src/python/grpcio_tests/tests/health_check/BUILD.bazel b/src/python/grpcio_tests/tests/health_check/BUILD.bazel index 797b6a48e91..49f076be9a1 100644 --- a/src/python/grpcio_tests/tests/health_check/BUILD.bazel +++ b/src/python/grpcio_tests/tests/health_check/BUILD.bazel @@ -1,7 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load("//bazel:python_rules.bzl", "py2and3_test") -py2and3_test( +py_test( name = "health_servicer_test", srcs = ["_health_servicer_test.py"], main = "_health_servicer_test.py", diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel index 4685852162b..8ac3f7d52d5 100644 --- a/src/python/grpcio_tests/tests/interop/BUILD.bazel +++ b/src/python/grpcio_tests/tests/interop/BUILD.bazel @@ -1,5 +1,4 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) @@ -81,7 +80,7 @@ py_library( ], ) -py2and3_test( +py_test( name = "_insecure_intraop_test", size = "small", srcs = ["_insecure_intraop_test.py"], @@ -100,7 +99,7 @@ py2and3_test( ], ) -py2and3_test( +py_test( name = "_secure_intraop_test", size = "small", srcs = ["_secure_intraop_test.py"], diff --git a/src/python/grpcio_tests/tests/reflection/BUILD.bazel b/src/python/grpcio_tests/tests/reflection/BUILD.bazel index 65a08c2a435..e9b56191df8 100644 --- a/src/python/grpcio_tests/tests/reflection/BUILD.bazel +++ b/src/python/grpcio_tests/tests/reflection/BUILD.bazel @@ -1,9 +1,8 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) -py2and3_test( +py_test( name="_reflection_servicer_test", size="small", timeout="moderate", diff --git a/src/python/grpcio_tests/tests/status/BUILD.bazel b/src/python/grpcio_tests/tests/status/BUILD.bazel index 0868de01acf..b163fe3975e 100644 --- a/src/python/grpcio_tests/tests/status/BUILD.bazel +++ b/src/python/grpcio_tests/tests/status/BUILD.bazel @@ -1,9 +1,8 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) -py2and3_test( +py_test( name = "grpc_status_test", srcs = ["_grpc_status_test.py"], main = "_grpc_status_test.py", diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index 587d8cb246f..49203b7fa16 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -1,5 +1,3 @@ -load("//bazel:python_rules.bzl", "py2and3_test") - package(default_visibility = ["//visibility:public"]) GRPCIO_TESTS_UNIT = [ @@ -82,7 +80,7 @@ py_library( ) [ - py2and3_test( + py_test( name=test_file_name[:-3], size="small", srcs=[test_file_name], diff --git a/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel b/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel index 867649a6a50..458a6b1fb8a 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel @@ -1,5 +1,4 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("//bazel:python_rules.bzl", "py2and3_test") package(default_visibility = ["//visibility:public"]) @@ -24,7 +23,7 @@ py_library( ) [ - py2and3_test( + py_test( name=test_file_name[:-3], size="small", srcs=[test_file_name], diff --git a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel index a93249301c4..d69186e1fde 100644 --- a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel @@ -1,12 +1,11 @@ package(default_visibility = ["//visibility:public"]) -load("//bazel:python_rules.bzl", "py2and3_test") py_library( name = "stream_testing", srcs = ["stream_testing.py"], ) -py2and3_test( +py_test( name = "logging_pool_test", srcs = ["_logging_pool_test.py"], main = "_logging_pool_test.py", diff --git a/templates/tools/dockerfile/bazel.include b/templates/tools/dockerfile/bazel.include index 12a22785623..adde6ed4787 100644 --- a/templates/tools/dockerfile/bazel.include +++ b/templates/tools/dockerfile/bazel.include @@ -2,7 +2,7 @@ # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 0.28.1 +ENV BAZEL_VERSION 0.26.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl index 8f010f85a03..2283c573bc3 100644 --- a/third_party/py/BUILD.tpl +++ b/third_party/py/BUILD.tpl @@ -2,36 +2,35 @@ package(default_visibility=["//visibility:public"]) -config_setting( - name="windows", - values={"cpu": "x64_windows"}, - visibility=["//visibility:public"], +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name="python_lib", + interface_library=select({ + ":windows": ":python_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided=1, ) -config_setting( - name="python2", - flag_values = {"@rules_python//python:python_version": "PY2"} +cc_library( + name="python_headers", + hdrs=[":python_include"], + deps=select({ + ":windows": [":python_lib"], + "//conditions:default": [], + }), + includes=["python_include"], ) config_setting( - name="python3", - flag_values = {"@rules_python//python:python_version": "PY3"} + name="windows", + values={"cpu": "x64_windows"}, + visibility=["//visibility:public"], ) -cc_library( - name = "python_lib", - deps = select({ - ":python2": ["//_python2:_python2_lib"], - ":python3": ["//_python3:_python3_lib"], - "//conditions:default": ["not-existing.lib"], - }) -) +%{PYTHON_INCLUDE_GENRULE} +%{PYTHON_IMPORT_LIB_GENRULE} + -cc_library( - name = "python_headers", - deps = select({ - ":python2": ["//_python2:_python2_headers"], - ":python3": ["//_python3:_python3_headers"], - "//conditions:default": ["not-existing.headers"], - }) -) diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 6f9a178a057..e6fa5ed10e9 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -3,15 +3,14 @@ `python_configure` depends on the following environment variables: - * `PYTHON2_BIN_PATH`: location of python binary. - * `PYTHON2_LIB_PATH`: Location of python libraries. + * `PYTHON_BIN_PATH`: location of python binary. + * `PYTHON_LIB_PATH`: Location of python libraries. """ _BAZEL_SH = "BAZEL_SH" -_PYTHON2_BIN_PATH = "PYTHON2_BIN_PATH" -_PYTHON2_LIB_PATH = "PYTHON2_LIB_PATH" -_PYTHON3_BIN_PATH = "PYTHON3_BIN_PATH" -_PYTHON3_LIB_PATH = "PYTHON3_LIB_PATH" +_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" +_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" def _tpl(repository_ctx, tpl, substitutions={}, out=None): @@ -137,9 +136,9 @@ def _symlink_genrule_for_dir(repository_ctx, "\n".join(outs)) -def _get_python_bin(repository_ctx, bin_path_key, default_bin_path): +def _get_python_bin(repository_ctx): """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(bin_path_key, default_bin_path) + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH, 'python') if not repository_ctx.path(python_bin).exists: # It's a command, use 'which' to find its path. python_bin_path = repository_ctx.which(python_bin) @@ -151,7 +150,7 @@ def _get_python_bin(repository_ctx, bin_path_key, default_bin_path): _fail("Cannot find python in PATH, please make sure " + "python is installed and add its directory in PATH, or --define " + "%s='/something/else'.\nPATH=%s" % - (bin_path_key, repository_ctx.os.environ.get("PATH", ""))) + (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) def _get_bash_bin(repository_ctx): @@ -171,9 +170,9 @@ def _get_bash_bin(repository_ctx): (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) -def _get_python_lib(repository_ctx, python_bin, lib_path_key): +def _get_python_lib(repository_ctx, python_bin): """Gets the python lib path.""" - python_lib = repository_ctx.os.environ.get(lib_path_key) + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) if python_lib != None: return python_lib print_lib = ( @@ -203,13 +202,13 @@ def _check_python_lib(repository_ctx, python_lib): _fail("Invalid python library path: %s" % python_lib) -def _check_python_bin(repository_ctx, python_bin, bin_path_key): +def _check_python_bin(repository_ctx, python_bin): """Checks the python bin path.""" cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) if result.return_code == 1: _fail("--define %s='%s' is not executable. Is it the python binary?" % - (bin_path_key, python_bin)) + (_PYTHON_BIN_PATH, python_bin)) def _get_python_include(repository_ctx, python_bin): @@ -223,11 +222,11 @@ def _get_python_include(repository_ctx, python_bin): error_msg="Problem getting python include path.", error_details=( "Is the Python binary path set up right? " + "(See ./configure or " - + _PYTHON2_BIN_PATH + ".) " + "Is distutils installed?")) + + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) return result.stdout.splitlines()[0] -def _get_python_import_lib_name(repository_ctx, python_bin, bin_path_key): +def _get_python_import_lib_name(repository_ctx, python_bin): """Get Python import library name (pythonXY.lib) on Windows.""" result = _execute( repository_ctx, [ @@ -237,85 +236,66 @@ def _get_python_import_lib_name(repository_ctx, python_bin, bin_path_key): ], error_msg="Problem getting python import library.", error_details=("Is the Python binary path set up right? " + - "(See ./configure or " + bin_path_key + ".) ")) + "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) return result.stdout.splitlines()[0] -def _create_single_version_package(repository_ctx, - variety_name, - bin_path_key, - default_bin_path, - lib_path_key): +def _create_local_python_repository(repository_ctx): """Creates the repository containing files set up to build with Python.""" - python_bin = _get_python_bin(repository_ctx, bin_path_key, default_bin_path) - _check_python_bin(repository_ctx, python_bin, bin_path_key) - python_lib = _get_python_lib(repository_ctx, python_bin, lib_path_key) + python_bin = _get_python_bin(repository_ctx) + _check_python_bin(repository_ctx, python_bin) + python_lib = _get_python_lib(repository_ctx, python_bin) _check_python_lib(repository_ctx, python_lib) python_include = _get_python_include(repository_ctx, python_bin) python_include_rule = _symlink_genrule_for_dir( - repository_ctx, python_include, '{}_include'.format(variety_name), - '{}_include'.format(variety_name)) + repository_ctx, python_include, 'python_include', 'python_include') python_import_lib_genrule = "" # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib # See https://docs.python.org/3/extending/windows.html if _is_windows(repository_ctx): python_include = _normalize_path(python_include) - python_import_lib_name = _get_python_import_lib_name, bin_path_key( + python_import_lib_name = _get_python_import_lib_name( repository_ctx, python_bin) python_import_lib_src = python_include.rsplit( '/', 1)[0] + "/libs/" + python_import_lib_name python_import_lib_genrule = _symlink_genrule_for_dir( - repository_ctx, None, '', '{}_import_lib'.format(variety_name), + repository_ctx, None, '', 'python_import_lib', [python_import_lib_src], [python_import_lib_name]) _tpl( - repository_ctx, "variety", { + repository_ctx, "BUILD", { "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, - "%{VARIETY_NAME}": variety_name, - }, - out="{}/BUILD".format(variety_name)) + }) + + +def _create_remote_python_repository(repository_ctx, remote_config_repo): + """Creates pointers to a remotely configured repo set up to build with Python. + """ + _tpl(repository_ctx, "remote.BUILD", { + "%{REMOTE_PYTHON_REPO}": remote_config_repo, + }, "BUILD") def _python_autoconf_impl(repository_ctx): """Implementation of the python_autoconf repository rule.""" - _create_single_version_package(repository_ctx, - "_python2", - _PYTHON2_BIN_PATH, - "python", - _PYTHON2_LIB_PATH) - _create_single_version_package(repository_ctx, - "_python3", - _PYTHON3_BIN_PATH, - "python3", - _PYTHON3_LIB_PATH) - _tpl(repository_ctx, "BUILD") + if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: + _create_remote_python_repository( + repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) + else: + _create_local_python_repository(repository_ctx) python_configure = repository_rule( - implementation = _python_autoconf_impl, - environ = [ + implementation=_python_autoconf_impl, + environ=[ _BAZEL_SH, - _PYTHON2_BIN_PATH, - _PYTHON2_LIB_PATH, - _PYTHON3_BIN_PATH, - _PYTHON3_LIB_PATH, + _PYTHON_BIN_PATH, + _PYTHON_LIB_PATH, + _PYTHON_CONFIG_REPO, ], - attrs={ - "_build_tpl": attr.label( - default = Label("//third_party/py:BUILD.tpl"), - allow_single_file = True, - ), - "_variety_tpl": attr.label( - default = Label("//third_party/py:variety.tpl"), - allow_single_file = True, - ), - }, ) """Detects and configures the local Python. -It is expected that the system have both a working Python 2 and python 3 -installation - Add the following to your WORKSPACE FILE: ```python diff --git a/third_party/py/remote.BUILD.tpl b/third_party/py/remote.BUILD.tpl new file mode 100644 index 00000000000..1bfe1f0bf65 --- /dev/null +++ b/third_party/py/remote.BUILD.tpl @@ -0,0 +1,10 @@ +# Adapted with modifications from tensorflow/third_party/py/ + +package(default_visibility=["//visibility:public"]) + +alias( + name="python_headers", + actual="%{REMOTE_PYTHON_REPO}:python_headers", +) + + diff --git a/third_party/py/variety.tpl b/third_party/py/variety.tpl deleted file mode 100644 index 0c466d6d8f0..00000000000 --- a/third_party/py/variety.tpl +++ /dev/null @@ -1,26 +0,0 @@ -package(default_visibility=["//visibility:public"]) - -# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib -# See https://docs.python.org/3/extending/windows.html -cc_import( - name="%{VARIETY_NAME}_lib", - interface_library=select({ - "//:windows": ":%{VARIETY_NAME}_import_lib", - # A placeholder for Unix platforms which makes --no_build happy. - "//conditions:default": "not-existing.lib", - }), - system_provided=1, -) - -cc_library( - name="%{VARIETY_NAME}_headers", - hdrs=[":%{VARIETY_NAME}_include"], - deps=select({ - "//:windows": [":%{VARIETY_NAME}_lib"], - "//conditions:default": [], - }), - includes=["%{VARIETY_NAME}_include"], -) - -%{PYTHON_INCLUDE_GENRULE} -%{PYTHON_IMPORT_LIB_GENRULE} diff --git a/tools/bazel b/tools/bazel index 4d1d57f60d9..4f08d18c656 100755 --- a/tools/bazel +++ b/tools/bazel @@ -32,7 +32,7 @@ then exec -a "$0" "${BAZEL_REAL}" "$@" fi -VERSION=0.28.1 +VERSION=0.26.0 echo "INFO: Running bazel wrapper (see //tools/bazel for details), bazel version $VERSION will be used instead of system-wide bazel installation." diff --git a/tools/bazel.rc b/tools/bazel.rc index fcbe9337b9f..b24f603ddda 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -82,3 +82,7 @@ build:basicprof --copt=-DNDEBUG build:basicprof --copt=-O2 build:basicprof --copt=-DGRPC_BASIC_PROFILER build:basicprof --copt=-DGRPC_TIMERS_RDTSC + +build:python3 --python_path=python3 +build:python3 --python_version=PY3 +build:python3 --action_env=PYTHON_BIN_PATH=python3 diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index 7141fb9c5f1..7e7903359e7 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -52,7 +52,7 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 0.28.1 +ENV BAZEL_VERSION 0.26.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index badff52de34..675378b305b 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -97,7 +97,7 @@ ENV CLANG_TIDY=clang-tidy # Bazel installation # Must be in sync with tools/bazel -ENV BAZEL_VERSION 0.28.1 +ENV BAZEL_VERSION 0.26.0 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. ENV DISABLE_BAZEL_WRAPPER 1 diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index 4a48760aab1..f725eb7a3d7 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -26,6 +26,9 @@ ${name}') cd /var/local/git/grpc/test bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... +bazel clean --expunge +bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... +bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //examples/python/... # TODO(https://github.com/grpc/grpc/issues/19854): Move this to a new Kokoro # job. diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/kokoro.bazelrc index 5c1b061bce3..064e94b2e15 100644 --- a/tools/remote_build/kokoro.bazelrc +++ b/tools/remote_build/kokoro.bazelrc @@ -16,12 +16,13 @@ import %workspace%/tools/remote_build/rbe_common.bazelrc -build --remote_cache=grpcs://remotebuildexecution.googleapis.com -build --remote_executor=grpcs://remotebuildexecution.googleapis.com +build --remote_cache=remotebuildexecution.googleapis.com +build --remote_executor=remotebuildexecution.googleapis.com +build --tls_enabled=true build --auth_enabled=true -build --bes_backend=grpcs://buildeventservice.googleapis.com +build --bes_backend=buildeventservice.googleapis.com build --bes_timeout=600s build --project_id=grpc-testing diff --git a/tools/remote_build/manual.bazelrc b/tools/remote_build/manual.bazelrc index c3c6af42877..fcd41f57521 100644 --- a/tools/remote_build/manual.bazelrc +++ b/tools/remote_build/manual.bazelrc @@ -17,8 +17,9 @@ import %workspace%/tools/remote_build/rbe_common.bazelrc -build --remote_cache=grpcs://remotebuildexecution.googleapis.com -build --remote_executor=grpcs://remotebuildexecution.googleapis.com +build --remote_cache=remotebuildexecution.googleapis.com +build --remote_executor=remotebuildexecution.googleapis.com +build --tls_enabled=true # Enable authentication. This will pick up application default credentials by # default. You can use --auth_credentials=some_file.json to use a service @@ -29,7 +30,7 @@ build --auth_enabled=true # Set flags for uploading to BES in order to view results in the Bazel Build # Results UI. -build --bes_backend=grpcs://buildeventservice.googleapis.com +build --bes_backend="buildeventservice.googleapis.com" build --bes_timeout=60s build --bes_results_url="https://source.cloud.google.com/results/invocations/" build --project_id=grpc-testing diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc index 6236003fe17..6baaf24732c 100644 --- a/tools/remote_build/rbe_common.bazelrc +++ b/tools/remote_build/rbe_common.bazelrc @@ -87,4 +87,4 @@ build:ubsan --test_timeout=3600 # how to update the bazel toolchain for ubsan: # - check for the latest released version in https://github.com/bazelbuild/bazel-toolchains/tree/master/configs/experimental/ubuntu16_04_clang # - you might need to update the bazel_toolchains dependency in grpc_deps.bzl -build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.28.0/ubsan:toolchain +build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.23.0/ubsan:toolchain