From f688f5fcf27674b70c6d80bec92e73d2dccf25e9 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 1 Apr 2022 15:04:50 -0700 Subject: [PATCH] Rehome python bazel distribtest (#29275) * WIP * Test multiple languages * Yapf * isort * Shellcheck * comment not provided * Fix import ordering issue. * Turn up Bazel logging --- .bazelignore | 3 ++- bazel/test/python_test_repo/tools/bazel | 1 - .../distrib/bazel/python}/.gitignore | 0 .../distrib/bazel/python}/BUILD | 0 .../distrib/bazel/python}/README.md | 0 .../distrib/bazel/python}/WORKSPACE | 2 +- .../distrib/bazel/python}/helloworld.proto | 0 .../distrib/bazel/python}/helloworld.py | 8 ++++---- .../distrib/bazel/python}/helloworld_moved.py | 9 ++++++--- .../python}/import_from_proto_library_package.py | 0 .../bazel/python}/import_from_this_package.py | 0 .../distrib/bazel/python}/in_subpackage/BUILD | 0 .../bazel/python}/in_subpackage/subpackage.proto | 0 .../bazel/python}/namespaced/upper/example/BUILD | 0 .../upper/example/import_no_strip_test.py | 10 +++++++--- .../namespaced/upper/example/import_strip_test.py | 4 +++- .../upper/example/namespaced_dependency.proto | 0 .../upper/example/namespaced_example.proto | 0 .../upper/example/no_import_no_strip_test.py | 10 +++++++--- .../upper/example/no_import_strip_test.py | 4 +++- .../distrib/bazel/python}/subdir/hello_dep.proto | 0 test/distrib/bazel/python/tools/bazel | 1 + .../distrib/bazel/python}/transitive_proto_dep.py | 0 .../bazel}/python_second_test_repo/WORKSPACE | 0 .../bazel}/python_second_test_repo/proto/BUILD | 0 test/distrib/bazel/test_single_bazel_version.sh | 13 +++++++++++-- .../linux/grpc_python_bazel_test_in_docker.sh | 7 ------- 27 files changed, 45 insertions(+), 27 deletions(-) delete mode 120000 bazel/test/python_test_repo/tools/bazel rename {bazel/test/python_test_repo => test/distrib/bazel/python}/.gitignore (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/BUILD (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/README.md (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/WORKSPACE (94%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/helloworld.proto (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/helloworld.py (97%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/helloworld_moved.py (94%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/import_from_proto_library_package.py (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/import_from_this_package.py (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/in_subpackage/BUILD (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/in_subpackage/subpackage.proto (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/BUILD (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/import_no_strip_test.py (86%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/import_strip_test.py (93%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/namespaced_dependency.proto (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/namespaced_example.proto (100%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/no_import_no_strip_test.py (87%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/namespaced/upper/example/no_import_strip_test.py (93%) rename {bazel/test/python_test_repo => test/distrib/bazel/python}/subdir/hello_dep.proto (100%) create mode 120000 test/distrib/bazel/python/tools/bazel rename {bazel/test/python_test_repo => test/distrib/bazel/python}/transitive_proto_dep.py (100%) rename {bazel/test => test/distrib/bazel}/python_second_test_repo/WORKSPACE (100%) rename {bazel/test => test/distrib/bazel}/python_second_test_repo/proto/BUILD (100%) diff --git a/.bazelignore b/.bazelignore index 12c4c1349c6..6385cc6e106 100644 --- a/.bazelignore +++ b/.bazelignore @@ -20,8 +20,9 @@ third_party/re2 third_party/upb third_party/xds -bazel/test/python_test_repo test/distrib/bazel/cpp +test/distrib/bazel/python +test/distrib/bazel/python_second_test_repo # Directories generated by setuptools build containing BUILD files. src/python/grpcio_tests/src/ diff --git a/bazel/test/python_test_repo/tools/bazel b/bazel/test/python_test_repo/tools/bazel deleted file mode 120000 index ad4a1267bcd..00000000000 --- a/bazel/test/python_test_repo/tools/bazel +++ /dev/null @@ -1 +0,0 @@ -../../../../tools/bazel \ No newline at end of file diff --git a/bazel/test/python_test_repo/.gitignore b/test/distrib/bazel/python/.gitignore similarity index 100% rename from bazel/test/python_test_repo/.gitignore rename to test/distrib/bazel/python/.gitignore diff --git a/bazel/test/python_test_repo/BUILD b/test/distrib/bazel/python/BUILD similarity index 100% rename from bazel/test/python_test_repo/BUILD rename to test/distrib/bazel/python/BUILD diff --git a/bazel/test/python_test_repo/README.md b/test/distrib/bazel/python/README.md similarity index 100% rename from bazel/test/python_test_repo/README.md rename to test/distrib/bazel/python/README.md diff --git a/bazel/test/python_test_repo/WORKSPACE b/test/distrib/bazel/python/WORKSPACE similarity index 94% rename from bazel/test/python_test_repo/WORKSPACE rename to test/distrib/bazel/python/WORKSPACE index 35cb1c8ed6b..e8120154545 100644 --- a/bazel/test/python_test_repo/WORKSPACE +++ b/test/distrib/bazel/python/WORKSPACE @@ -2,7 +2,7 @@ local_repository( name = "com_github_grpc_grpc", - path = "../../..", + path = "../../../..", ) # Ensure rules don't rely on __main__ naming convention. diff --git a/bazel/test/python_test_repo/helloworld.proto b/test/distrib/bazel/python/helloworld.proto similarity index 100% rename from bazel/test/python_test_repo/helloworld.proto rename to test/distrib/bazel/python/helloworld.proto diff --git a/bazel/test/python_test_repo/helloworld.py b/test/distrib/bazel/python/helloworld.py similarity index 97% rename from bazel/test/python_test_repo/helloworld.py rename to test/distrib/bazel/python/helloworld.py index 3f87191efb4..496f75e3688 100644 --- a/bazel/test/python_test_repo/helloworld.py +++ b/test/distrib/bazel/python/helloworld.py @@ -13,16 +13,15 @@ # limitations under the License. """The Python implementation of the GRPC helloworld.Greeter client.""" +from concurrent import futures import contextlib import datetime import logging import unittest -import grpc - from google.protobuf import duration_pb2 from google.protobuf import timestamp_pb2 -from concurrent import futures +import grpc import helloworld_pb2 import helloworld_pb2_grpc @@ -56,6 +55,7 @@ def _listening_server(): class ImportTest(unittest.TestCase): + def test_import(self): with _listening_server() as port: with grpc.insecure_channel('{}:{}'.format(_HOST, port)) as channel: @@ -66,7 +66,7 @@ class ImportTest(unittest.TestCase): name='you', request_initiation=request_timestamp, ), - wait_for_ready=True) + wait_for_ready=True) self.assertEqual(response.message, "Hello, you!") self.assertGreater(response.request_duration.nanos, 0) diff --git a/bazel/test/python_test_repo/helloworld_moved.py b/test/distrib/bazel/python/helloworld_moved.py similarity index 94% rename from bazel/test/python_test_repo/helloworld_moved.py rename to test/distrib/bazel/python/helloworld_moved.py index b32042cdfa9..c902735862a 100644 --- a/bazel/test/python_test_repo/helloworld_moved.py +++ b/test/distrib/bazel/python/helloworld_moved.py @@ -13,18 +13,20 @@ # limitations under the License. """The Python implementation of the GRPC helloworld.Greeter client.""" +from concurrent import futures import contextlib import datetime import logging import unittest +# TODO(https://github.com/grpc/grpc/issues/29284) +# isort: off import grpc - from google.protobuf import duration_pb2 from google.protobuf import timestamp_pb2 -from concurrent import futures from google.cloud import helloworld_pb2 from google.cloud import helloworld_pb2_grpc +# isort: on _HOST = 'localhost' _SERVER_ADDRESS = '{}:0'.format(_HOST) @@ -56,6 +58,7 @@ def _listening_server(): class ImportTest(unittest.TestCase): + def test_import(self): with _listening_server() as port: with grpc.insecure_channel('{}:{}'.format(_HOST, port)) as channel: @@ -66,7 +69,7 @@ class ImportTest(unittest.TestCase): name='you', request_initiation=request_timestamp, ), - wait_for_ready=True) + wait_for_ready=True) self.assertEqual(response.message, "Hello, you!") self.assertGreater(response.request_duration.nanos, 0) diff --git a/bazel/test/python_test_repo/import_from_proto_library_package.py b/test/distrib/bazel/python/import_from_proto_library_package.py similarity index 100% rename from bazel/test/python_test_repo/import_from_proto_library_package.py rename to test/distrib/bazel/python/import_from_proto_library_package.py diff --git a/bazel/test/python_test_repo/import_from_this_package.py b/test/distrib/bazel/python/import_from_this_package.py similarity index 100% rename from bazel/test/python_test_repo/import_from_this_package.py rename to test/distrib/bazel/python/import_from_this_package.py diff --git a/bazel/test/python_test_repo/in_subpackage/BUILD b/test/distrib/bazel/python/in_subpackage/BUILD similarity index 100% rename from bazel/test/python_test_repo/in_subpackage/BUILD rename to test/distrib/bazel/python/in_subpackage/BUILD diff --git a/bazel/test/python_test_repo/in_subpackage/subpackage.proto b/test/distrib/bazel/python/in_subpackage/subpackage.proto similarity index 100% rename from bazel/test/python_test_repo/in_subpackage/subpackage.proto rename to test/distrib/bazel/python/in_subpackage/subpackage.proto diff --git a/bazel/test/python_test_repo/namespaced/upper/example/BUILD b/test/distrib/bazel/python/namespaced/upper/example/BUILD similarity index 100% rename from bazel/test/python_test_repo/namespaced/upper/example/BUILD rename to test/distrib/bazel/python/namespaced/upper/example/BUILD diff --git a/bazel/test/python_test_repo/namespaced/upper/example/import_no_strip_test.py b/test/distrib/bazel/python/namespaced/upper/example/import_no_strip_test.py similarity index 86% rename from bazel/test/python_test_repo/namespaced/upper/example/import_no_strip_test.py rename to test/distrib/bazel/python/namespaced/upper/example/import_no_strip_test.py index 1b5505dfc73..f0c44539c20 100644 --- a/bazel/test/python_test_repo/namespaced/upper/example/import_no_strip_test.py +++ b/test/distrib/bazel/python/namespaced/upper/example/import_no_strip_test.py @@ -17,15 +17,19 @@ import unittest class ImportTest(unittest.TestCase): + def test_import(self): - from foo.bar.namespaced.upper.example.namespaced_example_pb2 import NamespacedExample + from foo.bar.namespaced.upper.example.namespaced_example_pb2 import \ + NamespacedExample namespaced_example = NamespacedExample() namespaced_example.value = "hello" - # Dummy assert, important part is namespaced example was imported. + # Superfluous assert, important part is namespaced example was imported. self.assertEqual(namespaced_example.value, "hello") def test_grpc(self): - from foo.bar.namespaced.upper.example.namespaced_example_pb2_grpc import NamespacedServiceStub + from foo.bar.namespaced.upper.example.namespaced_example_pb2_grpc import \ + NamespacedServiceStub + # No error from import self.assertEqual(1, 1) diff --git a/bazel/test/python_test_repo/namespaced/upper/example/import_strip_test.py b/test/distrib/bazel/python/namespaced/upper/example/import_strip_test.py similarity index 93% rename from bazel/test/python_test_repo/namespaced/upper/example/import_strip_test.py rename to test/distrib/bazel/python/namespaced/upper/example/import_strip_test.py index 3519a8aa470..1d0d14f48a2 100644 --- a/bazel/test/python_test_repo/namespaced/upper/example/import_strip_test.py +++ b/test/distrib/bazel/python/namespaced/upper/example/import_strip_test.py @@ -17,15 +17,17 @@ import unittest class ImportTest(unittest.TestCase): + def test_import(self): from foo.bar.namespaced_example_pb2 import NamespacedExample namespaced_example = NamespacedExample() namespaced_example.value = "hello" - # Dummy assert, important part is namespaced example was imported. + # Superfluous assert, important part is namespaced example was imported. self.assertEqual(namespaced_example.value, "hello") def test_grpc(self): from foo.bar.namespaced_example_pb2_grpc import NamespacedServiceStub + # No error from import self.assertEqual(1, 1) diff --git a/bazel/test/python_test_repo/namespaced/upper/example/namespaced_dependency.proto b/test/distrib/bazel/python/namespaced/upper/example/namespaced_dependency.proto similarity index 100% rename from bazel/test/python_test_repo/namespaced/upper/example/namespaced_dependency.proto rename to test/distrib/bazel/python/namespaced/upper/example/namespaced_dependency.proto diff --git a/bazel/test/python_test_repo/namespaced/upper/example/namespaced_example.proto b/test/distrib/bazel/python/namespaced/upper/example/namespaced_example.proto similarity index 100% rename from bazel/test/python_test_repo/namespaced/upper/example/namespaced_example.proto rename to test/distrib/bazel/python/namespaced/upper/example/namespaced_example.proto diff --git a/bazel/test/python_test_repo/namespaced/upper/example/no_import_no_strip_test.py b/test/distrib/bazel/python/namespaced/upper/example/no_import_no_strip_test.py similarity index 87% rename from bazel/test/python_test_repo/namespaced/upper/example/no_import_no_strip_test.py rename to test/distrib/bazel/python/namespaced/upper/example/no_import_no_strip_test.py index c08681758e5..6702e52ed11 100644 --- a/bazel/test/python_test_repo/namespaced/upper/example/no_import_no_strip_test.py +++ b/test/distrib/bazel/python/namespaced/upper/example/no_import_no_strip_test.py @@ -17,15 +17,19 @@ import unittest class ImportTest(unittest.TestCase): + def test_import(self): - from namespaced.upper.example.namespaced_example_pb2 import NamespacedExample + from namespaced.upper.example.namespaced_example_pb2 import \ + NamespacedExample namespaced_example = NamespacedExample() namespaced_example.value = "hello" - # Dummy assert, important part is namespaced example was imported. + # Superfluous assert, important part is namespaced example was imported. self.assertEqual(namespaced_example.value, "hello") def test_grpc(self): - from namespaced.upper.example.namespaced_example_pb2_grpc import NamespacedServiceStub + from namespaced.upper.example.namespaced_example_pb2_grpc import \ + NamespacedServiceStub + # No error from import self.assertEqual(1, 1) diff --git a/bazel/test/python_test_repo/namespaced/upper/example/no_import_strip_test.py b/test/distrib/bazel/python/namespaced/upper/example/no_import_strip_test.py similarity index 93% rename from bazel/test/python_test_repo/namespaced/upper/example/no_import_strip_test.py rename to test/distrib/bazel/python/namespaced/upper/example/no_import_strip_test.py index 38dd4fba247..8e756a04f9f 100644 --- a/bazel/test/python_test_repo/namespaced/upper/example/no_import_strip_test.py +++ b/test/distrib/bazel/python/namespaced/upper/example/no_import_strip_test.py @@ -17,15 +17,17 @@ import unittest class ImportTest(unittest.TestCase): + def test_import(self): from namespaced_example_pb2 import NamespacedExample namespaced_example = NamespacedExample() namespaced_example.value = "hello" - # Dummy assert, important part is namespaced example was imported. + # Superfluous assert, important part is namespaced example was imported. self.assertEqual(namespaced_example.value, "hello") def test_grpc(self): from namespaced_example_pb2_grpc import NamespacedServiceStub + # No error from import self.assertEqual(1, 1) diff --git a/bazel/test/python_test_repo/subdir/hello_dep.proto b/test/distrib/bazel/python/subdir/hello_dep.proto similarity index 100% rename from bazel/test/python_test_repo/subdir/hello_dep.proto rename to test/distrib/bazel/python/subdir/hello_dep.proto diff --git a/test/distrib/bazel/python/tools/bazel b/test/distrib/bazel/python/tools/bazel new file mode 120000 index 00000000000..e50c110d3a8 --- /dev/null +++ b/test/distrib/bazel/python/tools/bazel @@ -0,0 +1 @@ +../../../../../tools/bazel \ No newline at end of file diff --git a/bazel/test/python_test_repo/transitive_proto_dep.py b/test/distrib/bazel/python/transitive_proto_dep.py similarity index 100% rename from bazel/test/python_test_repo/transitive_proto_dep.py rename to test/distrib/bazel/python/transitive_proto_dep.py diff --git a/bazel/test/python_second_test_repo/WORKSPACE b/test/distrib/bazel/python_second_test_repo/WORKSPACE similarity index 100% rename from bazel/test/python_second_test_repo/WORKSPACE rename to test/distrib/bazel/python_second_test_repo/WORKSPACE diff --git a/bazel/test/python_second_test_repo/proto/BUILD b/test/distrib/bazel/python_second_test_repo/proto/BUILD similarity index 100% rename from bazel/test/python_second_test_repo/proto/BUILD rename to test/distrib/bazel/python_second_test_repo/proto/BUILD diff --git a/test/distrib/bazel/test_single_bazel_version.sh b/test/distrib/bazel/test_single_bazel_version.sh index 0fa68425411..663d9b19580 100755 --- a/test/distrib/bazel/test_single_bazel_version.sh +++ b/test/distrib/bazel/test_single_bazel_version.sh @@ -41,6 +41,11 @@ EXCLUDED_TARGETS=( "-//examples/android/binder/..." ) +TEST_DIRECTORIES=( + "cpp" + "python" +) + FAILED_TESTS="" export OVERRIDE_BAZEL_VERSION="$VERSION" @@ -48,9 +53,13 @@ export OVERRIDE_BAZEL_VERSION="$VERSION" export OVERRIDE_BAZEL_WRAPPER_DOWNLOAD_DIR=/tmp bazel build -- //... "${EXCLUDED_TARGETS[@]}" || FAILED_TESTS="${FAILED_TESTS}Build " -cd test/distrib/bazel/cpp/ +for TEST_DIRECTORY in "${TEST_DIRECTORIES[@]}"; do + pushd "test/distrib/bazel/$TEST_DIRECTORY/" + + bazel test --test_output=all //:all || FAILED_TESTS="${FAILED_TESTS}${TEST_DIRECTORY} Distribtest" -bazel test //:all || FAILED_TESTS="${FAILED_TESTS}C++ Distribtest" + popd +done if [ "$FAILED_TESTS" != "" ] then diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index 7724ca66c6f..e30516f5747 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -29,10 +29,3 @@ BAZEL_FLAGS="--test_output=errors" bazel test ${BAZEL_FLAGS} ${TEST_TARGETS} bazel test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS} bazel test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS} - - -# TODO(https://github.com/grpc/grpc/issues/19854): Move this to a new Kokoro -# job. -(cd /var/local/git/grpc/bazel/test/python_test_repo; - bazel test --test_output=errors //... -)