From dc63d6a53e8b9f2e993ee8f9702acae398f025fc Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 3 May 2021 11:54:34 -0700 Subject: [PATCH] Add CSDS API to Python (#26114) * Add grpcio-csds pacakge * Remove unused file * Fix the proto import path issue * Update the CSDS package and xds-protos for PY2 * Make tests happy * Fix Bazel proto dependency * Add Python2 tests for CSDS --- src/proto/grpc/testing/xds/v3/BUILD | 27 ++++ .../grpcio/grpc/_cython/_cygrpc/csds.pyx.pxi | 21 +++ .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 2 + src/python/grpcio/grpc/_cython/cygrpc.pyx | 1 + src/python/grpcio_csds/.gitignore | 6 + src/python/grpcio_csds/MANIFEST.in | 4 + src/python/grpcio_csds/README.rst | 10 ++ src/python/grpcio_csds/grpc_csds/BUILD.bazel | 31 ++++ src/python/grpcio_csds/grpc_csds/__init__.py | 49 +++++++ src/python/grpcio_csds/grpc_version.py | 17 +++ src/python/grpcio_csds/setup.py | 61 ++++++++ .../grpcio_tests/tests/csds/BUILD.bazel | 27 ++++ .../grpcio_tests/tests/csds/test_csds.py | 134 ++++++++++++++++++ .../grpcio_csds/grpc_version.py.template | 19 +++ .../distrib/python/grpc_prefixed/generate.py | 5 + tools/distrib/python/xds_protos/build.py | 24 ++-- .../xds_protos/build_validate_upload.sh | 14 +- tools/distrib/python/xds_protos/setup.py | 7 +- .../artifacts/build_artifact_python.sh | 20 ++- .../run_tests/helper_scripts/build_python.sh | 3 + 20 files changed, 460 insertions(+), 22 deletions(-) create mode 100644 src/python/grpcio/grpc/_cython/_cygrpc/csds.pyx.pxi create mode 100644 src/python/grpcio_csds/.gitignore create mode 100644 src/python/grpcio_csds/MANIFEST.in create mode 100644 src/python/grpcio_csds/README.rst create mode 100644 src/python/grpcio_csds/grpc_csds/BUILD.bazel create mode 100644 src/python/grpcio_csds/grpc_csds/__init__.py create mode 100644 src/python/grpcio_csds/grpc_version.py create mode 100644 src/python/grpcio_csds/setup.py create mode 100644 src/python/grpcio_tests/tests/csds/BUILD.bazel create mode 100644 src/python/grpcio_tests/tests/csds/test_csds.py create mode 100644 templates/src/python/grpcio_csds/grpc_version.py.template diff --git a/src/proto/grpc/testing/xds/v3/BUILD b/src/proto/grpc/testing/xds/v3/BUILD index f7a27c804e8..be730deaba1 100644 --- a/src/proto/grpc/testing/xds/v3/BUILD +++ b/src/proto/grpc/testing/xds/v3/BUILD @@ -15,6 +15,7 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_package", "grpc_proto_library") +load("//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library") grpc_package( name = "xds_v3", @@ -278,3 +279,29 @@ grpc_proto_library( "route_proto", ], ) + +py_proto_library( + name = "csds_py_pb2", + deps = [":_csds_proto_only"], +) + +py_grpc_library( + name = "csds_py_pb2_grpc", + srcs = [":_csds_proto_only"], + deps = [":csds_py_pb2"], +) + +py_proto_library( + name = "config_dump_py_pb2", + deps = [":_config_dump_proto_only"], +) + +py_proto_library( + name = "base_py_pb2", + deps = [":_base_proto_only"], +) + +py_proto_library( + name = "percent_py_pb2", + deps = [":_percent_proto_only"], +) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/csds.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/csds.pyx.pxi new file mode 100644 index 00000000000..c33eb76e47f --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/csds.pyx.pxi @@ -0,0 +1,21 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def dump_xds_configs(): + cdef grpc_slice client_config_in_slice + with nogil: + client_config_in_slice = grpc_dump_xds_configs() + cdef bytes result = _slice_bytes(client_config_in_slice) + return result diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 48f3461dd19..fb8ceae6a5d 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -433,6 +433,8 @@ cdef extern from "grpc/grpc.h": char* grpc_channelz_get_subchannel(intptr_t subchannel_id) char* grpc_channelz_get_socket(intptr_t socket_id) + grpc_slice grpc_dump_xds_configs() nogil + cdef extern from "grpc/grpc_security.h": diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx index 0ce8bda0d89..b958975bb80 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pyx +++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx @@ -41,6 +41,7 @@ include "_cygrpc/arguments.pyx.pxi" include "_cygrpc/call.pyx.pxi" include "_cygrpc/channel.pyx.pxi" include "_cygrpc/channelz.pyx.pxi" +include "_cygrpc/csds.pyx.pxi" include "_cygrpc/credentials.pyx.pxi" include "_cygrpc/completion_queue.pyx.pxi" include "_cygrpc/event.pyx.pxi" diff --git a/src/python/grpcio_csds/.gitignore b/src/python/grpcio_csds/.gitignore new file mode 100644 index 00000000000..0c5da6b5af1 --- /dev/null +++ b/src/python/grpcio_csds/.gitignore @@ -0,0 +1,6 @@ +*.proto +*_pb2.py +*_pb2_grpc.py +build/ +grpcio_channelz.egg-info/ +dist/ diff --git a/src/python/grpcio_csds/MANIFEST.in b/src/python/grpcio_csds/MANIFEST.in new file mode 100644 index 00000000000..4ecdf7428ed --- /dev/null +++ b/src/python/grpcio_csds/MANIFEST.in @@ -0,0 +1,4 @@ +include grpc_version.py +recursive-include grpc_csds *.py +global-exclude *.pyc +include LICENSE diff --git a/src/python/grpcio_csds/README.rst b/src/python/grpcio_csds/README.rst new file mode 100644 index 00000000000..f8803f253d8 --- /dev/null +++ b/src/python/grpcio_csds/README.rst @@ -0,0 +1,10 @@ +gRPC Python Client Status Discovery Service package +=================================================== + +CSDS is part of the Envoy xDS protocol: +https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/status/v3/csds.proto. +It allows the gRPC application to programmatically expose the received traffic +configuration (xDS resources). Welcome to explore with CLI tool "grpcdebug": +https://github.com/grpc-ecosystem/grpcdebug. + +For any issues or suggestions, please send to https://github.com/grpc/grpc/issues. diff --git a/src/python/grpcio_csds/grpc_csds/BUILD.bazel b/src/python/grpcio_csds/grpc_csds/BUILD.bazel new file mode 100644 index 00000000000..2e799ef58ab --- /dev/null +++ b/src/python/grpcio_csds/grpc_csds/BUILD.bazel @@ -0,0 +1,31 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "grpc_csds", + srcs = glob(["*.py"]), + imports = ["../"], + deps = [ + "//src/proto/grpc/testing/xds/v3:base_py_pb2", + "//src/proto/grpc/testing/xds/v3:config_dump_py_pb2", + "//src/proto/grpc/testing/xds/v3:csds_py_pb2", + "//src/proto/grpc/testing/xds/v3:csds_py_pb2_grpc", + "//src/proto/grpc/testing/xds/v3:percent_py_pb2", + "//src/python/grpcio/grpc:grpcio", + ], +) diff --git a/src/python/grpcio_csds/grpc_csds/__init__.py b/src/python/grpcio_csds/grpc_csds/__init__.py new file mode 100644 index 00000000000..ed463b7262c --- /dev/null +++ b/src/python/grpcio_csds/grpc_csds/__init__.py @@ -0,0 +1,49 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Channelz debug service implementation in gRPC Python.""" + +from grpc._cython import cygrpc + +from google.protobuf import json_format +try: + from envoy.service.status.v3 import csds_pb2, csds_pb2_grpc +except ImportError: + from src.proto.grpc.testing.xds.v3 import csds_pb2, csds_pb2_grpc + + +class ClientStatusDiscoveryServiceServicer( + csds_pb2_grpc.ClientStatusDiscoveryServiceServicer): + """CSDS Servicer works for both the sync API and asyncio API.""" + + @staticmethod + def FetchClientStatus(request, unused_context): + client_config = csds_pb2.ClientConfig.FromString( + cygrpc.dump_xds_configs()) + response = csds_pb2.ClientStatusResponse() + response.config.append(client_config) + return response + + @staticmethod + def StreamClientStatus(request_iterator, context): + for request in request_iterator: + yield ClientStatusDiscoveryServiceServicer.FetchClientStatus( + request, context) + + +def add_csds_servicer(server): + csds_pb2_grpc.add_ClientStatusDiscoveryServiceServicer_to_server( + ClientStatusDiscoveryServiceServicer(), server) + + +__all__ = ['ClientStatusDiscoveryServiceServicer', 'add_csds_servicer'] diff --git a/src/python/grpcio_csds/grpc_version.py b/src/python/grpcio_csds/grpc_version.py new file mode 100644 index 00000000000..510a148a9ef --- /dev/null +++ b/src/python/grpcio_csds/grpc_version.py @@ -0,0 +1,17 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_csds/grpc_version.py.template`!!! + +VERSION = '1.38.0.dev0' diff --git a/src/python/grpcio_csds/setup.py b/src/python/grpcio_csds/setup.py new file mode 100644 index 00000000000..c699fc7d921 --- /dev/null +++ b/src/python/grpcio_csds/setup.py @@ -0,0 +1,61 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Setup module for CSDS in gRPC Python.""" + +import os +import sys + +import setuptools + +_PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__)) +_README_PATH = os.path.join(_PACKAGE_PATH, 'README.rst') + +# Ensure we're in the proper directory whether or not we're being used by pip. +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +# Break import-style to ensure we can actually find our local modules. +import grpc_version + +CLASSIFIERS = [ + 'Development Status :: 5 - Production/Stable', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: Apache Software License', +] + +PACKAGE_DIRECTORIES = { + '': '.', +} + +INSTALL_REQUIRES = ( + 'protobuf>=3.6.0', + 'xds-protos>=0.0.7', + 'grpcio>={version}'.format(version=grpc_version.VERSION), +) +SETUP_REQUIRES = INSTALL_REQUIRES + +setuptools.setup(name='grpcio-csds', + version=grpc_version.VERSION, + license='Apache License 2.0', + description='xDS configuration dump library', + long_description=open(_README_PATH, 'r').read(), + author='The gRPC Authors', + author_email='grpc-io@googlegroups.com', + classifiers=CLASSIFIERS, + url='https://grpc.io', + package_dir=PACKAGE_DIRECTORIES, + packages=setuptools.find_packages('.'), + install_requires=INSTALL_REQUIRES, + setup_requires=SETUP_REQUIRES) diff --git a/src/python/grpcio_tests/tests/csds/BUILD.bazel b/src/python/grpcio_tests/tests/csds/BUILD.bazel new file mode 100644 index 00000000000..3cf916d727d --- /dev/null +++ b/src/python/grpcio_tests/tests/csds/BUILD.bazel @@ -0,0 +1,27 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:python_rules.bzl", "py2and3_test") + +py2and3_test( + name = "test_csds", + size = "small", + srcs = ["test_csds.py"], + main = "test_csds.py", + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_csds/grpc_csds", + "@six", + ], +) diff --git a/src/python/grpcio_tests/tests/csds/test_csds.py b/src/python/grpcio_tests/tests/csds/test_csds.py new file mode 100644 index 00000000000..f9709435923 --- /dev/null +++ b/src/python/grpcio_tests/tests/csds/test_csds.py @@ -0,0 +1,134 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A simple test to ensure that the Python wrapper can get xDS config.""" + +import logging +import os +import time +from six.moves import queue +import unittest +from concurrent.futures import ThreadPoolExecutor + +import grpc +import grpc_csds + +from google.protobuf import json_format +try: + from envoy.service.status.v3 import csds_pb2, csds_pb2_grpc +except ImportError: + from src.proto.grpc.testing.xds.v3 import csds_pb2, csds_pb2_grpc + +_DUMMY_XDS_ADDRESS = 'xds:///foo.bar' +_DUMMY_BOOTSTRAP_FILE = """ +{ + \"xds_servers\": [ + { + \"server_uri\": \"fake:///xds_server\", + \"channel_creds\": [ + { + \"type\": \"fake\" + } + ], + \"server_features\": [\"xds_v3\"] + } + ], + \"node\": { + \"id\": \"python_test_csds\", + \"cluster\": \"test\", + \"metadata\": { + \"foo\": \"bar\" + }, + \"locality\": { + \"region\": \"corp\", + \"zone\": \"svl\", + \"sub_zone\": \"mp3\" + } + } +}\ +""" + + +class TestCsds(unittest.TestCase): + + def setUp(self): + os.environ['GRPC_XDS_BOOTSTRAP_CONFIG'] = _DUMMY_BOOTSTRAP_FILE + self._server = grpc.server(ThreadPoolExecutor()) + port = self._server.add_insecure_port('localhost:0') + grpc_csds.add_csds_servicer(self._server) + self._server.start() + + self._channel = grpc.insecure_channel('localhost:%s' % port) + self._stub = csds_pb2_grpc.ClientStatusDiscoveryServiceStub( + self._channel) + + def tearDown(self): + self._channel.close() + self._server.stop(0) + os.environ.pop('GRPC_XDS_BOOTSTRAP_CONFIG', None) + + def get_xds_config_dump(self): + return self._stub.FetchClientStatus(csds_pb2.ClientStatusRequest()) + + def test_has_node(self): + resp = self.get_xds_config_dump() + self.assertEqual(1, len(resp.config)) + self.assertEqual(4, len(resp.config[0].xds_config)) + self.assertEqual('python_test_csds', resp.config[0].node.id) + self.assertEqual('test', resp.config[0].node.cluster) + + def test_no_lds_found(self): + dummy_channel = grpc.insecure_channel(_DUMMY_XDS_ADDRESS) + + # Force the XdsClient to initialize and request a resource + with self.assertRaises(grpc.RpcError) as rpc_error: + dummy_channel.unary_unary('')(b'', wait_for_ready=False) + self.assertEqual(grpc.StatusCode.UNAVAILABLE, + rpc_error.exception.code()) + + # The resource request will fail with DOES_NOT_EXIST (after 15s) + while True: + resp = self.get_xds_config_dump() + config = json_format.MessageToDict(resp) + ok = False + try: + for xds_config in config["config"][0]["xdsConfig"]: + if "listenerConfig" in xds_config: + listener = xds_config["listenerConfig"][ + "dynamicListeners"][0] + if listener['clientStatus'] == 'DOES_NOT_EXIST': + ok = True + break + except KeyError as e: + logging.debug("Invalid config: %s\n%s: %s", config, type(e), e) + pass + if ok: + break + time.sleep(1) + dummy_channel.close() + + +class TestCsdsStream(TestCsds): + + def get_xds_config_dump(self): + if not hasattr(self, 'request_queue'): + request_queue = queue.Queue() + response_iterator = self._stub.StreamClientStatus( + iter(request_queue.get, None)) + request_queue.put(csds_pb2.ClientStatusRequest()) + return next(response_iterator) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + unittest.main(verbosity=2) diff --git a/templates/src/python/grpcio_csds/grpc_version.py.template b/templates/src/python/grpcio_csds/grpc_version.py.template new file mode 100644 index 00000000000..b4a497a5ec9 --- /dev/null +++ b/templates/src/python/grpcio_csds/grpc_version.py.template @@ -0,0 +1,19 @@ +%YAML 1.2 +--- | + # Copyright 2021 The gRPC Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_csds/grpc_version.py.template`!!! + + VERSION = '${settings.python_version.pep440()}' diff --git a/tools/distrib/python/grpc_prefixed/generate.py b/tools/distrib/python/grpc_prefixed/generate.py index 864971f5c16..e383df994c9 100644 --- a/tools/distrib/python/grpc_prefixed/generate.py +++ b/tools/distrib/python/grpc_prefixed/generate.py @@ -134,6 +134,11 @@ def main(): name_long='gRPC Health Checking', destination_package='grpcio-health-checking')) + generate_package( + PackageMeta(name='grpc-csds', + name_long='gRPC Client Status Discovery Service', + destination_package='grpcio-csds')) + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) diff --git a/tools/distrib/python/xds_protos/build.py b/tools/distrib/python/xds_protos/build.py index 69cdf288205..8b793f2bc8a 100644 --- a/tools/distrib/python/xds_protos/build.py +++ b/tools/distrib/python/xds_protos/build.py @@ -43,6 +43,10 @@ OUTPUT_PATH = WORK_DIR TEST_FILE_NAME = 'generated_file_import_test.py' TEST_IMPORTS = [] +# The pkgutil-style namespace packaging __init__.py +PKGUTIL_STYLE_INIT = "__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n" +NAMESPACE_PACKAGES = ["google"] + def add_test_import(proto_package_path: str, file_name: str, @@ -95,6 +99,15 @@ def compile_protos(proto_root: str, sub_dir: str = '.') -> None: raise Exception('error: {} failed'.format(COMPILE_BOTH)) +def create_init_file(path: str, package_path: str = "") -> None: + with open(os.path.join(path, "__init__.py"), 'w') as f: + # Apply the pkgutil-style namespace packaging, which is compatible for 2 + # and 3. Here is the full table of namespace compatibility: + # https://github.com/pypa/sample-namespace-packages/blob/master/table.md + if package_path in NAMESPACE_PACKAGES: + f.write(PKGUTIL_STYLE_INIT) + + def main(): # Compile xDS protos compile_protos(XDS_PROTO_ROOT) @@ -109,22 +122,13 @@ def main(): compile_protos(OPENCENSUS_PROTO_ROOT) # Generate __init__.py files for all modules - def create_init_file(path: str) -> None: - f = open(os.path.join(path, "__init__.py"), 'w') - f.close() - create_init_file(WORK_DIR) for proto_root_module in [ 'envoy', 'google', 'opencensus', 'udpa', 'validate', 'xds' ]: for root, _, _ in os.walk(os.path.join(WORK_DIR, proto_root_module)): package_path = os.path.relpath(root, WORK_DIR) - if package_path == "google": - # Google packages are namespace packages. We don't want to create a - # package named "google", which will create many trouble down the - # line. - continue - create_init_file(root) + create_init_file(root, package_path) # Generate test file with open(os.path.join(WORK_DIR, TEST_FILE_NAME), 'w') as f: diff --git a/tools/distrib/python/xds_protos/build_validate_upload.sh b/tools/distrib/python/xds_protos/build_validate_upload.sh index 78710e9d604..4218940e7da 100755 --- a/tools/distrib/python/xds_protos/build_validate_upload.sh +++ b/tools/distrib/python/xds_protos/build_validate_upload.sh @@ -18,18 +18,28 @@ set -ex WORK_DIR=$(pwd)/"$(dirname "$0")" cd ${WORK_DIR} +# Remove existing wheels +rm -rf ${WORK_DIR}/dist + # Generate the package content then build the source wheel python3 build.py python3 setup.py sdist +python2 setup.py bdist_wheel python3 setup.py bdist_wheel # Run the tests to ensure all protos are importable, also avoid confusing normal # imports with relative imports pushd $(mktemp -d '/tmp/test_xds_protos.XXXXXX') +python2 -m virtualenv env +env/bin/python -m pip install ${WORK_DIR}/dist/xds-protos-*.tar.gz +cp ${WORK_DIR}/generated_file_import_test.py generated_file_import_test.py +env/bin/python generated_file_import_test.py +popd +pushd $(mktemp -d '/tmp/test_xds_protos.XXXXXX') python3 -m virtualenv env -env/bin/python3 -m pip install ${WORK_DIR}/dist/xds-protos-*.tar.gz +env/bin/python -m pip install ${WORK_DIR}/dist/xds-protos-*.tar.gz cp ${WORK_DIR}/generated_file_import_test.py generated_file_import_test.py -env/bin/python3 generated_file_import_test.py +env/bin/python generated_file_import_test.py popd # Upload the package diff --git a/tools/distrib/python/xds_protos/setup.py b/tools/distrib/python/xds_protos/setup.py index a775f89eb23..2ed04c421a4 100644 --- a/tools/distrib/python/xds_protos/setup.py +++ b/tools/distrib/python/xds_protos/setup.py @@ -23,10 +23,7 @@ EXCLUDE_PYTHON_FILES = ['generated_file_import_test.py', 'build.py'] # Use setuptools to build Python package with open(os.path.join(WORK_DIR, 'README.rst'), 'r') as f: LONG_DESCRIPTION = f.read() -PACKAGES = setuptools.find_packages( - where=".", - exclude=EXCLUDE_PYTHON_FILES) + setuptools.find_namespace_packages( - include=['google.*']) +PACKAGES = setuptools.find_packages(where=".", exclude=EXCLUDE_PYTHON_FILES) CLASSIFIERS = [ 'Development Status :: 3 - Alpha', 'Programming Language :: Python', @@ -41,7 +38,7 @@ INSTALL_REQUIRES = [ SETUP_REQUIRES = INSTALL_REQUIRES + ['grpcio-tools'] setuptools.setup( name='xds-protos', - version='0.0.5', + version='0.0.8', packages=PACKAGES, description='Generated Python code from envoyproxy/data-plane-api', long_description_content_type='text/x-rst', diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 7e5abcc4ab3..0cc6e00b1f5 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -154,30 +154,40 @@ then "${PIP}" install grpcio --no-index --find-links "file://$ARTIFACT_DIR/" "${PIP}" install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/" + # Note(lidiz) setuptools's "sdist" command creates a source tarball, which + # demands an extra step of building the wheel. The building step is merely ran + # through setup.py, but we can optimize it with "bdist_wheel" command, which + # skips the wheel building step. + # Build grpcio_testing source distribution ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_testing/setup.py preprocess \ - sdist + sdist bdist_wheel cp -r src/python/grpcio_testing/dist/* "$ARTIFACT_DIR" # Build grpcio_channelz source distribution ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_channelz/setup.py \ - preprocess build_package_protos sdist + preprocess build_package_protos sdist bdist_wheel cp -r src/python/grpcio_channelz/dist/* "$ARTIFACT_DIR" # Build grpcio_health_checking source distribution ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_health_checking/setup.py \ - preprocess build_package_protos sdist + preprocess build_package_protos sdist bdist_wheel cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR" # Build grpcio_reflection source distribution ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_reflection/setup.py \ - preprocess build_package_protos sdist + preprocess build_package_protos sdist bdist_wheel cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR" # Build grpcio_status source distribution ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_status/setup.py \ - preprocess sdist + preprocess sdist bdist_wheel cp -r src/python/grpcio_status/dist/* "$ARTIFACT_DIR" + + # Build grpcio_csds source distribution + ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_csds/setup.py \ + sdist bdist_wheel + cp -r src/python/grpcio_csds/dist/* "$ARTIFACT_DIR" fi if [ "$GRPC_SKIP_TWINE_CHECK" == "" ] diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 408d5f5b8d5..7f95d79477d 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -220,6 +220,9 @@ $VENV_PYTHON "$ROOT/src/python/grpcio_status/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_status/setup.py" build_package_protos pip_install_dir "$ROOT/src/python/grpcio_status" +# Build/install csds +pip_install_dir "$ROOT/src/python/grpcio_csds" + # Install testing pip_install_dir "$ROOT/src/python/grpcio_testing"