Merge pull request #17490 from lidizheng/py-status-3

Add grpcio-status extension package

requirements.bazel.txt

@@ -13,3 +13,4 @@ urllib3>=1.23
 chardet==3.0.4
 certifi==2017.4.17
 idna==2.7
+googleapis-common-protos==1.5.5

src/python/grpcio_status/.gitignore

@@ -0,0 +1,3 @@
+build/
+grpcio_status.egg-info/
+dist/

src/python/grpcio_status/MANIFEST.in

@@ -0,0 +1,4 @@
+include grpc_version.py
+recursive-include grpc_status *.py
+global-exclude *.pyc
+include LICENSE

src/python/grpcio_status/README.rst

@@ -0,0 +1,9 @@
+gRPC Python Status Proto
+===========================
+
+Reference package for GRPC Python status proto mapping.
+
+Dependencies
+------------
+
+Depends on the `grpcio` package, available from PyPI via `pip install grpcio`.

src/python/grpcio_status/grpc_status/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@grpc_python_dependencies//:requirements.bzl", "requirement")
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "grpc_status",
+    srcs = ["rpc_status.py",],
+    deps = [
+        "//src/python/grpcio/grpc:grpcio",
+        requirement('protobuf'),
+        requirement('googleapis-common-protos'),
+    ],
+    imports=["../",],
+)

src/python/grpcio_status/grpc_status/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/python/grpcio_status/grpc_status/rpc_status.py

@@ -0,0 +1,92 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Reference implementation for status mapping in gRPC Python."""
+
+import collections
+
+import grpc
+
+# TODO(https://github.com/bazelbuild/bazel/issues/6844)
+# Due to Bazel issue, the namespace packages won't resolve correctly.
+# Adding this unused-import as a workaround to avoid module-not-found error
+# under Bazel builds.
+import google.protobuf  # pylint: disable=unused-import
+from google.rpc import status_pb2
+
+_CODE_TO_GRPC_CODE_MAPPING = dict([(x.value[0], x) for x in grpc.StatusCode])
+
+_GRPC_DETAILS_METADATA_KEY = 'grpc-status-details-bin'
+
+
+class _Status(
+        collections.namedtuple(
+            '_Status', ('code', 'details', 'trailing_metadata')), grpc.Status):
+    pass
+
+
+def _code_to_grpc_status_code(code):
+    try:
+        return _CODE_TO_GRPC_CODE_MAPPING[code]
+    except KeyError:
+        raise ValueError('Invalid status code %s' % code)
+
+
+def from_call(call):
+    """Returns a google.rpc.status.Status message corresponding to a given grpc.Call.
+
+    This is an EXPERIMENTAL API.
+
+    Args:
+      call: A grpc.Call instance.
+
+    Returns:
+      A google.rpc.status.Status message representing the status of the RPC.
+
+    Raises:
+      ValueError: If the gRPC call's code or details are inconsistent with the
+        status code and message inside of the google.rpc.status.Status.
+    """
+    for key, value in call.trailing_metadata():
+        if key == _GRPC_DETAILS_METADATA_KEY:
+            rich_status = status_pb2.Status.FromString(value)
+            if call.code().value[0] != rich_status.code:
+                raise ValueError(
+                    'Code in Status proto (%s) doesn\'t match status code (%s)'
+                    % (_code_to_grpc_status_code(rich_status.code),
+                       call.code()))
+            if call.details() != rich_status.message:
+                raise ValueError(
+                    'Message in Status proto (%s) doesn\'t match status details (%s)'
+                    % (rich_status.message, call.details()))
+            return rich_status
+    return None
+
+
+def to_status(status):
+    """Convert a google.rpc.status.Status message to grpc.Status.
+
+    This is an EXPERIMENTAL API.
+
+    Args:
+      status: a google.rpc.status.Status message representing the non-OK status
+        to terminate the RPC with and communicate it to the client.
+
+    Returns:
+      A grpc.Status instance representing the input google.rpc.status.Status message.
+    """
+    return _Status(
+        code=_code_to_grpc_status_code(status.code),
+        details=status.message,
+        trailing_metadata=((_GRPC_DETAILS_METADATA_KEY,
+                            status.SerializeToString()),))

src/python/grpcio_status/grpc_version.py

@@ -0,0 +1,17 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!!
+
+VERSION = '1.18.0.dev0'

src/python/grpcio_status/setup.py

@@ -0,0 +1,93 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Setup module for the GRPC Python package's status mapping."""
+
+import os
+
+import setuptools
+
+# Ensure we're in the proper directory whether or not we're being used by pip.
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+# Break import-style to ensure we can actually find our local modules.
+import grpc_version
+
+
+class _NoOpCommand(setuptools.Command):
+    """No-op command."""
+
+    description = ''
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        pass
+
+
+CLASSIFIERS = [
+    'Development Status :: 5 - Production/Stable',
+    'Programming Language :: Python',
+    'Programming Language :: Python :: 2',
+    'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.4',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'License :: OSI Approved :: Apache Software License',
+]
+
+PACKAGE_DIRECTORIES = {
+    '': '.',
+}
+
+INSTALL_REQUIRES = (
+    'protobuf>=3.6.0',
+    'grpcio>={version}'.format(version=grpc_version.VERSION),
+    'googleapis-common-protos>=1.5.5',
+)
+
+try:
+    import status_commands as _status_commands
+    # we are in the build environment, otherwise the above import fails
+    COMMAND_CLASS = {
+        # Run preprocess from the repository *before* doing any packaging!
+        'preprocess': _status_commands.Preprocess,
+        'build_package_protos': _NoOpCommand,
+    }
+except ImportError:
+    COMMAND_CLASS = {
+        # wire up commands to no-op not to break the external dependencies
+        'preprocess': _NoOpCommand,
+        'build_package_protos': _NoOpCommand,
+    }
+
+setuptools.setup(
+    name='grpcio-status',
+    version=grpc_version.VERSION,
+    description='Status proto mapping for gRPC',
+    author='The gRPC Authors',
+    author_email='grpc-io@googlegroups.com',
+    url='https://grpc.io',
+    license='Apache License 2.0',
+    classifiers=CLASSIFIERS,
+    package_dir=PACKAGE_DIRECTORIES,
+    packages=setuptools.find_packages('.'),
+    install_requires=INSTALL_REQUIRES,
+    cmdclass=COMMAND_CLASS)

src/python/grpcio_status/status_commands.py

@@ -0,0 +1,39 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Provides distutils command classes for the GRPC Python setup process."""
+
+import os
+import shutil
+
+import setuptools
+
+ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
+LICENSE = os.path.join(ROOT_DIR, '../../../LICENSE')
+
+
+class Preprocess(setuptools.Command):
+    """Command to copy LICENSE from root directory."""
+
+    description = ''
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        if os.path.isfile(LICENSE):
+            shutil.copyfile(LICENSE, os.path.join(ROOT_DIR, 'LICENSE'))

src/python/grpcio_tests/setup.py

@@ -40,6 +40,7 @@ INSTALL_REQUIRES = (
     'coverage>=4.0', 'enum34>=1.0.4',
     'grpcio>={version}'.format(version=grpc_version.VERSION),
     'grpcio-channelz>={version}'.format(version=grpc_version.VERSION),
+    'grpcio-status>={version}'.format(version=grpc_version.VERSION),
     'grpcio-tools>={version}'.format(version=grpc_version.VERSION),
     'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION),
     'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0',

src/python/grpcio_tests/tests/status/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@grpc_python_dependencies//:requirements.bzl", "requirement")
+
+package(default_visibility = ["//visibility:public"])
+
+py_test(
+    name = "grpc_status_test",
+    srcs = ["_grpc_status_test.py"],
+    main = "_grpc_status_test.py",
+    size = "small",
+    deps = [
+        "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_status/grpc_status:grpc_status",
+        "//src/python/grpcio_tests/tests/unit:test_common",
+        "//src/python/grpcio_tests/tests/unit/framework/common:common",
+        requirement('protobuf'),
+        requirement('googleapis-common-protos'),
+    ],
+    imports = ["../../",],
+)

src/python/grpcio_tests/tests/status/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/python/grpcio_tests/tests/status/_grpc_status_test.py

@@ -0,0 +1,173 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests of grpc_status."""
+
+import unittest
+
+import logging
+import traceback
+
+import grpc
+from grpc_status import rpc_status
+
+from tests.unit import test_common
+
+from google.protobuf import any_pb2
+from google.rpc import code_pb2, status_pb2, error_details_pb2
+
+_STATUS_OK = '/test/StatusOK'
+_STATUS_NOT_OK = '/test/StatusNotOk'
+_ERROR_DETAILS = '/test/ErrorDetails'
+_INCONSISTENT = '/test/Inconsistent'
+_INVALID_CODE = '/test/InvalidCode'
+
+_REQUEST = b'\x00\x00\x00'
+_RESPONSE = b'\x01\x01\x01'
+
+_GRPC_DETAILS_METADATA_KEY = 'grpc-status-details-bin'
+
+_STATUS_DETAILS = 'This is an error detail'
+_STATUS_DETAILS_ANOTHER = 'This is another error detail'
+
+
+def _ok_unary_unary(request, servicer_context):
+    return _RESPONSE
+
+
+def _not_ok_unary_unary(request, servicer_context):
+    servicer_context.abort(grpc.StatusCode.INTERNAL, _STATUS_DETAILS)
+
+
+def _error_details_unary_unary(request, servicer_context):
+    details = any_pb2.Any()
+    details.Pack(
+        error_details_pb2.DebugInfo(
+            stack_entries=traceback.format_stack(),
+            detail='Intentionally invoked'))
+    rich_status = status_pb2.Status(
+        code=code_pb2.INTERNAL,
+        message=_STATUS_DETAILS,
+        details=[details],
+    )
+    servicer_context.abort_with_status(rpc_status.to_status(rich_status))
+
+
+def _inconsistent_unary_unary(request, servicer_context):
+    rich_status = status_pb2.Status(
+        code=code_pb2.INTERNAL,
+        message=_STATUS_DETAILS,
+    )
+    servicer_context.set_code(grpc.StatusCode.NOT_FOUND)
+    servicer_context.set_details(_STATUS_DETAILS_ANOTHER)
+    # User put inconsistent status information in trailing metadata
+    servicer_context.set_trailing_metadata(((_GRPC_DETAILS_METADATA_KEY,
+                                             rich_status.SerializeToString()),))
+
+
+def _invalid_code_unary_unary(request, servicer_context):
+    rich_status = status_pb2.Status(
+        code=42,
+        message='Invalid code',
+    )
+    servicer_context.abort_with_status(rpc_status.to_status(rich_status))
+
+
+class _GenericHandler(grpc.GenericRpcHandler):
+
+    def service(self, handler_call_details):
+        if handler_call_details.method == _STATUS_OK:
+            return grpc.unary_unary_rpc_method_handler(_ok_unary_unary)
+        elif handler_call_details.method == _STATUS_NOT_OK:
+            return grpc.unary_unary_rpc_method_handler(_not_ok_unary_unary)
+        elif handler_call_details.method == _ERROR_DETAILS:
+            return grpc.unary_unary_rpc_method_handler(
+                _error_details_unary_unary)
+        elif handler_call_details.method == _INCONSISTENT:
+            return grpc.unary_unary_rpc_method_handler(
+                _inconsistent_unary_unary)
+        elif handler_call_details.method == _INVALID_CODE:
+            return grpc.unary_unary_rpc_method_handler(
+                _invalid_code_unary_unary)
+        else:
+            return None
+
+
+class StatusTest(unittest.TestCase):
+
+    def setUp(self):
+        self._server = test_common.test_server()
+        self._server.add_generic_rpc_handlers((_GenericHandler(),))
+        port = self._server.add_insecure_port('[::]:0')
+        self._server.start()
+
+        self._channel = grpc.insecure_channel('localhost:%d' % port)
+
+    def tearDown(self):
+        self._server.stop(None)
+        self._channel.close()
+
+    def test_status_ok(self):
+        _, call = self._channel.unary_unary(_STATUS_OK).with_call(_REQUEST)
+
+        # Succeed RPC doesn't have status
+        status = rpc_status.from_call(call)
+        self.assertIs(status, None)
+
+    def test_status_not_ok(self):
+        with self.assertRaises(grpc.RpcError) as exception_context:
+            self._channel.unary_unary(_STATUS_NOT_OK).with_call(_REQUEST)
+        rpc_error = exception_context.exception
+
+        self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL)
+        # Failed RPC doesn't automatically generate status
+        status = rpc_status.from_call(rpc_error)
+        self.assertIs(status, None)
+
+    def test_error_details(self):
+        with self.assertRaises(grpc.RpcError) as exception_context:
+            self._channel.unary_unary(_ERROR_DETAILS).with_call(_REQUEST)
+        rpc_error = exception_context.exception
+
+        status = rpc_status.from_call(rpc_error)
+        self.assertEqual(rpc_error.code(), grpc.StatusCode.INTERNAL)
+        self.assertEqual(status.code, code_pb2.Code.Value('INTERNAL'))
+
+        # Check if the underlying proto message is intact
+        self.assertEqual(status.details[0].Is(
+            error_details_pb2.DebugInfo.DESCRIPTOR), True)
+        info = error_details_pb2.DebugInfo()
+        status.details[0].Unpack(info)
+        self.assertIn('_error_details_unary_unary', info.stack_entries[-1])
+
+    def test_code_message_validation(self):
+        with self.assertRaises(grpc.RpcError) as exception_context:
+            self._channel.unary_unary(_INCONSISTENT).with_call(_REQUEST)
+        rpc_error = exception_context.exception
+        self.assertEqual(rpc_error.code(), grpc.StatusCode.NOT_FOUND)
+
+        # Code/Message validation failed
+        self.assertRaises(ValueError, rpc_status.from_call, rpc_error)
+
+    def test_invalid_code(self):
+        with self.assertRaises(grpc.RpcError) as exception_context:
+            self._channel.unary_unary(_INVALID_CODE).with_call(_REQUEST)
+        rpc_error = exception_context.exception
+        self.assertEqual(rpc_error.code(), grpc.StatusCode.UNKNOWN)
+        # Invalid status code exception raised during coversion
+        self.assertIn('Invalid status code', rpc_error.details())
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    unittest.main(verbosity=2)

src/python/grpcio_tests/tests/tests.json

@@ -15,6 +15,7 @@
   "protoc_plugin._split_definitions_test.SplitProtoSingleProtocExecutionProtocStyleTest",
   "protoc_plugin.beta_python_plugin_test.PythonPluginTest",
   "reflection._reflection_servicer_test.ReflectionServicerTest",
+  "status._grpc_status_test.StatusTest",
   "testing._client_test.ClientTest",
   "testing._server_test.FirstServiceServicerTest",
   "testing._time_test.StrictFakeTimeTest",

templates/src/python/grpcio_status/grpc_version.py.template

@@ -0,0 +1,19 @@
+%YAML 1.2
+--- |
+  # Copyright 2018 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_status/grpc_version.py.template`!!!
+
+  VERSION = '${settings.python_version.pep440()}'

tools/distrib/pylint_code.sh

@@ -24,6 +24,7 @@ DIRS=(
     'src/python/grpcio_health_checking/grpc_health'
     'src/python/grpcio_reflection/grpc_reflection'
     'src/python/grpcio_testing/grpc_testing'
+    'src/python/grpcio_status/grpc_status'
 )
 
 TEST_DIRS=(

tools/run_tests/artifacts/build_artifact_python.sh

@@ -123,6 +123,11 @@ then
   ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_reflection/setup.py \
       preprocess build_package_protos sdist
   cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR"
+
+  # Build grpcio_status source distribution
+  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_status/setup.py \
+      preprocess sdist
+  cp -r src/python/grpcio_status/dist/* "$ARTIFACT_DIR"
 fi
 
 cp -r dist/* "$ARTIFACT_DIR"

tools/run_tests/helper_scripts/build_python.sh

@@ -204,12 +204,18 @@ $VENV_PYTHON "$ROOT/src/python/grpcio_reflection/setup.py" preprocess
 $VENV_PYTHON "$ROOT/src/python/grpcio_reflection/setup.py" build_package_protos
 pip_install_dir "$ROOT/src/python/grpcio_reflection"
 
+# Build/install status proto mapping
+$VENV_PYTHON "$ROOT/src/python/grpcio_status/setup.py" preprocess
+$VENV_PYTHON "$ROOT/src/python/grpcio_status/setup.py" build_package_protos
+pip_install_dir "$ROOT/src/python/grpcio_status"
+
 # Install testing
 pip_install_dir "$ROOT/src/python/grpcio_testing"
 
 # Build/install tests
 $VENV_PYTHON -m pip install coverage==4.4 oauth2client==4.1.0 \
-                            google-auth==1.0.0 requests==2.14.2
+                            google-auth==1.0.0 requests==2.14.2 \
+                            googleapis-common-protos==1.5.5
 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess
 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" build_package_protos
 pip_install_dir "$ROOT/src/python/grpcio_tests"