From 707c9e29e08f1db8f27253c1951ff62b55561e13 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Wed, 20 Apr 2016 09:42:19 -0700 Subject: [PATCH] Changed python proto build to use shared protos --- src/python/.gitignore | 1 + src/python/grpcio/commands.py | 50 ++++-- .../tests/interop/_insecure_interop_test.py | 2 +- .../tests/interop/_secure_interop_test.py | 2 +- src/python/grpcio/tests/interop/client.py | 2 +- src/python/grpcio/tests/interop/empty.proto | 43 ----- .../grpcio/tests/interop/messages.proto | 167 ------------------ src/python/grpcio/tests/interop/methods.py | 6 +- src/python/grpcio/tests/interop/server.py | 2 +- src/python/grpcio/tests/interop/test.proto | 86 --------- tools/run_tests/run_interop_tests.py | 3 +- tools/run_tests/run_tests.py | 1 + 12 files changed, 45 insertions(+), 320 deletions(-) create mode 100644 src/python/.gitignore delete mode 100644 src/python/grpcio/tests/interop/empty.proto delete mode 100644 src/python/grpcio/tests/interop/messages.proto delete mode 100644 src/python/grpcio/tests/interop/test.proto diff --git a/src/python/.gitignore b/src/python/.gitignore new file mode 100644 index 00000000000..f158efa4bf1 --- /dev/null +++ b/src/python/.gitignore @@ -0,0 +1 @@ +gens/ diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 9e745701c14..295dab2d274 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -50,6 +50,9 @@ from setuptools.command import test import support PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) +GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../') +PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto') +PROTO_GEN_STEM = os.path.join(GRPC_STEM, 'src', 'python', 'gens') CONF_PY_ADDENDUM = """ extensions.append('sphinx.ext.napoleon') @@ -157,30 +160,45 @@ class BuildProtoModules(setuptools.Command): if not self.grpc_python_plugin_command: raise CommandError('could not find grpc_python_plugin ' '(protoc plugin for GRPC Python)') + + if not os.path.exists(PROTO_GEN_STEM): + os.makedirs(PROTO_GEN_STEM) + include_regex = re.compile(self.include) exclude_regex = re.compile(self.exclude) if self.exclude else None paths = [] - root_directory = PYTHON_STEM - for walk_root, directories, filenames in os.walk(root_directory): + for walk_root, directories, filenames in os.walk(PROTO_STEM): for filename in filenames: path = os.path.join(walk_root, filename) if include_regex.match(path) and not ( exclude_regex and exclude_regex.match(path)): paths.append(path) - command = [ - self.protoc_command, - '--plugin=protoc-gen-python-grpc={}'.format( - self.grpc_python_plugin_command), - '-I {}'.format(root_directory), - '--python_out={}'.format(root_directory), - '--python-grpc_out={}'.format(root_directory), - ] + paths - try: - subprocess.check_output(' '.join(command), cwd=root_directory, shell=True, - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise CommandError('Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format( - command, e.message, e.output)) + + # TODO(kpayson): It would be nice to do this in a batch command, + # but we currently have name conflicts in src/proto + for path in paths: + command = [ + self.protoc_command, + '--plugin=protoc-gen-python-grpc={}'.format( + self.grpc_python_plugin_command), + '-I {}'.format(GRPC_STEM), + '--python_out={}'.format(PROTO_GEN_STEM), + '--python-grpc_out={}'.format(PROTO_GEN_STEM), + ] + [path] + try: + subprocess.check_output(' '.join(command), cwd=PYTHON_STEM, shell=True, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + sys.stderr.write( + 'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format( + command, e.message, e.output)) + + # Generated proto directories dont include __init__.py, but + # these are needed for python package resolution + for walk_root, _, _ in os.walk(PROTO_GEN_STEM): + if walk_root != PROTO_GEN_STEM: + path = os.path.join(walk_root, '__init__.py') + open(path, 'a').close() class BuildProjectMetadata(setuptools.Command): diff --git a/src/python/grpcio/tests/interop/_insecure_interop_test.py b/src/python/grpcio/tests/interop/_insecure_interop_test.py index 00b49aba374..91519b6fba2 100644 --- a/src/python/grpcio/tests/interop/_insecure_interop_test.py +++ b/src/python/grpcio/tests/interop/_insecure_interop_test.py @@ -32,11 +32,11 @@ import unittest from grpc.beta import implementations +from src.proto.grpc.testing import test_pb2 from tests.interop import _interop_test_case from tests.interop import methods from tests.interop import server -from tests.interop import test_pb2 class InsecureInteropTest( diff --git a/src/python/grpcio/tests/interop/_secure_interop_test.py b/src/python/grpcio/tests/interop/_secure_interop_test.py index 86d7e433512..c61547b9778 100644 --- a/src/python/grpcio/tests/interop/_secure_interop_test.py +++ b/src/python/grpcio/tests/interop/_secure_interop_test.py @@ -32,11 +32,11 @@ import unittest from grpc.beta import implementations +from src.proto.grpc.testing import test_pb2 from tests.interop import _interop_test_case from tests.interop import methods from tests.interop import resources -from tests.interop import test_pb2 from tests.unit.beta import test_utilities diff --git a/src/python/grpcio/tests/interop/client.py b/src/python/grpcio/tests/interop/client.py index 1d10d7e45de..db29eb4aa76 100644 --- a/src/python/grpcio/tests/interop/client.py +++ b/src/python/grpcio/tests/interop/client.py @@ -33,10 +33,10 @@ import argparse from oauth2client import client as oauth2client_client from grpc.beta import implementations +from src.proto.grpc.testing import test_pb2 from tests.interop import methods from tests.interop import resources -from tests.interop import test_pb2 from tests.unit.beta import test_utilities _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/src/python/grpcio/tests/interop/empty.proto b/src/python/grpcio/tests/interop/empty.proto deleted file mode 100644 index 6d0eb937d67..00000000000 --- a/src/python/grpcio/tests/interop/empty.proto +++ /dev/null @@ -1,43 +0,0 @@ - -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package grpc.testing; - -// An empty message that you can re-use to avoid defining duplicated empty -// messages in your project. A typical example is to use it as argument or the -// return value of a service API. For instance: -// -// service Foo { -// rpc Bar (grpc.testing.Empty) returns (grpc.testing.Empty) { }; -// }; -// -message Empty {} diff --git a/src/python/grpcio/tests/interop/messages.proto b/src/python/grpcio/tests/interop/messages.proto deleted file mode 100644 index 193b6c41712..00000000000 --- a/src/python/grpcio/tests/interop/messages.proto +++ /dev/null @@ -1,167 +0,0 @@ - -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Message definitions to be used by integration test service definitions. - -syntax = "proto3"; - -package grpc.testing; - -// The type of payload that should be returned. -enum PayloadType { - // Compressable text format. - COMPRESSABLE = 0; - - // Uncompressable binary format. - UNCOMPRESSABLE = 1; - - // Randomly chosen from all other formats defined in this enum. - RANDOM = 2; -} - -// Compression algorithms -enum CompressionType { - // No compression - NONE = 0; - GZIP = 1; - DEFLATE = 2; -} - -// A block of data, to simply increase gRPC message size. -message Payload { - // The type of data in body. - PayloadType type = 1; - // Primary contents of payload. - bytes body = 2; -} - -// A protobuf representation for grpc status. This is used by test -// clients to specify a status that the server should attempt to return. -message EchoStatus { - int32 code = 1; - string message = 2; -} - -// Unary request. -message SimpleRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - PayloadType response_type = 1; - - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 response_size = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Whether SimpleResponse should include username. - bool fill_username = 4; - - // Whether SimpleResponse should include OAuth scope. - bool fill_oauth_scope = 5; - - // Compression algorithm to be used by the server for the response (stream) - CompressionType response_compression = 6; - - // Whether server should return a given status - EchoStatus response_status = 7; -} - -// Unary response, as configured by the request. -message SimpleResponse { - // Payload to increase message size. - Payload payload = 1; - // The user the request came from, for verifying authentication was - // successful when the client expected it. - string username = 2; - // OAuth scope. - string oauth_scope = 3; -} - -// Client-streaming request. -message StreamingInputCallRequest { - // Optional input payload sent along with the request. - Payload payload = 1; - - // Not expecting any payload from the response. -} - -// Client-streaming response. -message StreamingInputCallResponse { - // Aggregated size of payloads received from the client. - int32 aggregated_payload_size = 1; -} - -// Configuration for a particular response. -message ResponseParameters { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 size = 1; - - // Desired interval between consecutive responses in the response stream in - // microseconds. - int32 interval_us = 2; -} - -// Server-streaming request. -message StreamingOutputCallRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - PayloadType response_type = 1; - - // Configuration for each expected response message. - repeated ResponseParameters response_parameters = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Compression algorithm to be used by the server for the response (stream) - CompressionType response_compression = 6; - - // Whether server should return a given status - EchoStatus response_status = 7; -} - -// Server-streaming response, as configured by the request and parameters. -message StreamingOutputCallResponse { - // Payload to increase response size. - Payload payload = 1; -} - -// For reconnect interop test only. -// Server tells client whether its reconnects are following the spec and the -// reconnect backoffs it saw. -message ReconnectInfo { - bool passed = 1; - repeated int32 backoff_ms = 2; -} diff --git a/src/python/grpcio/tests/interop/methods.py b/src/python/grpcio/tests/interop/methods.py index 03810338ed2..67862ed7d37 100644 --- a/src/python/grpcio/tests/interop/methods.py +++ b/src/python/grpcio/tests/interop/methods.py @@ -42,9 +42,9 @@ from oauth2client import client as oauth2client_client from grpc.framework.common import cardinality from grpc.framework.interfaces.face import face -from tests.interop import empty_pb2 -from tests.interop import messages_pb2 -from tests.interop import test_pb2 +from src.proto.grpc.testing import empty_pb2 +from src.proto.grpc.testing import messages_pb2 +from src.proto.grpc.testing import test_pb2 _TIMEOUT = 7 diff --git a/src/python/grpcio/tests/interop/server.py b/src/python/grpcio/tests/interop/server.py index 6dd55f008c3..ab2c3c708f4 100644 --- a/src/python/grpcio/tests/interop/server.py +++ b/src/python/grpcio/tests/interop/server.py @@ -34,10 +34,10 @@ import logging import time from grpc.beta import implementations +from src.proto.grpc.testing import test_pb2 from tests.interop import methods from tests.interop import resources -from tests.interop import test_pb2 _ONE_DAY_IN_SECONDS = 60 * 60 * 24 diff --git a/src/python/grpcio/tests/interop/test.proto b/src/python/grpcio/tests/interop/test.proto deleted file mode 100644 index 9feecc02785..00000000000 --- a/src/python/grpcio/tests/interop/test.proto +++ /dev/null @@ -1,86 +0,0 @@ - -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// An integration test service that covers all the method signature permutations -// of unary/streaming requests/responses. - -syntax = "proto3"; - -import "tests/interop/empty.proto"; -import "tests/interop/messages.proto"; - -package grpc.testing; - -// A simple service to test the various types of RPCs and experiment with -// performance with various types of payload. -service TestService { - // One empty request followed by one empty response. - rpc EmptyCall(grpc.testing.Empty) returns (grpc.testing.Empty); - - // One request followed by one response. - rpc UnaryCall(SimpleRequest) returns (SimpleResponse); - - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - rpc StreamingOutputCall(StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); - - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - rpc StreamingInputCall(stream StreamingInputCallRequest) - returns (StreamingInputCallResponse); - - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - rpc FullDuplexCall(stream StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); - - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - rpc HalfDuplexCall(stream StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); -} - - -// A simple service NOT implemented at servers so clients can test for -// that case. -service UnimplementedService { - // A call that no server should implement - rpc UnimplementedCall(grpc.testing.Empty) returns(grpc.testing.Empty); -} - -// A service used to control reconnect server. -service ReconnectService { - rpc Start(grpc.testing.Empty) returns (grpc.testing.Empty); - rpc Stop(grpc.testing.Empty) returns (grpc.testing.ReconnectInfo); -} diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 28b91f8b623..fb2be22944d 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -314,7 +314,8 @@ class PythonLanguage: ] def global_env(self): - return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT)} + return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), + 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT)} def unimplemented_test_cases(self): return _SKIP_ADVANCED + _SKIP_COMPRESSION + ['jwt_token_creds', diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9dff686bbf6..4b9898539d5 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -371,6 +371,7 @@ class PythonLanguage(object): tests_json = json.load(tests_json_file) environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) environment['PYVER'] = '2.7' + environment['PYTHONPATH'] = os.path.abspath('src/python/gens') if self.config.build_config != 'gcov': return [self.config.job_spec( ['tools/run_tests/run_python.sh'],