diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index a6dd59fc13a..a54bfa1f5b4 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1968,15 +1968,18 @@ class Compression(enum.IntEnum): def _default_get_protos(*args, **kwargs): - raise NotImplementedError("Install the grpcio-tools package to use get_protos.") + raise NotImplementedError( + "Install the grpcio-tools package to use get_protos.") def _default_get_services(*args, **kwargs): - raise NotImplementedError("Install the grpcio-tools package to use get_services.") + raise NotImplementedError( + "Install the grpcio-tools package to use get_services.") def _default_get_protos_and_services(*args, **kwargs): - raise NotImplementedError("Install the grpcio-tools package to use get_protos_and_services.") + raise NotImplementedError( + "Install the grpcio-tools package to use get_protos_and_services.") try: diff --git a/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py b/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py index 2c70e44cf1e..4b0d43b2fc2 100644 --- a/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py +++ b/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py @@ -21,7 +21,7 @@ import multiprocessing import functools # TODO: Support setup.py as test runner. -# TODO: Support Blaze as test runner. + @contextlib.contextmanager def _grpc_tools_unimportable(): @@ -35,6 +35,7 @@ def _grpc_tools_unimportable(): # TODO: Dedupe with grpc_tools test? def _wrap_in_subprocess(error_queue, fn): + @functools.wraps(fn) def _wrapped(): try: @@ -42,6 +43,7 @@ def _wrap_in_subprocess(error_queue, fn): except Exception as e: error_queue.put(e) raise + return _wrapped @@ -53,7 +55,8 @@ def _run_in_subprocess(test_case): proc.join() if not error_queue.empty(): raise error_queue.get() - assert proc.exitcode == 0, "Process exited with code {}".format(proc.exitcode) + assert proc.exitcode == 0, "Process exited with code {}".format( + proc.exitcode) def _test_sunny_day(): @@ -67,7 +70,8 @@ def _test_grpc_tools_unimportable(): with _grpc_tools_unimportable(): import grpc try: - protos, services = grpc.protos_and_services("tests/unit/data/foo/bar.proto") + protos, services = grpc.protos_and_services( + "tests/unit/data/foo/bar.proto") except NotImplementedError as e: assert "grpcio-tools" in str(e) else: @@ -82,6 +86,7 @@ class DynamicStubTest(unittest.TestCase): def test_grpc_tools_unimportable(self): _run_in_subprocess(_test_grpc_tools_unimportable) + if __name__ == "__main__": logging.basicConfig() unittest.main(verbosity=2) diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py b/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py index e71fd40c161..5a00f27976b 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py @@ -12,5 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .protoc import main diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py b/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py index 8f78dbb991a..37ee20630b1 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py @@ -31,6 +31,7 @@ from grpc_tools import _protoc_compiler _PROTO_MODULE_SUFFIX = "_pb2" _SERVICE_MODULE_SUFFIX = "_pb2_grpc" + def main(command_arguments): """Run the protocol buffer compiler with the given command-line arguments. @@ -41,108 +42,124 @@ def main(command_arguments): command_arguments = [argument.encode() for argument in command_arguments] return _protoc_compiler.run_main(command_arguments) + def _module_name_to_proto_file(suffix, module_name): - components = module_name.split(".") - proto_name = components[-1][:-1*len(suffix)] - return os.path.sep.join(components[:-1] + [proto_name + ".proto"]) + components = module_name.split(".") + proto_name = components[-1][:-1 * len(suffix)] + return os.path.sep.join(components[:-1] + [proto_name + ".proto"]) + def _proto_file_to_module_name(suffix, proto_file): - components = proto_file.split(os.path.sep) - proto_base_name = os.path.splitext(components[-1])[0] - return ".".join(components[:-1] + [proto_base_name + suffix]) + components = proto_file.split(os.path.sep) + proto_base_name = os.path.splitext(components[-1])[0] + return ".".join(components[:-1] + [proto_base_name + suffix]) @contextlib.contextmanager def _augmented_syspath(new_paths): - original_sys_path = sys.path - if new_paths is not None: - sys.path = sys.path + new_paths - try: - yield - finally: - sys.path = original_sys_path + original_sys_path = sys.path + if new_paths is not None: + sys.path = sys.path + new_paths + try: + yield + finally: + sys.path = original_sys_path # TODO: Investigate making this even more of a no-op in the case that we have # truly already imported the module. def _protos(protobuf_path, include_paths=None): - with _augmented_syspath(include_paths): - module_name = _proto_file_to_module_name(_PROTO_MODULE_SUFFIX, protobuf_path) - module = importlib.import_module(module_name) - return module + with _augmented_syspath(include_paths): + module_name = _proto_file_to_module_name(_PROTO_MODULE_SUFFIX, + protobuf_path) + module = importlib.import_module(module_name) + return module def _services(protobuf_path, include_paths=None): - _protos(protobuf_path, include_paths) - with _augmented_syspath(include_paths): - module_name = _proto_file_to_module_name(_SERVICE_MODULE_SUFFIX, protobuf_path) - module = importlib.import_module(module_name) - return module + _protos(protobuf_path, include_paths) + with _augmented_syspath(include_paths): + module_name = _proto_file_to_module_name(_SERVICE_MODULE_SUFFIX, + protobuf_path) + module = importlib.import_module(module_name) + return module def _protos_and_services(protobuf_path, include_paths=None): - return (_protos(protobuf_path, include_paths=include_paths), - _services(protobuf_path, include_paths=include_paths)) + return (_protos(protobuf_path, include_paths=include_paths), + _services(protobuf_path, include_paths=include_paths)) _proto_code_cache = {} + class ProtoLoader(importlib.abc.Loader): - def __init__(self, suffix, codegen_fn, module_name, protobuf_path, proto_root): - self._suffix = suffix - self._codegen_fn = codegen_fn - self._module_name = module_name - self._protobuf_path = protobuf_path - self._proto_root = proto_root - - def create_module(self, spec): - return None - - def _generated_file_to_module_name(self, filepath): - components = filepath.split(os.path.sep) - return ".".join(components[:-1] + [os.path.splitext(components[-1])[0]]) - - def exec_module(self, module): - assert module.__name__ == self._module_name - code = None - if self._module_name in _proto_code_cache: - code = _proto_code_cache[self._module_name] - six.exec_(code, module.__dict__) - else: - files = self._codegen_fn(self._protobuf_path.encode('ascii'), [path.encode('ascii') for path in sys.path]) - # NOTE: The files are returned in topological order of dependencies. Each - # entry is guaranteed to depend only on the modules preceding it in the - # list and the last entry is guaranteed to be our requested module. We - # cache the code from the first invocation at module-scope so that we - # don't have to regenerate code that has already been generated by protoc. - for f in files[:-1]: - module_name = self._generated_file_to_module_name(f[0].decode('ascii')) - if module_name not in sys.modules: - if module_name not in _proto_code_cache: - _proto_code_cache[module_name] = f[1] - importlib.import_module(module_name) - six.exec_(files[-1][1], module.__dict__) + + def __init__(self, suffix, codegen_fn, module_name, protobuf_path, + proto_root): + self._suffix = suffix + self._codegen_fn = codegen_fn + self._module_name = module_name + self._protobuf_path = protobuf_path + self._proto_root = proto_root + + def create_module(self, spec): + return None + + def _generated_file_to_module_name(self, filepath): + components = filepath.split(os.path.sep) + return ".".join(components[:-1] + [os.path.splitext(components[-1])[0]]) + + def exec_module(self, module): + assert module.__name__ == self._module_name + code = None + if self._module_name in _proto_code_cache: + code = _proto_code_cache[self._module_name] + six.exec_(code, module.__dict__) + else: + files = self._codegen_fn( + self._protobuf_path.encode('ascii'), + [path.encode('ascii') for path in sys.path]) + # NOTE: The files are returned in topological order of dependencies. Each + # entry is guaranteed to depend only on the modules preceding it in the + # list and the last entry is guaranteed to be our requested module. We + # cache the code from the first invocation at module-scope so that we + # don't have to regenerate code that has already been generated by protoc. + for f in files[:-1]: + module_name = self._generated_file_to_module_name( + f[0].decode('ascii')) + if module_name not in sys.modules: + if module_name not in _proto_code_cache: + _proto_code_cache[module_name] = f[1] + importlib.import_module(module_name) + six.exec_(files[-1][1], module.__dict__) class ProtoFinder(importlib.abc.MetaPathFinder): - def __init__(self, suffix, codegen_fn): - self._suffix = suffix - self._codegen_fn = codegen_fn - - def find_spec(self, fullname, path, target=None): - filepath = _module_name_to_proto_file(self._suffix, fullname) - for search_path in sys.path: - try: - prospective_path = os.path.join(search_path, filepath) - os.stat(prospective_path) - except FileNotFoundError: - continue - else: - # TODO: Use a stdlib helper function to construct this. - return importlib.machinery.ModuleSpec(fullname, ProtoLoader(self._suffix, self._codegen_fn, fullname, filepath, search_path)) - -sys.meta_path.extend([ProtoFinder(_PROTO_MODULE_SUFFIX, _protoc_compiler.get_protos), - ProtoFinder(_SERVICE_MODULE_SUFFIX, _protoc_compiler.get_services)]) + + def __init__(self, suffix, codegen_fn): + self._suffix = suffix + self._codegen_fn = codegen_fn + + def find_spec(self, fullname, path, target=None): + filepath = _module_name_to_proto_file(self._suffix, fullname) + for search_path in sys.path: + try: + prospective_path = os.path.join(search_path, filepath) + os.stat(prospective_path) + except FileNotFoundError: + continue + else: + return importlib.machinery.ModuleSpec( + fullname, + ProtoLoader(self._suffix, self._codegen_fn, fullname, + filepath, search_path)) + + +sys.meta_path.extend([ + ProtoFinder(_PROTO_MODULE_SUFFIX, _protoc_compiler.get_protos), + ProtoFinder(_SERVICE_MODULE_SUFFIX, _protoc_compiler.get_services) +]) if __name__ == '__main__': proto_include = pkg_resources.resource_filename('grpc_tools', '_proto') diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/protoc_test.py b/tools/distrib/python/grpcio_tools/grpc_tools/protoc_test.py index 2fb358ee8bb..d5a9e039dae 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/protoc_test.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/protoc_test.py @@ -9,7 +9,9 @@ import unittest import multiprocessing import functools + def _wrap_in_subprocess(error_queue, fn): + @functools.wraps(fn) def _wrapped(): try: @@ -17,6 +19,7 @@ def _wrap_in_subprocess(error_queue, fn): except Exception as e: error_queue.put(e) raise + return _wrapped @@ -28,7 +31,8 @@ def _run_in_subprocess(test_case): proc.join() if not error_queue.empty(): raise error_queue.get() - assert proc.exitcode == 0, "Process exited with code {}".format(proc.exitcode) + assert proc.exitcode == 0, "Process exited with code {}".format( + proc.exitcode) def _test_import_protos(): @@ -65,9 +69,11 @@ def _test_proto_module_imported_once(): proto_path = "tools/distrib/python/grpcio_tools/" protos = protoc._protos("grpc_tools/simple.proto", [proto_path]) services = protoc._services("grpc_tools/simple.proto", [proto_path]) - complicated_protos = protoc._protos("grpc_tools/complicated.proto", [proto_path]) + complicated_protos = protoc._protos("grpc_tools/complicated.proto", + [proto_path]) assert (complicated_protos.grpc__tools_dot_simplest__pb2.SimplestMessage is - protos.grpc__tools_dot_simpler__pb2.grpc__tools_dot_simplest__pb2.SimplestMessage) + protos.grpc__tools_dot_simpler__pb2.grpc__tools_dot_simplest__pb2. + SimplestMessage) def _test_static_dynamic_combo(): @@ -76,7 +82,8 @@ def _test_static_dynamic_combo(): proto_path = "tools/distrib/python/grpcio_tools/" protos = protoc._protos("grpc_tools/simple.proto", [proto_path]) assert (complicated_pb2.grpc__tools_dot_simplest__pb2.SimplestMessage is - protos.grpc__tools_dot_simpler__pb2.grpc__tools_dot_simplest__pb2.SimplestMessage) + protos.grpc__tools_dot_simpler__pb2.grpc__tools_dot_simplest__pb2. + SimplestMessage) def _test_combined_import(): @@ -98,6 +105,7 @@ def _test_syntax_errors(): else: assert False, "Compile error expected. None occurred." + # TODO: Test warnings. @@ -129,5 +137,6 @@ class ProtocTest(unittest.TestCase): # TODO: Write test to ensure the right module loader is used. + if __name__ == '__main__': unittest.main() diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/simple.proto b/tools/distrib/python/grpcio_tools/grpc_tools/simple.proto index 2f7d56ffa95..94cc880d641 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/simple.proto +++ b/tools/distrib/python/grpcio_tools/grpc_tools/simple.proto @@ -1,4 +1,3 @@ -// TODO: Move these proto files to a dedicated directory. syntax = "proto3"; package grpc_tools.simple;