Merge pull request #21458 from gnossen/dynamic_stubs

Enable Runtime Import of .proto Files
pull/23877/head
Richard Belleville 5 years ago committed by GitHub
commit c79bef55ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 7
      doc/python/sphinx/grpc.rst
  2. 40
      examples/python/no_codegen/greeter_client.py
  3. 40
      examples/python/no_codegen/greeter_server.py
  4. 38
      examples/python/no_codegen/helloworld.proto
  5. 6
      src/python/grpcio/grpc/BUILD.bazel
  6. 5
      src/python/grpcio/grpc/__init__.py
  7. 161
      src/python/grpcio/grpc/_runtime_protos.py
  8. 31
      src/python/grpcio_tests/setup.py
  9. 1
      src/python/grpcio_tests/tests/tests.json
  10. 17
      src/python/grpcio_tests/tests/unit/BUILD.bazel
  11. 3
      src/python/grpcio_tests/tests/unit/_api_test.py
  12. 118
      src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py
  13. 25
      src/python/grpcio_tests/tests/unit/data/foo/bar.proto
  14. 53
      tools/distrib/python/grpcio_tools/BUILD.bazel
  15. 7
      tools/distrib/python/grpcio_tools/_parallel_compile_patch.py
  16. 2
      tools/distrib/python/grpcio_tools/grpc_tools/__init__.py
  17. 114
      tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx
  18. 146
      tools/distrib/python/grpcio_tools/grpc_tools/main.cc
  19. 30
      tools/distrib/python/grpcio_tools/grpc_tools/main.h
  20. 132
      tools/distrib/python/grpcio_tools/grpc_tools/protoc.py
  21. 55
      tools/distrib/python/grpcio_tools/grpc_tools/test/BUILD.bazel
  22. 26
      tools/distrib/python/grpcio_tools/grpc_tools/test/complicated.proto
  23. 23
      tools/distrib/python/grpcio_tools/grpc_tools/test/flawed.proto
  24. 160
      tools/distrib/python/grpcio_tools/grpc_tools/test/protoc_test.py
  25. 40
      tools/distrib/python/grpcio_tools/grpc_tools/test/simple.proto
  26. 25
      tools/distrib/python/grpcio_tools/grpc_tools/test/simpler.proto
  27. 22
      tools/distrib/python/grpcio_tools/grpc_tools/test/simplest.proto
  28. 2
      tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh
  29. 2
      tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh

@ -187,3 +187,10 @@ Compression
^^^^^^^^^^^
.. autoclass:: Compression
Runtime Protobuf Parsing
^^^^^^^^^^^^^^^^^^^^^^^^
.. autofunction:: protos
.. autofunction:: services
.. autofunction:: protos_and_services

@ -0,0 +1,40 @@
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Hello World without using protoc.
This example parses message and service schemas directly from a
.proto file on the filesystem.
Several APIs used in this example are in an experimental state.
"""
from __future__ import print_function
import logging
import grpc
import grpc.experimental
# NOTE: The path to the .proto file must be reachable from an entry
# on sys.path. Use sys.path.insert or set the $PYTHONPATH variable to
# import from files located elsewhere on the filesystem.
protos = grpc.protos("helloworld.proto")
services = grpc.services("helloworld.proto")
logging.basicConfig()
response = services.Greeter.SayHello(protos.HelloRequest(name='you'),
'localhost:50051',
insecure=True)
print("Greeter client received: " + response.message)

@ -0,0 +1,40 @@
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Python implementation of the GRPC helloworld.Greeter server."""
from concurrent import futures
import logging
import grpc
protos, services = grpc.protos_and_services("helloworld.proto")
class Greeter(services.GreeterServicer):
def SayHello(self, request, context):
return protos.HelloReply(message='Hello, %s!' % request.name)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
services.add_GreeterServicer_to_server(Greeter(), server)
server.add_insecure_port('[::]:50051')
server.start()
server.wait_for_termination()
if __name__ == '__main__':
logging.basicConfig()
serve()

@ -0,0 +1,38 @@
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.helloworld";
option java_outer_classname = "HelloWorldProto";
option objc_class_prefix = "HLW";
package helloworld;
// The greeting service definition.
service Greeter {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
}
// The request message containing the user's name.
message HelloRequest {
string name = 1;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}

@ -66,6 +66,11 @@ py_library(
srcs = ["_simple_stubs.py"],
)
py_library(
name = "_runtime_protos",
srcs = ["_runtime_protos.py"],
)
py_library(
name = "grpcio",
srcs = ["__init__.py"],
@ -82,6 +87,7 @@ py_library(
":server",
":compression",
":_simple_stubs",
":_runtime_protos",
"//src/python/grpcio/grpc/_cython:cygrpc",
"//src/python/grpcio/grpc/experimental",
"//src/python/grpcio/grpc/framework",

@ -2038,6 +2038,8 @@ class Compression(enum.IntEnum):
Gzip = _compression.Gzip
from grpc._runtime_protos import protos, services, protos_and_services # pylint: disable=wrong-import-position
################################### __all__ #################################
__all__ = (
@ -2098,6 +2100,9 @@ __all__ = (
'secure_channel',
'intercept_channel',
'server',
'protos',
'services',
'protos_and_services',
)
############################### Extension Shims ################################

@ -0,0 +1,161 @@
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
def _uninstalled_protos(*args, **kwargs):
raise NotImplementedError(
"Install the grpcio-tools package to use the protos function.")
def _uninstalled_services(*args, **kwargs):
raise NotImplementedError(
"Install the grpcio-tools package to use the services function.")
def _uninstalled_protos_and_services(*args, **kwargs):
raise NotImplementedError(
"Install the grpcio-tools package to use the protos_and_services function."
)
def _interpreter_version_protos(*args, **kwargs):
raise NotImplementedError(
"The protos function is only on available on Python 3.X interpreters.")
def _interpreter_version_services(*args, **kwargs):
raise NotImplementedError(
"The services function is only on available on Python 3.X interpreters."
)
def _interpreter_version_protos_and_services(*args, **kwargs):
raise NotImplementedError(
"The protos_and_services function is only on available on Python 3.X interpreters."
)
def protos(protobuf_path): # pylint: disable=unused-argument
"""Returns a module generated by the indicated .proto file.
THIS IS AN EXPERIMENTAL API.
Use this function to retrieve classes corresponding to message
definitions in the .proto file.
To inspect the contents of the returned module, use the dir function.
For example:
```
protos = grpc.protos("foo.proto")
print(dir(protos))
```
The returned module object corresponds to the _pb2.py file generated
by protoc. The path is expected to be relative to an entry on sys.path
and all transitive dependencies of the file should also be resolveable
from an entry on sys.path.
To completely disable the machinery behind this function, set the
GRPC_PYTHON_DISABLE_DYNAMIC_STUBS environment variable to "true".
Args:
protobuf_path: The path to the .proto file on the filesystem. This path
must be resolveable from an entry on sys.path and so must all of its
transitive dependencies.
Returns:
A module object corresponding to the message code for the indicated
.proto file. Equivalent to a generated _pb2.py file.
"""
def services(protobuf_path): # pylint: disable=unused-argument
"""Returns a module generated by the indicated .proto file.
THIS IS AN EXPERIMENTAL API.
Use this function to retrieve classes and functions corresponding to
service definitions in the .proto file, including both stub and servicer
definitions.
To inspect the contents of the returned module, use the dir function.
For example:
```
services = grpc.services("foo.proto")
print(dir(services))
```
The returned module object corresponds to the _pb2_grpc.py file generated
by protoc. The path is expected to be relative to an entry on sys.path
and all transitive dependencies of the file should also be resolveable
from an entry on sys.path.
To completely disable the machinery behind this function, set the
GRPC_PYTHON_DISABLE_DYNAMIC_STUBS environment variable to "true".
Args:
protobuf_path: The path to the .proto file on the filesystem. This path
must be resolveable from an entry on sys.path and so must all of its
transitive dependencies.
Returns:
A module object corresponding to the stub/service code for the indicated
.proto file. Equivalent to a generated _pb2_grpc.py file.
"""
def protos_and_services(protobuf_path): # pylint: disable=unused-argument
"""Returns a 2-tuple of modules corresponding to protos and services.
THIS IS AN EXPERIMENTAL API.
The return value of this function is equivalent to a call to protos and a
call to services.
To completely disable the machinery behind this function, set the
GRPC_PYTHON_DISABLE_DYNAMIC_STUBS environment variable to "true".
Args:
protobuf_path: The path to the .proto file on the filesystem. This path
must be resolveable from an entry on sys.path and so must all of its
transitive dependencies.
Returns:
A 2-tuple of module objects corresponding to (protos(path), services(path)).
"""
if sys.version_info < (3, 5, 0):
protos = _interpreter_version_protos
services = _interpreter_version_services
protos_and_services = _interpreter_version_protos_and_services
else:
try:
import grpc_tools # pylint: disable=unused-import
except ImportError as e:
# NOTE: It's possible that we're encountering a transitive ImportError, so
# we check for that and re-raise if so.
if "grpc_tools" not in e.args[0]:
raise
protos = _uninstalled_protos
services = _uninstalled_services
protos_and_services = _uninstalled_protos_and_services
else:
from grpc_tools.protoc import _protos as protos # pylint: disable=unused-import
from grpc_tools.protoc import _services as services # pylint: disable=unused-import
from grpc_tools.protoc import _protos_and_services as protos_and_services # pylint: disable=unused-import

@ -13,6 +13,7 @@
# limitations under the License.
"""A setup module for the gRPC Python package."""
import multiprocessing
import os
import os.path
import sys
@ -94,17 +95,19 @@ TESTS_REQUIRE = INSTALL_REQUIRES
PACKAGES = setuptools.find_packages('.')
setuptools.setup(
name='grpcio-tests',
version=grpc_version.VERSION,
license=LICENSE,
packages=list(PACKAGES),
package_dir=PACKAGE_DIRECTORIES,
package_data=PACKAGE_DATA,
install_requires=INSTALL_REQUIRES,
cmdclass=COMMAND_CLASS,
tests_require=TESTS_REQUIRE,
test_suite=TEST_SUITE,
test_loader=TEST_LOADER,
test_runner=TEST_RUNNER,
)
if __name__ == "__main__":
multiprocessing.freeze_support()
setuptools.setup(
name='grpcio-tests',
version=grpc_version.VERSION,
license=LICENSE,
packages=list(PACKAGES),
package_dir=PACKAGE_DIRECTORIES,
package_data=PACKAGE_DATA,
install_requires=INSTALL_REQUIRES,
cmdclass=COMMAND_CLASS,
tests_require=TESTS_REQUIRE,
test_suite=TEST_SUITE,
test_loader=TEST_LOADER,
test_runner=TEST_RUNNER,
)

@ -49,6 +49,7 @@
"unit._cython.cygrpc_test.SecureServerSecureClient",
"unit._cython.cygrpc_test.TypeSmokeTest",
"unit._dns_resolver_test.DNSResolverTest",
"unit._dynamic_stubs_test.DynamicStubTest",
"unit._empty_message_test.EmptyMessageTest",
"unit._error_message_encoding_test.ErrorMessageEncodingTest",
"unit._exit_test.ExitTest",

@ -111,3 +111,20 @@ py_library(
)
for test_file_name in GRPCIO_TESTS_UNIT
]
py2and3_test(
name = "_dynamic_stubs_test",
size = "small",
srcs = ["_dynamic_stubs_test.py"],
data = [
"data/foo/bar.proto",
],
imports = ["../../"],
main = "_dynamic_stubs_test.py",
deps = [
"//src/python/grpcio/grpc:grpcio",
"//src/python/grpcio_tests/tests/testing",
"//tools/distrib/python/grpcio_tools:grpc_tools",
"@six",
],
)

@ -84,6 +84,9 @@ class AllTest(unittest.TestCase):
'secure_channel',
'intercept_channel',
'server',
'protos',
'services',
'protos_and_services',
)
six.assertCountEqual(self, expected_grpc_code_elements,

@ -0,0 +1,118 @@
# Copyright 2019 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test of dynamic stub import API."""
import contextlib
import functools
import logging
import multiprocessing
import os
import sys
import unittest
@contextlib.contextmanager
def _grpc_tools_unimportable():
original_sys_path = sys.path
sys.path = [path for path in sys.path if "grpcio_tools" not in path]
try:
import grpc_tools
except ImportError:
pass
else:
del grpc_tools
sys.path = original_sys_path
raise unittest.SkipTest("Failed to make grpc_tools unimportable.")
try:
yield
finally:
sys.path = original_sys_path
def _collect_errors(fn):
@functools.wraps(fn)
def _wrapped(error_queue):
try:
fn()
except Exception as e:
error_queue.put(e)
raise
return _wrapped
def _run_in_subprocess(test_case):
sys.path.insert(
0, os.path.join(os.path.realpath(os.path.dirname(__file__)), ".."))
error_queue = multiprocessing.Queue()
proc = multiprocessing.Process(target=test_case, args=(error_queue,))
proc.start()
proc.join()
sys.path.pop(0)
if not error_queue.empty():
raise error_queue.get()
assert proc.exitcode == 0, "Process exited with code {}".format(
proc.exitcode)
def _assert_unimplemented(msg_substr):
import grpc
try:
protos, services = grpc.protos_and_services(
"tests/unit/data/foo/bar.proto")
except NotImplementedError as e:
assert msg_substr in str(e), "{} was not in '{}'".format(
msg_substr, str(e))
else:
assert False, "Did not raise NotImplementedError"
@_collect_errors
def _test_sunny_day():
if sys.version_info[0] == 3:
import grpc
protos, services = grpc.protos_and_services(
os.path.join("tests", "unit", "data", "foo", "bar.proto"))
assert protos.BarMessage is not None
assert services.BarStub is not None
else:
_assert_unimplemented("Python 3")
@_collect_errors
def _test_grpc_tools_unimportable():
with _grpc_tools_unimportable():
if sys.version_info[0] == 3:
_assert_unimplemented("grpcio-tools")
else:
_assert_unimplemented("Python 3")
# NOTE(rbellevi): multiprocessing.Process fails to pickle function objects
# when they do not come from the "__main__" module, so this test passes
# if run directly on Windows, but not if started by the test runner.
@unittest.skipIf(os.name == "nt", "Windows multiprocessing unsupported")
class DynamicStubTest(unittest.TestCase):
def test_sunny_day(self):
_run_in_subprocess(_test_sunny_day)
def test_grpc_tools_unimportable(self):
_run_in_subprocess(_test_grpc_tools_unimportable)
if __name__ == "__main__":
logging.basicConfig()
unittest.main(verbosity=2)

@ -0,0 +1,25 @@
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package tests.unit.data.foo.bar;
message BarMessage {
string a = 1;
};
service Bar {
rpc GetBar(BarMessage) returns (BarMessage);
};

@ -0,0 +1,53 @@
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = [
"//examples/python:__subpackages__",
"//src/python:__subpackages__",
"//tools/distrib/python/grpcio_tools:__subpackages__",
])
load("//bazel:cython_library.bzl", "pyx_library")
cc_library(
name = "protoc_lib",
srcs = ["grpc_tools/main.cc"],
hdrs = ["grpc_tools/main.h"],
includes = ["."],
deps = [
"//src/compiler:grpc_plugin_support",
"@com_google_protobuf//:protoc_lib",
],
)
pyx_library(
name = "cyprotoc",
srcs = ["grpc_tools/_protoc_compiler.pyx"],
deps = [":protoc_lib"],
)
py_library(
name = "grpc_tools",
srcs = [
"grpc_tools/__init__.py",
"grpc_tools/protoc.py",
],
imports = ["."],
srcs_version = "PY2AND3",
deps = [
":cyprotoc",
"//src/python/grpcio/grpc:grpcio",
"@com_google_protobuf//:protobuf_python",
],
)

@ -22,9 +22,10 @@ import os
try:
BUILD_EXT_COMPILER_JOBS = int(
os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1'))
except ValueError:
BUILD_EXT_COMPILER_JOBS = 1
os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'))
except KeyError:
import multiprocessing
BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count()
# monkey-patch for parallel compilation

@ -11,3 +11,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .protoc import main

@ -1,4 +1,4 @@
# Copyright 2016 gRPC authors.
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,12 +13,122 @@
# limitations under the License.
from libc cimport stdlib
from libcpp.vector cimport vector
from libcpp.utility cimport pair
from libcpp.string cimport string
from cython.operator cimport dereference
import warnings
cdef extern from "grpc_tools/main.h" namespace "grpc_tools":
cppclass cProtocError "::grpc_tools::ProtocError":
string filename
int line
int column
string message
cppclass cProtocWarning "::grpc_tools::ProtocWarning":
string filename
int line
int column
string message
cdef extern from "grpc_tools/main.h":
int protoc_main(int argc, char *argv[])
int protoc_get_protos(char* protobuf_path,
vector[string]* include_path,
vector[pair[string, string]]* files_out,
vector[cProtocError]* errors,
vector[cProtocWarning]* wrnings) nogil except +
int protoc_get_services(char* protobuf_path,
vector[string]* include_path,
vector[pair[string, string]]* files_out,
vector[cProtocError]* errors,
vector[cProtocWarning]* wrnings) nogil except +
def run_main(list args not None):
cdef char **argv = <char **>stdlib.malloc(len(args)*sizeof(char *))
for i in range(len(args)):
argv[i] = args[i]
return protoc_main(len(args), argv)
class ProtocError(Exception):
def __init__(self, filename, line, column, message):
self.filename = filename
self.line = line
self.column = column
self.message = message
def __repr__(self):
return "ProtocError(filename=\"{}\", line={}, column={}, message=\"{}\")".format(
self.filename, self.line, self.column, self.message)
def __str__(self):
return "{}:{}:{} error: {}".format(self.filename.decode("ascii"),
self.line, self.column, self.message.decode("ascii"))
class ProtocWarning(Warning):
def __init__(self, filename, line, column, message):
self.filename = filename
self.line = line
self.column = column
self.message = message
def __repr__(self):
return "ProtocWarning(filename=\"{}\", line={}, column={}, message=\"{}\")".format(
self.filename, self.line, self.column, self.message)
__str__ = __repr__
class ProtocErrors(Exception):
def __init__(self, errors):
self._errors = errors
def errors(self):
return self._errors
def __repr__(self):
return "ProtocErrors[{}]".join(repr(err) for err in self._errors)
def __str__(self):
return "\n".join(str(err) for err in self._errors)
cdef _c_protoc_error_to_protoc_error(cProtocError c_protoc_error):
return ProtocError(c_protoc_error.filename, c_protoc_error.line,
c_protoc_error.column, c_protoc_error.message)
cdef _c_protoc_warning_to_protoc_warning(cProtocWarning c_protoc_warning):
return ProtocWarning(c_protoc_warning.filename, c_protoc_warning.line,
c_protoc_warning.column, c_protoc_warning.message)
cdef _handle_errors(int rc, vector[cProtocError]* errors, vector[cProtocWarning]* wrnings, bytes protobuf_path):
for warning in dereference(wrnings):
warnings.warn(_c_protoc_warning_to_protoc_warning(warning))
if rc != 0:
if dereference(errors).size() != 0:
py_errors = [_c_protoc_error_to_protoc_error(c_error)
for c_error in dereference(errors)]
raise ProtocErrors(py_errors)
raise Exception("An unknown error occurred while compiling {}".format(protobuf_path))
def get_protos(bytes protobuf_path, list include_paths):
cdef vector[string] c_include_paths = include_paths
cdef vector[pair[string, string]] files
cdef vector[cProtocError] errors
# NOTE: Abbreviated name used to avoid shadowing of the module name.
cdef vector[cProtocWarning] wrnings
rc = protoc_get_protos(protobuf_path, &c_include_paths, &files, &errors, &wrnings)
_handle_errors(rc, &errors, &wrnings, protobuf_path)
return files
def get_services(bytes protobuf_path, list include_paths):
cdef vector[string] c_include_paths = include_paths
cdef vector[pair[string, string]] files
cdef vector[cProtocError] errors
# NOTE: Abbreviated name used to avoid shadowing of the module name.
cdef vector[cProtocWarning] wrnings
rc = protoc_get_services(protobuf_path, &c_include_paths, &files, &errors, &wrnings)
_handle_errors(rc, &errors, &wrnings, protobuf_path)
return files

@ -19,6 +19,28 @@
#include "grpc_tools/main.h"
#include <google/protobuf/compiler/code_generator.h>
#include <google/protobuf/compiler/importer.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <algorithm>
#include <map>
#include <string>
#include <tuple>
#include <unordered_set>
#include <vector>
using ::google::protobuf::FileDescriptor;
using ::google::protobuf::compiler::CodeGenerator;
using ::google::protobuf::compiler::DiskSourceTree;
using ::google::protobuf::compiler::GeneratorContext;
using ::google::protobuf::compiler::Importer;
using ::google::protobuf::compiler::MultiFileErrorCollector;
using ::google::protobuf::io::StringOutputStream;
using ::google::protobuf::io::ZeroCopyOutputStream;
namespace grpc_tools {
int protoc_main(int argc, char* argv[]) {
google::protobuf::compiler::CommandLineInterface cli;
cli.AllowPlugins("protoc-");
@ -36,3 +58,127 @@ int protoc_main(int argc, char* argv[]) {
return cli.Run(argc, argv);
}
namespace internal {
class GeneratorContextImpl : public GeneratorContext {
public:
GeneratorContextImpl(
const std::vector<const FileDescriptor*>& parsed_files,
std::vector<std::pair<std::string, std::string>>* files_out)
: files_(files_out), parsed_files_(parsed_files) {}
ZeroCopyOutputStream* Open(const std::string& filename) {
files_->emplace_back(filename, "");
return new StringOutputStream(&(files_->back().second));
}
// NOTE(rbellevi): Equivalent to Open, since all files start out empty.
ZeroCopyOutputStream* OpenForAppend(const std::string& filename) {
return Open(filename);
}
// NOTE(rbellevi): Equivalent to Open, since all files start out empty.
ZeroCopyOutputStream* OpenForInsert(const std::string& filename,
const std::string& insertion_point) {
return Open(filename);
}
void ListParsedFiles(
std::vector<const ::google::protobuf::FileDescriptor*>* output) {
*output = parsed_files_;
}
private:
std::vector<std::pair<std::string, std::string>>* files_;
const std::vector<const FileDescriptor*>& parsed_files_;
};
class ErrorCollectorImpl : public MultiFileErrorCollector {
public:
ErrorCollectorImpl(std::vector<::grpc_tools::ProtocError>* errors,
std::vector<::grpc_tools::ProtocWarning>* warnings)
: errors_(errors), warnings_(warnings) {}
void AddError(const std::string& filename, int line, int column,
const std::string& message) {
errors_->emplace_back(filename, line, column, message);
}
void AddWarning(const std::string& filename, int line, int column,
const std::string& message) {
warnings_->emplace_back(filename, line, column, message);
}
private:
std::vector<::grpc_tools::ProtocError>* errors_;
std::vector<::grpc_tools::ProtocWarning>* warnings_;
};
static void calculate_transitive_closure(
const FileDescriptor* descriptor,
std::vector<const FileDescriptor*>* transitive_closure,
std::unordered_set<const ::google::protobuf::FileDescriptor*>* visited) {
for (int i = 0; i < descriptor->dependency_count(); ++i) {
const FileDescriptor* dependency = descriptor->dependency(i);
if (visited->find(dependency) == visited->end()) {
calculate_transitive_closure(dependency, transitive_closure, visited);
}
}
transitive_closure->push_back(descriptor);
visited->insert(descriptor);
}
} // end namespace internal
static int generate_code(
CodeGenerator* code_generator, char* protobuf_path,
const std::vector<std::string>* include_paths,
std::vector<std::pair<std::string, std::string>>* files_out,
std::vector<::grpc_tools::ProtocError>* errors,
std::vector<::grpc_tools::ProtocWarning>* warnings) {
std::unique_ptr<internal::ErrorCollectorImpl> error_collector(
new internal::ErrorCollectorImpl(errors, warnings));
std::unique_ptr<DiskSourceTree> source_tree(new DiskSourceTree());
for (const auto& include_path : *include_paths) {
source_tree->MapPath("", include_path);
}
Importer importer(source_tree.get(), error_collector.get());
const FileDescriptor* parsed_file = importer.Import(protobuf_path);
if (parsed_file == nullptr) {
return 1;
}
std::vector<const FileDescriptor*> transitive_closure;
std::unordered_set<const FileDescriptor*> visited;
internal::calculate_transitive_closure(parsed_file, &transitive_closure,
&visited);
internal::GeneratorContextImpl generator_context(transitive_closure,
files_out);
std::string error;
for (const auto descriptor : transitive_closure) {
code_generator->Generate(descriptor, "", &generator_context, &error);
}
return 0;
}
int protoc_get_protos(
char* protobuf_path, const std::vector<std::string>* include_paths,
std::vector<std::pair<std::string, std::string>>* files_out,
std::vector<::grpc_tools::ProtocError>* errors,
std::vector<::grpc_tools::ProtocWarning>* warnings) {
::google::protobuf::compiler::python::Generator python_generator;
return generate_code(&python_generator, protobuf_path, include_paths,
files_out, errors, warnings);
}
int protoc_get_services(
char* protobuf_path, const std::vector<std::string>* include_paths,
std::vector<std::pair<std::string, std::string>>* files_out,
std::vector<::grpc_tools::ProtocError>* errors,
std::vector<::grpc_tools::ProtocWarning>* warnings) {
grpc_python_generator::GeneratorConfiguration grpc_py_config;
grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config);
return generate_code(&grpc_py_generator, protobuf_path, include_paths,
files_out, errors, warnings);
}
} // end namespace grpc_tools

@ -12,7 +12,35 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <utility>
#include <vector>
namespace grpc_tools {
// We declare `protoc_main` here since we want access to it from Cython as an
// extern but *without* triggering a dllimport declspec when on Windows.
int protoc_main(int argc, char *argv[]);
int protoc_main(int argc, char* argv[]);
struct ProtocError {
std::string filename;
int line;
int column;
std::string message;
ProtocError() {}
ProtocError(std::string filename, int line, int column, std::string message)
: filename(filename), line(line), column(column), message(message) {}
};
typedef ProtocError ProtocWarning;
int protoc_get_protos(
char* protobuf_path, const std::vector<std::string>* include_paths,
std::vector<std::pair<std::string, std::string>>* files_out,
std::vector<ProtocError>* errors, std::vector<ProtocWarning>* warnings);
int protoc_get_services(
char* protobuf_path, const std::vector<std::string>* include_paths,
std::vector<std::pair<std::string, std::string>>* files_out,
std::vector<ProtocError>* errors, std::vector<ProtocWarning>* warnings);
} // end namespace grpc_tools

@ -17,8 +17,15 @@
import pkg_resources
import sys
import os
from grpc_tools import _protoc_compiler
_PROTO_MODULE_SUFFIX = "_pb2"
_SERVICE_MODULE_SUFFIX = "_pb2_grpc"
_DISABLE_DYNAMIC_STUBS = "GRPC_PYTHON_DISABLE_DYNAMIC_STUBS"
def main(command_arguments):
"""Run the protocol buffer compiler with the given command-line arguments.
@ -31,6 +38,131 @@ def main(command_arguments):
return _protoc_compiler.run_main(command_arguments)
if sys.version_info[0] > 2:
import contextlib
import importlib
import importlib.machinery
import threading
_FINDERS_INSTALLED = False
_FINDERS_INSTALLED_LOCK = threading.Lock()
def _maybe_install_proto_finders():
global _FINDERS_INSTALLED
with _FINDERS_INSTALLED_LOCK:
if not _FINDERS_INSTALLED:
sys.meta_path.extend([
ProtoFinder(_PROTO_MODULE_SUFFIX,
_protoc_compiler.get_protos),
ProtoFinder(_SERVICE_MODULE_SUFFIX,
_protoc_compiler.get_services)
])
_FINDERS_INSTALLED = True
def _module_name_to_proto_file(suffix, module_name):
components = module_name.split(".")
proto_name = components[-1][:-1 * len(suffix)]
# NOTE(rbellevi): The Protobuf library expects this path to use
# forward slashes on every platform.
return "/".join(components[:-1] + [proto_name + ".proto"])
def _proto_file_to_module_name(suffix, proto_file):
components = proto_file.split(os.path.sep)
proto_base_name = os.path.splitext(components[-1])[0]
return ".".join(components[:-1] + [proto_base_name + suffix])
def _protos(protobuf_path):
"""Returns a gRPC module generated from the indicated proto file."""
_maybe_install_proto_finders()
module_name = _proto_file_to_module_name(_PROTO_MODULE_SUFFIX,
protobuf_path)
module = importlib.import_module(module_name)
return module
def _services(protobuf_path):
"""Returns a module generated from the indicated proto file."""
_maybe_install_proto_finders()
_protos(protobuf_path)
module_name = _proto_file_to_module_name(_SERVICE_MODULE_SUFFIX,
protobuf_path)
module = importlib.import_module(module_name)
return module
def _protos_and_services(protobuf_path):
"""Returns two modules, corresponding to _pb2.py and _pb2_grpc.py files."""
return (_protos(protobuf_path), _services(protobuf_path))
_proto_code_cache = {}
_proto_code_cache_lock = threading.RLock()
class ProtoLoader(importlib.abc.Loader):
def __init__(self, suffix, codegen_fn, module_name, protobuf_path,
proto_root):
self._suffix = suffix
self._codegen_fn = codegen_fn
self._module_name = module_name
self._protobuf_path = protobuf_path
self._proto_root = proto_root
def create_module(self, spec):
return None
def _generated_file_to_module_name(self, filepath):
components = filepath.split(os.path.sep)
return ".".join(components[:-1] +
[os.path.splitext(components[-1])[0]])
def exec_module(self, module):
assert module.__name__ == self._module_name
code = None
with _proto_code_cache_lock:
if self._module_name in _proto_code_cache:
code = _proto_code_cache[self._module_name]
exec(code, module.__dict__)
else:
files = self._codegen_fn(
self._protobuf_path.encode('ascii'),
[path.encode('ascii') for path in sys.path])
# NOTE: The files are returned in topological order of dependencies. Each
# entry is guaranteed to depend only on the modules preceding it in the
# list and the last entry is guaranteed to be our requested module. We
# cache the code from the first invocation at module-scope so that we
# don't have to regenerate code that has already been generated by protoc.
for f in files[:-1]:
module_name = self._generated_file_to_module_name(
f[0].decode('ascii'))
if module_name not in sys.modules:
if module_name not in _proto_code_cache:
_proto_code_cache[module_name] = f[1]
importlib.import_module(module_name)
exec(files[-1][1], module.__dict__)
class ProtoFinder(importlib.abc.MetaPathFinder):
def __init__(self, suffix, codegen_fn):
self._suffix = suffix
self._codegen_fn = codegen_fn
def find_spec(self, fullname, path, target=None):
filepath = _module_name_to_proto_file(self._suffix, fullname)
for search_path in sys.path:
try:
prospective_path = os.path.join(search_path, filepath)
os.stat(prospective_path)
except (FileNotFoundError, NotADirectoryError):
continue
else:
return importlib.machinery.ModuleSpec(
fullname,
ProtoLoader(self._suffix, self._codegen_fn, fullname,
filepath, search_path))
# NOTE(rbellevi): We provide an environment variable that enables users to completely
# disable this behavior if it is not desired, e.g. for performance reasons.
if not os.getenv(_DISABLE_DYNAMIC_STUBS):
_maybe_install_proto_finders()
if __name__ == '__main__':
proto_include = pkg_resources.resource_filename('grpc_tools', '_proto')
sys.exit(main(sys.argv + ['-I{}'.format(proto_include)]))

@ -0,0 +1,55 @@
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_testonly = 1)
load("//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
proto_library(
name = "simplest_proto",
testonly = True,
srcs = ["simplest.proto"],
strip_import_prefix = "/tools/distrib/python/grpcio_tools/grpc_tools/test/",
)
proto_library(
name = "complicated_proto",
testonly = True,
srcs = ["complicated.proto"],
strip_import_prefix = "/tools/distrib/python/grpcio_tools/grpc_tools/test/",
deps = [":simplest_proto"],
)
py_proto_library(
name = "complicated_py_pb2",
testonly = True,
deps = ["complicated_proto"],
)
py_test(
name = "protoc_test",
srcs = ["protoc_test.py"],
data = [
"complicated.proto",
"flawed.proto",
"simple.proto",
"simpler.proto",
"simplest.proto",
],
python_version = "PY3",
deps = [
":complicated_py_pb2",
"//tools/distrib/python/grpcio_tools:grpc_tools",
],
)

@ -0,0 +1,26 @@
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package test.complicated;
import "simplest.proto";
message ComplicatedMessage {
bool yes = 1;
bool no = 2;
bool why = 3;
simplest.SimplestMessage simplest_message = 4;
};

@ -0,0 +1,23 @@
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
message Broken {
int32 no_field_number;
};
message Broken2 {
int32 no_field_number;
};

@ -0,0 +1,160 @@
# Copyright 2020 The gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for protoc."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import contextlib
import functools
import multiprocessing
import sys
import unittest
# TODO(https://github.com/grpc/grpc/issues/23847): Deduplicate this mechanism with
# the grpcio_tests module.
def _wrap_in_subprocess(error_queue, fn):
@functools.wraps(fn)
def _wrapped():
try:
fn()
except Exception as e:
error_queue.put(e)
raise
return _wrapped
def _run_in_subprocess(test_case):
error_queue = multiprocessing.Queue()
wrapped_case = _wrap_in_subprocess(error_queue, test_case)
proc = multiprocessing.Process(target=wrapped_case)
proc.start()
proc.join()
if not error_queue.empty():
raise error_queue.get()
assert proc.exitcode == 0, "Process exited with code {}".format(
proc.exitcode)
@contextlib.contextmanager
def _augmented_syspath(new_paths):
original_sys_path = sys.path
if new_paths is not None:
sys.path = list(new_paths) + sys.path
try:
yield
finally:
sys.path = original_sys_path
def _test_import_protos():
from grpc_tools import protoc
with _augmented_syspath(
("tools/distrib/python/grpcio_tools/grpc_tools/test/",)):
protos = protoc._protos("simple.proto")
assert protos.SimpleMessage is not None
def _test_import_services():
from grpc_tools import protoc
with _augmented_syspath(
("tools/distrib/python/grpcio_tools/grpc_tools/test/",)):
protos = protoc._protos("simple.proto")
services = protoc._services("simple.proto")
assert services.SimpleMessageServiceStub is not None
def _test_import_services_without_protos():
from grpc_tools import protoc
with _augmented_syspath(
("tools/distrib/python/grpcio_tools/grpc_tools/test/",)):
services = protoc._services("simple.proto")
assert services.SimpleMessageServiceStub is not None
def _test_proto_module_imported_once():
from grpc_tools import protoc
with _augmented_syspath(
("tools/distrib/python/grpcio_tools/grpc_tools/test/",)):
protos = protoc._protos("simple.proto")
services = protoc._services("simple.proto")
complicated_protos = protoc._protos("complicated.proto")
simple_message = protos.SimpleMessage()
complicated_message = complicated_protos.ComplicatedMessage()
assert (simple_message.simpler_message.simplest_message.__class__ is
complicated_message.simplest_message.__class__)
def _test_static_dynamic_combo():
with _augmented_syspath(
("tools/distrib/python/grpcio_tools/grpc_tools/test/",)):
from grpc_tools import protoc
import complicated_pb2
protos = protoc._protos("simple.proto")
static_message = complicated_pb2.ComplicatedMessage()
dynamic_message = protos.SimpleMessage()
assert (dynamic_message.simpler_message.simplest_message.__class__ is
static_message.simplest_message.__class__)
def _test_combined_import():
from grpc_tools import protoc
protos, services = protoc._protos_and_services("simple.proto")
assert protos.SimpleMessage is not None
assert services.SimpleMessageServiceStub is not None
def _test_syntax_errors():
from grpc_tools import protoc
try:
protos = protoc._protos("flawed.proto")
except Exception as e:
error_str = str(e)
assert "flawed.proto" in error_str
assert "17:23" in error_str
assert "21:23" in error_str
else:
assert False, "Compile error expected. None occurred."
class ProtocTest(unittest.TestCase):
def test_import_protos(self):
_run_in_subprocess(_test_import_protos)
def test_import_services(self):
_run_in_subprocess(_test_import_services)
def test_import_services_without_protos(self):
_run_in_subprocess(_test_import_services_without_protos)
def test_proto_module_imported_once(self):
_run_in_subprocess(_test_proto_module_imported_once)
def test_static_dynamic_combo(self):
_run_in_subprocess(_test_static_dynamic_combo)
def test_combined_import(self):
_run_in_subprocess(_test_combined_import)
def test_syntax_errors(self):
_run_in_subprocess(_test_syntax_errors)
if __name__ == '__main__':
unittest.main()

@ -0,0 +1,40 @@
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package simple;
import "simpler.proto";
message SimpleMessage {
string msg = 1;
oneof personal_or_business {
bool personal = 2;
bool business = 3;
};
simpler.SimplerMessage simpler_message = 4;
};
message SimpleMessageRequest {
SimpleMessage simple_msg = 1;
};
message SimpleMessageResponse {
bool understood = 1;
};
service SimpleMessageService {
rpc Tell(SimpleMessageRequest) returns (SimpleMessageResponse);
};

@ -0,0 +1,25 @@
syntax = "proto3";
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package simpler;
import "simplest.proto";
message SimplerMessage {
int64 do_i_even_exist = 1;
simplest.SimplestMessage simplest_message = 2;
};

@ -0,0 +1,22 @@
// Copyright 2020 The gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package simplest;
message SimplestMessage {
int64 i_definitely_dont_exist = 1;
};

@ -16,7 +16,7 @@
set -e
# directories to run against
DIRS="src/core/lib src/core/tsi src/core/ext src/cpp test/core test/cpp include src/compiler src/csharp src/ruby third_party/address_sorting src/objective-c"
DIRS="src/core/lib src/core/tsi src/core/ext src/cpp test/core test/cpp include src/compiler src/csharp src/ruby third_party/address_sorting src/objective-c tools/distrib/python"
# file matching patterns to check
GLOB="*.h *.c *.cc *.m *.mm"

@ -24,7 +24,7 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
${name}')
cd /var/local/git/grpc/test
TEST_TARGETS="//src/python/... //examples/python/..."
TEST_TARGETS="//src/python/... //tools/distrib/python/grpcio_tools/... //examples/python/..."
BAZEL_FLAGS="--spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors"
bazel test ${BAZEL_FLAGS} ${TEST_TARGETS}
bazel test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS}

Loading…
Cancel
Save