Add common proto files to Python grpcio-tools

pull/6828/head
Masood Malekghassemi 9 years ago
parent e5d37dd50e
commit 2dbc217e5e
  1. 2
      tools/distrib/python/check_grpcio_tools.py
  2. 5
      tools/distrib/python/grpcio_tools/grpc/tools/protoc.py
  3. 4
      tools/distrib/python/grpcio_tools/protoc_lib_deps.py
  4. 33
      tools/distrib/python/grpcio_tools/setup.py
  5. 35
      tools/distrib/python/make_grpcio_tools.py

@ -37,7 +37,7 @@ OUT_OF_DATE_MESSAGE = """file {} is out of date
Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
check_protoc_lib_deps_content = make.get_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY)
check_protoc_lib_deps_content = make.get_deps()
with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
if protoc_lib_deps_file.read() != check_protoc_lib_deps_content:

@ -29,10 +29,13 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pkg_resources
import sys
from grpc.tools import protoc_compiler
if __name__ == '__main__':
protoc_compiler.run_main(sys.argv)
proto_include = pkg_resources.resource_filename('grpc.tools', '_proto')
protoc_compiler.run_main(
sys.argv + ['-I{}'.format(proto_include)])

File diff suppressed because one or more lines are too long

@ -28,9 +28,11 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from distutils import extension
import errno
import os
import os.path
import shlex
import shutil
import sys
import setuptools
@ -47,18 +49,41 @@ sys.path.insert(0, os.path.abspath('.'))
# ourselves in w.r.t. the multitude of operating systems this ought to build on.
# By default we assume a GCC-like compiler.
EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
'-frtti -std=c++11'))
'-fno-wrapv -frtti -std=c++11'))
EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
'-lpthread'))
GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
import protoc_lib_deps
import grpc_version
def package_data():
tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep)
proto_resources_path = os.path.join(tools_path,
GRPC_PYTHON_PROTO_RESOURCES_NAME)
proto_files = []
for proto_file in protoc_lib_deps.PROTO_FILES:
source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file)
target = os.path.join(proto_resources_path, proto_file)
relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file)
try:
os.makedirs(os.path.dirname(target))
except OSError as error:
if error.errno == errno.EEXIST:
pass
else:
raise
shutil.copy(source, target)
proto_files.append(relative_target)
return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files}
def protoc_ext_module():
plugin_sources = [
'grpc/tools/main.cc',
'grpc_root/src/compiler/python_generator.cc'] + [
os.path.join('third_party/protobuf/src', cc_file)
os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
for cc_file in protoc_lib_deps.CC_FILES]
plugin_ext = extension.Extension(
name='grpc.tools.protoc_compiler',
@ -67,7 +92,7 @@ def protoc_ext_module():
'.',
'grpc_root',
'grpc_root/include',
'third_party/protobuf/src',
protoc_lib_deps.CC_INCLUDE,
],
language='c++',
define_macros=[('HAVE_PTHREAD', 1)],
@ -92,5 +117,7 @@ setuptools.setup(
#namespace_packages=['grpc'],
install_requires=[
'protobuf>=3.0.0a3',
'grpcio>=0.14.0',
],
package_data=package_data(),
)

@ -67,11 +67,16 @@ DEPS_FILE_CONTENT="""
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# AUTO-GENERATED BY make_grpcio_tools.py!
CC_FILES={}
CC_FILES={cc_files}
PROTO_FILES={proto_files}
CC_INCLUDE={cc_include}
PROTO_INCLUDE={proto_include}
"""
# Bazel query result prefix for expected source files in protobuf.
PROTOBUF_CC_PREFIX = '//:src/'
PROTOBUF_PROTO_PREFIX = '//:src/'
GRPC_ROOT = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
@ -79,7 +84,8 @@ GRPC_ROOT = os.path.abspath(
GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools')
GRPC_PROTOBUF = os.path.join(GRPC_ROOT, 'third_party/protobuf/src')
GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = 'third_party/protobuf/src'
GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler')
GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
'third_party/protobuf/src')
@ -93,18 +99,29 @@ GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')
BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
def bazel_query(query):
output = subprocess.check_output([BAZEL_DEPS, query])
return output.splitlines()
def get_deps(query):
def get_deps():
"""Write the result of the bazel query `query` against protobuf to
`out_file`."""
output = subprocess.check_output([BAZEL_DEPS, query])
output = output.splitlines()
cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY)
cc_files = [
name for name in output
name[len(PROTOBUF_CC_PREFIX):] for name in cc_files_output
if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)]
cc_files = [cc_file[len(PROTOBUF_CC_PREFIX):] for cc_file in cc_files]
deps_file_content = DEPS_FILE_CONTENT.format(cc_files)
proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY)
proto_files = [
name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
deps_file_content = DEPS_FILE_CONTENT.format(
cc_files=cc_files,
proto_files=proto_files,
cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
return deps_file_content
@ -123,7 +140,7 @@ def main():
shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
try:
protoc_lib_deps_content = get_deps(BAZEL_DEPS_PROTOC_LIB_QUERY)
protoc_lib_deps_content = get_deps()
except Exception as error:
# We allow this script to succeed even if we couldn't get the dependencies,
# as then we can assume that even without a successful bazel run the

Loading…
Cancel
Save