Add sanity test for grpcio protoc package

templates/tools/dockerfile/test/sanity/Dockerfile.template

@@ -44,6 +44,16 @@
         python-lxml
   RUN pip install simplejson mako
   
+  #======================================
+  # More sanity test dependencies (bazel)
+  RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > \
+    /etc/apt/sources.list.d/backports.list
+  RUN apt-get update
+  RUN apt-get -t jessie-backports install -y openjdk-8-jdk
+  RUN git clone https://github.com/bazelbuild/bazel.git /bazel
+  RUN cd /bazel && ./compile.sh
+  RUN ln -s /bazel/output/bazel /bin/
+  
   #===================
   # Docker "inception"
   # Note this is quite the ugly hack.

tools/distrib/python/bazel_deps.sh

@@ -31,9 +31,16 @@
 
 cd $(dirname $0)/../../../
 
-docker build -t bazel `realpath ./tools/dockerfile/bazel/`
-docker run -v "`realpath .`:/src/grpc/" \
-  -w /src/grpc/third_party/protobuf              \
-  bazel                                          \
+# First check if bazel is installed on the machine. If it is, then we don't need
+# to invoke the docker bazel.
+if [ "bazel version" ]
+then
+  cd third_party/protobuf
   bazel query 'deps('$1')'
-
+else
+  docker build -t bazel `realpath ./tools/dockerfile/bazel/`
+  docker run -v "`realpath .`:/src/grpc/"          \
+    -w /src/grpc/third_party/protobuf              \
+    bazel                                          \
+    bazel query 'deps('$1')'
+fi

tools/distrib/python/check_grpcio_tools.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import cStringIO
+
+import make_grpcio_tools as make
+
+OUT_OF_DATE_MESSAGE = """file {} is out of date
+
+Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
+
+check_protoc_deps_file = cStringIO.StringIO()
+check_protoc_lib_deps_file = cStringIO.StringIO()
+make.write_deps(make.BAZEL_DEPS_PROTOC_QUERY, check_protoc_deps_file)
+make.write_deps(make.BAZEL_DEPS_PROTOC_LIB_QUERY, check_protoc_lib_deps_file)
+
+with open(make.GRPC_PYTHON_PROTOC_DEPS, 'r') as protoc_deps_file:
+  if protoc_deps_file.read() != check_protoc_deps_file.getvalue():
+    print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_DEPS))
+    raise SystemExit(1)
+
+with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
+  if protoc_lib_deps_file.read() != check_protoc_lib_deps_file.getvalue():
+    print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS))
+    raise SystemExit(1)

tools/distrib/python/grpcio_tools/setup.py

@@ -83,7 +83,7 @@ def maybe_cythonize(exts):
   return Build.cythonize(exts)
 
 setuptools.setup(
-  name='grpcio_protoc',
+  name='grpcio_tools',
   version='0.14.0rc1',
   license='',
   ext_modules=maybe_cythonize([

tools/distrib/python/make_grpcio_tools.py

@@ -64,20 +64,21 @@ DEPS_FILE_CONTENT="""
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# AUTO-GENERATED BY mk_grpcio_protoc.py!
+# AUTO-GENERATED BY mk_grpcio_tools.py!
 CC_FILES={}
 """
 
 # Bazel query result prefix for expected source files in protobuf.
 PROTOBUF_CC_PREFIX = '//:src/'
 
-os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                      '..', '..', '..'))
+GRPC_ROOT = os.path.abspath(
+    os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                 '..', '..', '..'))
 
-GRPC_PYTHON_ROOT = os.path.abspath('tools/distrib/python/grpcio_protoc')
+GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools')
 
-GRPC_PROTOBUF = os.path.abspath('third_party/protobuf/src')
-GRPC_PROTOC_PLUGINS = os.path.abspath('src/compiler')
+GRPC_PROTOBUF = os.path.join(GRPC_ROOT, 'third_party/protobuf/src')
+GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler')
 GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
                                     'third_party/protobuf/src')
 GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT,
@@ -87,32 +88,46 @@ GRPC_PYTHON_PROTOC_DEPS = os.path.join(GRPC_PYTHON_ROOT,
 GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT,
                                            'protoc_lib_deps.py')
 
-GRPC_INCLUDE = os.path.abspath('include')
+GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include')
 GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')
 
-for tree in [GRPC_PYTHON_PROTOBUF,
-             GRPC_PYTHON_PROTOC_PLUGINS,
-             GRPC_PYTHON_INCLUDE]:
-  try:
-    shutil.rmtree(tree)
-  except Exception as _:
-    pass
-shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF)
-shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
-shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
-
-def write_deps(query, out_filename):
-  output = subprocess.check_output(['tools/distrib/python/bazel_deps.sh', query])
+BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
+BAZEL_DEPS_PROTOC_QUERY = '//:protoc'
+BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
+
+
+def write_deps(query, out_file):
+  """Write the result of the bazel query `query` against protobuf to
+     `out_file`."""
+  output = subprocess.check_output([BAZEL_DEPS, query])
   output = output.splitlines()
   cc_files = [
       name for name in output
       if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)]
   cc_files = [cc_file[len(PROTOBUF_CC_PREFIX):] for cc_file in cc_files]
-
   deps_file_content = DEPS_FILE_CONTENT.format(cc_files)
+  out_file.write(deps_file_content)
+
+
+def main():
+  os.chdir(GRPC_ROOT)
+
+  for tree in [GRPC_PYTHON_PROTOBUF,
+               GRPC_PYTHON_PROTOC_PLUGINS,
+               GRPC_PYTHON_INCLUDE]:
+    try:
+      shutil.rmtree(tree)
+    except Exception as _:
+      pass
+  shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF)
+  shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
+  shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
+
+  with open(GRPC_PYTHON_PROTOC_DEPS, 'w') as deps_file:
+    write_deps(BAZEL_DEPS_PROTOC_QUERY, deps_file)
+  with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file:
+    write_deps(BAZEL_DEPS_PROTOC_LIB_QUERY, deps_file)
 
-  with open(out_filename, 'w') as deps_file:
-    deps_file.write(deps_file_content)
+if __name__ == '__main__':
+  main()
 
-write_deps('//:protoc', GRPC_PYTHON_PROTOC_DEPS)
-write_deps('//:protoc_lib', GRPC_PYTHON_PROTOC_LIB_DEPS)

tools/dockerfile/bazel/Dockerfile

@@ -44,10 +44,9 @@ RUN apt-get -y install \
 	git
 
 RUN git clone https://github.com/bazelbuild/bazel.git /bazel
-RUN cd /bazel &&     \
-	./compile.sh
+RUN cd /bazel && ./compile.sh
 
-RUN cp /bazel/output/bazel /bin/bazel
+RUN ln -s /bazel/output/bazel /bin/
 
 # ensure the installation has been extracted
 RUN bazel

tools/dockerfile/test/sanity/Dockerfile

@@ -75,6 +75,15 @@ RUN apt-get update && apt-get install -y \
       python-lxml
 RUN pip install simplejson mako
 
+#======================================
+# More sanity test dependencies (bazel)
+RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" >   /etc/apt/sources.list.d/backports.list
+RUN apt-get update
+RUN apt-get -t jessie-backports install -y openjdk-8-jdk
+RUN git clone https://github.com/bazelbuild/bazel.git /bazel
+RUN cd /bazel && ./compile.sh
+RUN ln -s /bazel/output/bazel /bin/
+
 #===================
 # Docker "inception"
 # Note this is quite the ugly hack.

tools/run_tests/sanity/sanity_tests.yaml

@@ -10,3 +10,4 @@
 - script: tools/distrib/check_trailing_newlines.sh
 - script: tools/distrib/check_nanopb_output.sh
 - script: tools/distrib/check_include_guards.py
+- script: tools/distrib/python/check_grpcio_tools.py