Do not require bazel for check_grpcio_tools.py

The `make_grpcio_tools.py` rube-goldberg machinery relies on
`bazel` to extract the list of files required to compile the
`grpcio-tools` package that are provided by the `protobuf`
submodule.  In order to ensure that list is up to date,
`check_grpcio_tools.py` sanity check does the same `bazel`
query, and checks the full contents against the already
existing list in the repository.  This has the downside of
requiring `bazel` to run that particular check at sanity test
time, and flakiness has been seen there.

This commit changes the code generation process to include
the git hash of the submodule at the time `make_grpcio_tools.py`
is invoked and stores it in the generated file and instead of
doing the whole process at test time, the `check_grpcio_tools.py`
sanity test simply checks the submodule version at test time
and verifies it against the version included in the file by
`make_grpcio_tools.py` thus removing the `bazel` dependency
at test time and increasing test robustness and speed.
pull/13463/head
Mehrdad Afshari 7 years ago
parent ad671ff561
commit 9b4315682b
  1. 20
      tools/distrib/python/check_grpcio_tools.py
  2. 4
      tools/distrib/python/grpcio_tools/protoc_lib_deps.py
  3. 20
      tools/distrib/python/make_grpcio_tools.py

@ -14,17 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import cStringIO
import make_grpcio_tools as make
import make_grpcio_tools as _make
OUT_OF_DATE_MESSAGE = """file {} is out of date
Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?"""
check_protoc_lib_deps_content = make.get_deps()
submodule_commit_hash = _make.protobuf_submodule_commit_hash()
with open(_make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as _protoc_lib_deps_file:
content = _protoc_lib_deps_file.read().splitlines()
testString = (_make.COMMIT_HASH_PREFIX +
submodule_commit_hash +
_make.COMMIT_HASH_SUFFIX)
with open(make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as protoc_lib_deps_file:
if protoc_lib_deps_file.read() != check_protoc_lib_deps_content:
print(OUT_OF_DATE_MESSAGE.format(make.GRPC_PYTHON_PROTOC_LIB_DEPS))
raise SystemExit(1)
if testString not in content:
print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS))
raise SystemExit(1)

@ -1,5 +1,5 @@
# Copyright 2016 gRPC authors.
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -19,3 +19,5 @@ PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'go
CC_INCLUDE='third_party/protobuf/src'
PROTO_INCLUDE='third_party/protobuf/src'
PROTOBUF_SUBMODULE_VERSION="80a37e0782d2d702d52234b62dd4b9ec74fd2c95"

@ -28,7 +28,7 @@ import traceback
import uuid
DEPS_FILE_CONTENT="""
# Copyright 2016 gRPC authors.
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -48,8 +48,13 @@ PROTO_FILES={proto_files}
CC_INCLUDE={cc_include}
PROTO_INCLUDE={proto_include}
{commit_hash}
"""
COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="'
COMMIT_HASH_SUFFIX = '"'
# Bazel query result prefix for expected source files in protobuf.
PROTOBUF_CC_PREFIX = '//:src/'
PROTOBUF_PROTO_PREFIX = '//:src/'
@ -63,6 +68,7 @@ GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib',
GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src')
GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'protobuf')
GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler')
GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf',
'src')
@ -78,6 +84,14 @@ BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.s
BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
def protobuf_submodule_commit_hash():
"""Gets the commit hash for the HEAD of the protobuf submodule currently
checked out."""
cwd = os.getcwd()
os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT)
output = subprocess.check_output(['git', 'rev-parse', 'HEAD'])
os.chdir(cwd)
return output.splitlines()[0].strip()
def bazel_query(query):
output = subprocess.check_output([BAZEL_DEPS, query])
@ -94,11 +108,13 @@ def get_deps():
proto_files = [
name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output
if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)]
commit_hash = protobuf_submodule_commit_hash()
deps_file_content = DEPS_FILE_CONTENT.format(
cc_files=cc_files,
proto_files=proto_files,
cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT),
commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX)
return deps_file_content
def long_path(path):

Loading…
Cancel
Save