yapf tools/interop_matrix

pull/13719/head
ncteisen 7 years ago
parent 7a2be20a8a
commit e4bef08a8c
  1. 1
      tools/distrib/yapf_code.sh
  2. 269
      tools/interop_matrix/client_matrix.py
  3. 484
      tools/interop_matrix/create_matrix_images.py
  4. 320
      tools/interop_matrix/run_interop_matrix_tests.py

@ -23,6 +23,7 @@ DIRS=(
'tools/buildgen'
'tools/codegen'
'tools/distrib'
'tools/interop_matrix'
)
EXCLUSIONS=(
'grpcio/grpc_*.py'

@ -15,29 +15,34 @@
# Dictionaries used for client matrix testing.
def get_github_repo(lang):
return {
'go': 'git@github.com:grpc/grpc-go.git',
'java': 'git@github.com:grpc/grpc-java.git',
'node': 'git@github.com:grpc/grpc-node.git',
# all other languages use the grpc.git repo.
}.get(lang, 'git@github.com:grpc/grpc.git')
return {
'go': 'git@github.com:grpc/grpc-go.git',
'java': 'git@github.com:grpc/grpc-java.git',
'node': 'git@github.com:grpc/grpc-node.git',
# all other languages use the grpc.git repo.
}.get(lang, 'git@github.com:grpc/grpc.git')
def get_release_tags(lang):
return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang])
return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang])
def get_release_tag_name(release_info):
assert len(release_info.keys()) == 1
return release_info.keys()[0]
assert len(release_info.keys()) == 1
return release_info.keys()[0]
def should_build_docker_interop_image_from_release_tag(lang):
if lang in ['go', 'java', 'node']:
return False
return True
if lang in ['go', 'java', 'node']:
return False
return True
# Dictionary of runtimes per language
LANG_RUNTIME_MATRIX = {
'cxx': ['cxx'], # This is actually debian8.
'cxx': ['cxx'], # This is actually debian8.
'go': ['go1.7', 'go1.8'],
'java': ['java_oracle8'],
'python': ['python'],
@ -51,81 +56,197 @@ LANG_RUNTIME_MATRIX = {
# a release tag pointing to the latest build of the branch.
LANG_RELEASE_MATRIX = {
'cxx': [
{'v1.0.1': None},
{'v1.1.4': None},
{'v1.2.5': None},
{'v1.3.9': None},
{'v1.4.2': None},
{'v1.6.6': None},
{'v1.7.2': None},
{
'v1.0.1': None
},
{
'v1.1.4': None
},
{
'v1.2.5': None
},
{
'v1.3.9': None
},
{
'v1.4.2': None
},
{
'v1.6.6': None
},
{
'v1.7.2': None
},
],
'go': [
{'v1.0.5': None},
{'v1.2.1': None},
{'v1.3.0': None},
{'v1.4.2': None},
{'v1.5.2': None},
{'v1.6.0': None},
{'v1.7.4': None},
{'v1.8.1': None},
{
'v1.0.5': None
},
{
'v1.2.1': None
},
{
'v1.3.0': None
},
{
'v1.4.2': None
},
{
'v1.5.2': None
},
{
'v1.6.0': None
},
{
'v1.7.4': None
},
{
'v1.8.1': None
},
],
'java': [
{'v1.0.3': None},
{'v1.1.2': None},
{'v1.2.0': None},
{'v1.3.1': None},
{'v1.4.0': None},
{'v1.5.0': None},
{'v1.6.1': None},
{'v1.7.0': None},
{'v1.8.0': None},
{
'v1.0.3': None
},
{
'v1.1.2': None
},
{
'v1.2.0': None
},
{
'v1.3.1': None
},
{
'v1.4.0': None
},
{
'v1.5.0': None
},
{
'v1.6.1': None
},
{
'v1.7.0': None
},
{
'v1.8.0': None
},
],
'python': [
{'v1.0.x': None},
{'v1.1.4': None},
{'v1.2.5': None},
{'v1.3.9': None},
{'v1.4.2': None},
{'v1.6.6': None},
{'v1.7.2': None},
{
'v1.0.x': None
},
{
'v1.1.4': None
},
{
'v1.2.5': None
},
{
'v1.3.9': None
},
{
'v1.4.2': None
},
{
'v1.6.6': None
},
{
'v1.7.2': None
},
],
'node': [
{'v1.0.1': None},
{'v1.1.4': None},
{'v1.2.5': None},
{'v1.3.9': None},
{'v1.4.2': None},
{'v1.6.6': None},
{
'v1.0.1': None
},
{
'v1.1.4': None
},
{
'v1.2.5': None
},
{
'v1.3.9': None
},
{
'v1.4.2': None
},
{
'v1.6.6': None
},
#{'v1.7.1': None}, Failing tests
],
'ruby': [
{'v1.0.1': {'patch': [
'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile',
'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh',
]}},
{'v1.1.4': None},
{'v1.2.5': None},
{'v1.3.9': None},
{'v1.4.2': None},
{'v1.6.6': None},
{'v1.7.2': None},
{
'v1.0.1': {
'patch': [
'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile',
'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh',
]
}
},
{
'v1.1.4': None
},
{
'v1.2.5': None
},
{
'v1.3.9': None
},
{
'v1.4.2': None
},
{
'v1.6.6': None
},
{
'v1.7.2': None
},
],
'php': [
{'v1.0.1': None},
{'v1.1.4': None},
{'v1.2.5': None},
{'v1.3.9': None},
{'v1.4.2': None},
{'v1.6.6': None},
{'v1.7.2': None},
{
'v1.0.1': None
},
{
'v1.1.4': None
},
{
'v1.2.5': None
},
{
'v1.3.9': None
},
{
'v1.4.2': None
},
{
'v1.6.6': None
},
{
'v1.7.2': None
},
],
'csharp': [
'csharp': [
#{'v1.0.1': None},
{'v1.1.4': None},
{'v1.2.5': None},
{'v1.3.9': None},
{'v1.4.2': None},
{'v1.6.6': None},
{'v1.7.2': None},
{
'v1.1.4': None
},
{
'v1.2.5': None
},
{
'v1.3.9': None
},
{
'v1.4.2': None
},
{
'v1.6.6': None
},
{
'v1.7.2': None
},
],
}

@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Build and upload docker images to Google Container Registry per matrix."""
from __future__ import print_function
@ -29,8 +28,8 @@ import tempfile
# Langauage Runtime Matrix
import client_matrix
python_util_dir = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../run_tests/python_utils'))
python_util_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../run_tests/python_utils'))
sys.path.append(python_util_dir)
import dockerjob
import jobset
@ -38,267 +37,304 @@ import jobset
_IMAGE_BUILDER = 'tools/run_tests/dockerize/build_interop_image.sh'
_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
# All gRPC release tags, flattened, deduped and sorted.
_RELEASES = sorted(list(set(
client_matrix.get_release_tag_name(info) for lang in client_matrix.LANG_RELEASE_MATRIX.values() for info in lang)))
_RELEASES = sorted(
list(
set(
client_matrix.get_release_tag_name(info)
for lang in client_matrix.LANG_RELEASE_MATRIX.values()
for info in lang)))
# Destination directory inside docker image to keep extra info from build time.
_BUILD_INFO = '/var/local/build_info'
argp = argparse.ArgumentParser(description='Run interop tests.')
argp.add_argument('--gcr_path',
default='gcr.io/grpc-testing',
help='Path of docker images in Google Container Registry')
argp.add_argument('--release',
default='master',
choices=['all', 'master'] + _RELEASES,
help='github commit tag to checkout. When building all '
'releases defined in client_matrix.py, use "all". Valid only '
'with --git_checkout.')
argp.add_argument('-l', '--language',
choices=['all'] + sorted(_LANGUAGES),
nargs='+',
default=['all'],
help='Test languages to build docker images for.')
argp.add_argument('--git_checkout',
action='store_true',
help='Use a separate git clone tree for building grpc stack. '
'Required when using --release flag. By default, current'
'tree and the sibling will be used for building grpc stack.')
argp.add_argument('--git_checkout_root',
default='/export/hda3/tmp/grpc_matrix',
help='Directory under which grpc-go/java/main repo will be '
'cloned. Valid only with --git_checkout.')
argp.add_argument('--keep',
action='store_true',
help='keep the created local images after uploading to GCR')
argp.add_argument('--reuse_git_root',
default=False,
action='store_const',
const=True,
help='reuse the repo dir. If False, the existing git root '
'directory will removed before a clean checkout, because '
'reusing the repo can cause git checkout error if you switch '
'between releases.')
argp.add_argument(
'--gcr_path',
default='gcr.io/grpc-testing',
help='Path of docker images in Google Container Registry')
argp.add_argument(
'--release',
default='master',
choices=['all', 'master'] + _RELEASES,
help='github commit tag to checkout. When building all '
'releases defined in client_matrix.py, use "all". Valid only '
'with --git_checkout.')
argp.add_argument(
'-l',
'--language',
choices=['all'] + sorted(_LANGUAGES),
nargs='+',
default=['all'],
help='Test languages to build docker images for.')
argp.add_argument(
'--git_checkout',
action='store_true',
help='Use a separate git clone tree for building grpc stack. '
'Required when using --release flag. By default, current'
'tree and the sibling will be used for building grpc stack.')
argp.add_argument(
'--git_checkout_root',
default='/export/hda3/tmp/grpc_matrix',
help='Directory under which grpc-go/java/main repo will be '
'cloned. Valid only with --git_checkout.')
argp.add_argument(
'--keep',
action='store_true',
help='keep the created local images after uploading to GCR')
argp.add_argument(
'--reuse_git_root',
default=False,
action='store_const',
const=True,
help='reuse the repo dir. If False, the existing git root '
'directory will removed before a clean checkout, because '
'reusing the repo can cause git checkout error if you switch '
'between releases.')
args = argp.parse_args()
def add_files_to_image(image, with_files, label=None):
"""Add files to a docker image.
"""Add files to a docker image.
image: docker image name, i.e. grpc_interop_java:26328ad8
with_files: additional files to include in the docker image.
label: label string to attach to the image.
"""
tag_idx = image.find(':')
if tag_idx == -1:
jobset.message('FAILED', 'invalid docker image %s' % image, do_newline=True)
sys.exit(1)
orig_tag = '%s_' % image
subprocess.check_output(['docker', 'tag', image, orig_tag])
lines = ['FROM ' + orig_tag]
if label:
lines.append('LABEL %s' % label)
temp_dir = tempfile.mkdtemp()
atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir]))
# Copy with_files inside the tmp directory, which will be the docker build
# context.
for f in with_files:
shutil.copy(f, temp_dir)
lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO))
# Create a Dockerfile.
with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f:
f.write('\n'.join(lines))
jobset.message('START', 'Repackaging %s' % image, do_newline=True)
build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir]
subprocess.check_output(build_cmd)
dockerjob.remove_image(orig_tag, skip_nonexistent=True)
tag_idx = image.find(':')
if tag_idx == -1:
jobset.message(
'FAILED', 'invalid docker image %s' % image, do_newline=True)
sys.exit(1)
orig_tag = '%s_' % image
subprocess.check_output(['docker', 'tag', image, orig_tag])
lines = ['FROM ' + orig_tag]
if label:
lines.append('LABEL %s' % label)
temp_dir = tempfile.mkdtemp()
atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir]))
# Copy with_files inside the tmp directory, which will be the docker build
# context.
for f in with_files:
shutil.copy(f, temp_dir)
lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO))
# Create a Dockerfile.
with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f:
f.write('\n'.join(lines))
jobset.message('START', 'Repackaging %s' % image, do_newline=True)
build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir]
subprocess.check_output(build_cmd)
dockerjob.remove_image(orig_tag, skip_nonexistent=True)
def build_image_jobspec(runtime, env, gcr_tag, stack_base):
"""Build interop docker image for a language with runtime.
"""Build interop docker image for a language with runtime.
runtime: a <lang><version> string, for example go1.8.
env: dictionary of env to passed to the build script.
gcr_tag: the tag for the docker image (i.e. v1.3.0).
stack_base: the local gRPC repo path.
"""
basename = 'grpc_interop_%s' % runtime
tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag)
build_env = {
'INTEROP_IMAGE': tag,
'BASE_NAME': basename,
'TTY_FLAG': '-t'
}
build_env.update(env)
image_builder_path = _IMAGE_BUILDER
if client_matrix.should_build_docker_interop_image_from_release_tag(lang):
image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER)
build_job = jobset.JobSpec(
cmdline=[image_builder_path],
environ=build_env,
shortname='build_docker_%s' % runtime,
timeout_seconds=30*60)
build_job.tag = tag
return build_job
basename = 'grpc_interop_%s' % runtime
tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag)
build_env = {'INTEROP_IMAGE': tag, 'BASE_NAME': basename, 'TTY_FLAG': '-t'}
build_env.update(env)
image_builder_path = _IMAGE_BUILDER
if client_matrix.should_build_docker_interop_image_from_release_tag(lang):
image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER)
build_job = jobset.JobSpec(
cmdline=[image_builder_path],
environ=build_env,
shortname='build_docker_%s' % runtime,
timeout_seconds=30 * 60)
build_job.tag = tag
return build_job
def build_all_images_for_lang(lang):
"""Build all docker images for a language across releases and runtimes."""
if not args.git_checkout:
if args.release != 'master':
print('WARNING: --release is set but will be ignored\n')
releases = ['master']
else:
if args.release == 'all':
releases = client_matrix.get_release_tags(lang)
"""Build all docker images for a language across releases and runtimes."""
if not args.git_checkout:
if args.release != 'master':
print('WARNING: --release is set but will be ignored\n')
releases = ['master']
else:
# Build a particular release.
if args.release not in ['master'] + client_matrix.get_release_tags(lang):
jobset.message('SKIPPED',
'%s for %s is not defined' % (args.release, lang),
do_newline=True)
return []
releases = [args.release]
images = []
for release in releases:
images += build_all_images_for_release(lang, release)
jobset.message('SUCCESS',
'All docker images built for %s at %s.' % (lang, releases),
do_newline=True)
return images
if args.release == 'all':
releases = client_matrix.get_release_tags(lang)
else:
# Build a particular release.
if args.release not in ['master'] + client_matrix.get_release_tags(
lang):
jobset.message(
'SKIPPED',
'%s for %s is not defined' % (args.release, lang),
do_newline=True)
return []
releases = [args.release]
images = []
for release in releases:
images += build_all_images_for_release(lang, release)
jobset.message(
'SUCCESS',
'All docker images built for %s at %s.' % (lang, releases),
do_newline=True)
return images
def build_all_images_for_release(lang, release):
"""Build all docker images for a release across all runtimes."""
docker_images = []
build_jobs = []
env = {}
# If we not using current tree or the sibling for grpc stack, do checkout.
stack_base = ''
if args.git_checkout:
stack_base = checkout_grpc_stack(lang, release)
var ={'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT', 'node': 'GRPC_NODE_ROOT'}.get(lang, 'GRPC_ROOT')
env[var] = stack_base
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
job = build_image_jobspec(runtime, env, release, stack_base)
docker_images.append(job.tag)
build_jobs.append(job)
jobset.message('START', 'Building interop docker images.', do_newline=True)
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=multiprocessing.cpu_count())
if num_failures:
jobset.message('FAILED', 'Failed to build interop docker images.',
do_newline=True)
docker_images_cleanup.extend(docker_images)
sys.exit(1)
jobset.message('SUCCESS',
'All docker images built for %s at %s.' % (lang, release),
do_newline=True)
if release != 'master':
commit_log = os.path.join(stack_base, 'commit_log')
if os.path.exists(commit_log):
for image in docker_images:
add_files_to_image(image, [commit_log], 'release=%s' % release)
return docker_images
"""Build all docker images for a release across all runtimes."""
docker_images = []
build_jobs = []
env = {}
# If we not using current tree or the sibling for grpc stack, do checkout.
stack_base = ''
if args.git_checkout:
stack_base = checkout_grpc_stack(lang, release)
var = {
'go': 'GRPC_GO_ROOT',
'java': 'GRPC_JAVA_ROOT',
'node': 'GRPC_NODE_ROOT'
}.get(lang, 'GRPC_ROOT')
env[var] = stack_base
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
job = build_image_jobspec(runtime, env, release, stack_base)
docker_images.append(job.tag)
build_jobs.append(job)
jobset.message('START', 'Building interop docker images.', do_newline=True)
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(
build_jobs,
newline_on_success=True,
maxjobs=multiprocessing.cpu_count())
if num_failures:
jobset.message(
'FAILED', 'Failed to build interop docker images.', do_newline=True)
docker_images_cleanup.extend(docker_images)
sys.exit(1)
jobset.message(
'SUCCESS',
'All docker images built for %s at %s.' % (lang, release),
do_newline=True)
if release != 'master':
commit_log = os.path.join(stack_base, 'commit_log')
if os.path.exists(commit_log):
for image in docker_images:
add_files_to_image(image, [commit_log], 'release=%s' % release)
return docker_images
def cleanup():
if not args.keep:
for image in docker_images_cleanup:
dockerjob.remove_image(image, skip_nonexistent=True)
if not args.keep:
for image in docker_images_cleanup:
dockerjob.remove_image(image, skip_nonexistent=True)
docker_images_cleanup = []
atexit.register(cleanup)
def maybe_apply_patches_on_git_tag(stack_base, lang, release):
files_to_patch = []
for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]:
if client_matrix.get_release_tag_name(release_info) == release:
files_to_patch = release_info[release].get('patch')
break
if not files_to_patch:
return
patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release)
patch_file = os.path.abspath(os.path.join(os.path.dirname(__file__),
patch_file_relative_path))
if not os.path.exists(patch_file):
jobset.message('FAILED', 'expected patch file |%s| to exist' % patch_file)
sys.exit(1)
subprocess.check_output(
['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT)
for repo_relative_path in files_to_patch:
files_to_patch = []
for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]:
if client_matrix.get_release_tag_name(release_info) == release:
files_to_patch = release_info[release].get('patch')
break
if not files_to_patch:
return
patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release)
patch_file = os.path.abspath(
os.path.join(os.path.dirname(__file__), patch_file_relative_path))
if not os.path.exists(patch_file):
jobset.message('FAILED', 'expected patch file |%s| to exist' %
patch_file)
sys.exit(1)
subprocess.check_output(
['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT)
for repo_relative_path in files_to_patch:
subprocess.check_output(
['git', 'add', repo_relative_path],
cwd=stack_base,
stderr=subprocess.STDOUT)
subprocess.check_output(
['git', 'add', repo_relative_path],
[
'git', 'commit', '-m',
('Hack performed on top of %s git '
'tag in order to build and run the %s '
'interop tests on that tag.' % (lang, release))
],
cwd=stack_base,
stderr=subprocess.STDOUT)
subprocess.check_output(
['git', 'commit', '-m', ('Hack performed on top of %s git '
'tag in order to build and run the %s '
'interop tests on that tag.' % (lang, release))],
cwd=stack_base, stderr=subprocess.STDOUT)
def checkout_grpc_stack(lang, release):
"""Invokes 'git check' for the lang/release and returns directory created."""
assert args.git_checkout and args.git_checkout_root
if not os.path.exists(args.git_checkout_root):
os.makedirs(args.git_checkout_root)
repo = client_matrix.get_github_repo(lang)
# Get the subdir name part of repo
# For example, 'git@github.com:grpc/grpc-go.git' should use 'grpc-go'.
repo_dir = os.path.splitext(os.path.basename(repo))[0]
stack_base = os.path.join(args.git_checkout_root, repo_dir)
# Clean up leftover repo dir if necessary.
if not args.reuse_git_root and os.path.exists(stack_base):
jobset.message('START', 'Removing git checkout root.', do_newline=True)
shutil.rmtree(stack_base)
if not os.path.exists(stack_base):
subprocess.check_call(['git', 'clone', '--recursive', repo],
cwd=os.path.dirname(stack_base))
# git checkout.
jobset.message('START', 'git checkout %s from %s' % (release, stack_base),
do_newline=True)
# We should NEVER do checkout on current tree !!!
assert not os.path.dirname(__file__).startswith(stack_base)
output = subprocess.check_output(
['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT)
maybe_apply_patches_on_git_tag(stack_base, lang, release)
commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base)
jobset.message('SUCCESS', 'git checkout',
'%s: %s' % (str(output), commit_log),
do_newline=True)
# Write git log to commit_log so it can be packaged with the docker image.
with open(os.path.join(stack_base, 'commit_log'), 'w') as f:
f.write(commit_log)
return stack_base
"""Invokes 'git check' for the lang/release and returns directory created."""
assert args.git_checkout and args.git_checkout_root
if not os.path.exists(args.git_checkout_root):
os.makedirs(args.git_checkout_root)
repo = client_matrix.get_github_repo(lang)
# Get the subdir name part of repo
# For example, 'git@github.com:grpc/grpc-go.git' should use 'grpc-go'.
repo_dir = os.path.splitext(os.path.basename(repo))[0]
stack_base = os.path.join(args.git_checkout_root, repo_dir)
# Clean up leftover repo dir if necessary.
if not args.reuse_git_root and os.path.exists(stack_base):
jobset.message('START', 'Removing git checkout root.', do_newline=True)
shutil.rmtree(stack_base)
if not os.path.exists(stack_base):
subprocess.check_call(
['git', 'clone', '--recursive', repo],
cwd=os.path.dirname(stack_base))
# git checkout.
jobset.message(
'START',
'git checkout %s from %s' % (release, stack_base),
do_newline=True)
# We should NEVER do checkout on current tree !!!
assert not os.path.dirname(__file__).startswith(stack_base)
output = subprocess.check_output(
['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT)
maybe_apply_patches_on_git_tag(stack_base, lang, release)
commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base)
jobset.message(
'SUCCESS',
'git checkout',
'%s: %s' % (str(output), commit_log),
do_newline=True)
# Write git log to commit_log so it can be packaged with the docker image.
with open(os.path.join(stack_base, 'commit_log'), 'w') as f:
f.write(commit_log)
return stack_base
languages = args.language if args.language != ['all'] else _LANGUAGES
for lang in languages:
docker_images = build_all_images_for_lang(lang)
for image in docker_images:
jobset.message('START', 'Uploading %s' % image, do_newline=True)
# docker image name must be in the format <gcr_path>/<image>:<gcr_tag>
assert image.startswith(args.gcr_path) and image.find(':') != -1
docker_images = build_all_images_for_lang(lang)
for image in docker_images:
jobset.message('START', 'Uploading %s' % image, do_newline=True)
# docker image name must be in the format <gcr_path>/<image>:<gcr_tag>
assert image.startswith(args.gcr_path) and image.find(':') != -1
subprocess.call(['gcloud', 'docker', '--', 'push', image])
subprocess.call(['gcloud', 'docker', '--', 'push', image])

@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run tests using docker images in Google Container Registry per matrix."""
from __future__ import print_function
@ -30,8 +29,8 @@ import uuid
# Langauage Runtime Matrix
import client_matrix
python_util_dir = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../run_tests/python_utils'))
python_util_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../run_tests/python_utils'))
sys.path.append(python_util_dir)
import dockerjob
import jobset
@ -40,46 +39,56 @@ import upload_test_results
_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
# All gRPC release tags, flattened, deduped and sorted.
_RELEASES = sorted(list(set(
client_matrix.get_release_tag_name(info) for lang in client_matrix.LANG_RELEASE_MATRIX.values() for info in lang)))
_RELEASES = sorted(
list(
set(
client_matrix.get_release_tag_name(info)
for lang in client_matrix.LANG_RELEASE_MATRIX.values()
for info in lang)))
_TEST_TIMEOUT = 30
argp = argparse.ArgumentParser(description='Run interop tests.')
argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
argp.add_argument('--gcr_path',
default='gcr.io/grpc-testing',
help='Path of docker images in Google Container Registry')
argp.add_argument('--release',
default='all',
choices=['all', 'master'] + _RELEASES,
help='Release tags to test. When testing all '
'releases defined in client_matrix.py, use "all".')
argp.add_argument('-l', '--language',
choices=['all'] + sorted(_LANGUAGES),
nargs='+',
default=['all'],
help='Languages to test')
argp.add_argument('--keep',
action='store_true',
help='keep the created local images after finishing the tests.')
argp.add_argument('--report_file',
default='report.xml',
help='The result file to create.')
argp.add_argument('--allow_flakes',
default=False,
action='store_const',
const=True,
help=('Allow flaky tests to show as passing (re-runs failed '
'tests up to five times)'))
argp.add_argument('--bq_result_table',
default='',
type=str,
nargs='?',
help='Upload test results to a specified BQ table.')
argp.add_argument(
'--gcr_path',
default='gcr.io/grpc-testing',
help='Path of docker images in Google Container Registry')
argp.add_argument(
'--release',
default='all',
choices=['all', 'master'] + _RELEASES,
help='Release tags to test. When testing all '
'releases defined in client_matrix.py, use "all".')
argp.add_argument(
'-l',
'--language',
choices=['all'] + sorted(_LANGUAGES),
nargs='+',
default=['all'],
help='Languages to test')
argp.add_argument(
'--keep',
action='store_true',
help='keep the created local images after finishing the tests.')
argp.add_argument(
'--report_file', default='report.xml', help='The result file to create.')
argp.add_argument(
'--allow_flakes',
default=False,
action='store_const',
const=True,
help=('Allow flaky tests to show as passing (re-runs failed '
'tests up to five times)'))
argp.add_argument(
'--bq_result_table',
default='',
type=str,
nargs='?',
help='Upload test results to a specified BQ table.')
args = argp.parse_args()
@ -87,138 +96,153 @@ print(str(args))
def find_all_images_for_lang(lang):
"""Find docker images for a language across releases and runtimes.
"""Find docker images for a language across releases and runtimes.
Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime.
"""
# Find all defined releases.
if args.release == 'all':
releases = ['master'] + client_matrix.get_release_tags(lang)
else:
# Look for a particular release.
if args.release not in ['master'] + client_matrix.get_release_tags(lang):
jobset.message('SKIPPED',
'%s for %s is not defined' % (args.release, lang),
do_newline=True)
return {}
releases = [args.release]
# Images tuples keyed by runtime.
images = {}
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
output = subprocess.check_output(['gcloud', 'beta', 'container', 'images',
'list-tags', '--format=json', image_path])
docker_image_list = json.loads(output)
# All images should have a single tag or no tag.
# TODO(adelez): Remove tagless images.
tags = [i['tags'][0] for i in docker_image_list if i['tags']]
jobset.message('START', 'Found images for %s: %s' % (image_path, tags),
do_newline=True)
skipped = len(docker_image_list) - len(tags)
jobset.message('SKIPPED', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
do_newline=True)
# Filter tags based on the releases.
images[runtime] = [(tag,'%s:%s' % (image_path,tag)) for tag in tags if
tag in releases]
return images
# Find all defined releases.
if args.release == 'all':
releases = ['master'] + client_matrix.get_release_tags(lang)
else:
# Look for a particular release.
if args.release not in ['master'] + client_matrix.get_release_tags(
lang):
jobset.message(
'SKIPPED',
'%s for %s is not defined' % (args.release, lang),
do_newline=True)
return {}
releases = [args.release]
# Images tuples keyed by runtime.
images = {}
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
output = subprocess.check_output([
'gcloud', 'beta', 'container', 'images', 'list-tags',
'--format=json', image_path
])
docker_image_list = json.loads(output)
# All images should have a single tag or no tag.
# TODO(adelez): Remove tagless images.
tags = [i['tags'][0] for i in docker_image_list if i['tags']]
jobset.message(
'START',
'Found images for %s: %s' % (image_path, tags),
do_newline=True)
skipped = len(docker_image_list) - len(tags)
jobset.message(
'SKIPPED',
'Skipped images (no-tag/unknown-tag): %d' % skipped,
do_newline=True)
# Filter tags based on the releases.
images[runtime] = [(tag, '%s:%s' % (image_path, tag)) for tag in tags
if tag in releases]
return images
# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime.
def find_test_cases(lang, runtime, release, suite_name):
"""Returns the list of test cases from testcase files per lang/release."""
file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s')
testcase_release = release
filename_prefix = lang
if lang == 'csharp':
filename_prefix = runtime
if not os.path.exists(file_tmpl % (filename_prefix, release)):
testcase_release = 'master'
testcases = file_tmpl % (filename_prefix, testcase_release)
job_spec_list=[]
try:
with open(testcases) as f:
# Only line start with 'docker run' are test cases.
for line in f.readlines():
if line.startswith('docker run'):
m = re.search('--test_case=(.*)"', line)
shortname = m.group(1) if m else 'unknown_test'
m = re.search('--server_host_override=(.*).sandbox.googleapis.com',
"""Returns the list of test cases from testcase files per lang/release."""
file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s')
testcase_release = release
filename_prefix = lang
if lang == 'csharp':
filename_prefix = runtime
if not os.path.exists(file_tmpl % (filename_prefix, release)):
testcase_release = 'master'
testcases = file_tmpl % (filename_prefix, testcase_release)
job_spec_list = []
try:
with open(testcases) as f:
# Only line start with 'docker run' are test cases.
for line in f.readlines():
if line.startswith('docker run'):
m = re.search('--test_case=(.*)"', line)
shortname = m.group(1) if m else 'unknown_test'
m = re.search(
'--server_host_override=(.*).sandbox.googleapis.com',
line)
server = m.group(1) if m else 'unknown_server'
spec = jobset.JobSpec(cmdline=line,
shortname='%s:%s:%s:%s' % (suite_name, lang,
server, shortname),
timeout_seconds=_TEST_TIMEOUT,
shell=True,
flake_retries=5 if args.allow_flakes else 0)
job_spec_list.append(spec)
jobset.message('START',
'Loaded %s tests from %s' % (len(job_spec_list), testcases),
do_newline=True)
except IOError as err:
jobset.message('FAILED', err, do_newline=True)
return job_spec_list
server = m.group(1) if m else 'unknown_server'
spec = jobset.JobSpec(
cmdline=line,
shortname='%s:%s:%s:%s' % (suite_name, lang, server,
shortname),
timeout_seconds=_TEST_TIMEOUT,
shell=True,
flake_retries=5 if args.allow_flakes else 0)
job_spec_list.append(spec)
jobset.message(
'START',
'Loaded %s tests from %s' % (len(job_spec_list), testcases),
do_newline=True)
except IOError as err:
jobset.message('FAILED', err, do_newline=True)
return job_spec_list
_xml_report_tree = report_utils.new_junit_xml_tree()
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message('FAILED', 'No test cases were found.', do_newline=True)
return 1
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image':image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(
_xml_report_tree,
resultset,
'grpc_interop_matrix',
suite_name,
str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message(
'FAILED', 'No test cases were found.', do_newline=True)
return 1
num_failures, resultset = jobset.run(
job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix', suite_name,
str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
def cleanup(image):
jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True)
dockerjob.remove_image(image, skip_nonexistent=True)
jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True)
dockerjob.remove_image(image, skip_nonexistent=True)
languages = args.language if args.language != ['all'] else _LANGUAGES
total_num_failures = 0
for lang in languages:
docker_images = find_all_images_for_lang(lang)
for runtime in sorted(docker_images.keys()):
total_num_failures += run_tests_for_lang(lang, runtime, docker_images[runtime])
docker_images = find_all_images_for_lang(lang)
for runtime in sorted(docker_images.keys()):
total_num_failures += run_tests_for_lang(lang, runtime,
docker_images[runtime])
report_utils.create_xml_report_file(_xml_report_tree, args.report_file)
if total_num_failures:
sys.exit(1)
sys.exit(1)
sys.exit(0)

Loading…
Cancel
Save