From 4f11b9650099212b9bb3c01b6b73e977a20c2c8e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Jan 2019 16:35:35 +0100 Subject: [PATCH 1/6] stop testing older releases with go1.7: 1.8 should be enough --- tools/interop_matrix/README.md | 4 ++-- tools/interop_matrix/client_matrix.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 6676f5d470c..531bef82b8e 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -42,10 +42,10 @@ For more details on each step, refer to sections below. - The output for all the test cases is recorded in a junit style xml file (default to 'report.xml'). ## Instructions for running test cases against a GCR image manually -- Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.7:master`. +- Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.8:master`. - Run test cases by specifying `docker_image` variable inline with the test case script created above. For example: - - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.7:master ./testcases/go__master` will run go__master test cases against `go1.7` with gRPC release `master` docker image in GCR. + - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.8:master ./testcases/go__master` will run go__master test cases against `go1.8` with gRPC release `master` docker image in GCR. Note: - File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`). diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 12051e70a0b..9a7c90acd32 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -57,7 +57,7 @@ def should_build_docker_interop_image_from_release_tag(lang): # Dictionary of runtimes per language LANG_RUNTIME_MATRIX = { 'cxx': ['cxx'], # This is actually debian8. - 'go': ['go1.7', 'go1.8', 'go1.11'], + 'go': ['go1.8', 'go1.11'], 'java': ['java_oracle8'], 'python': ['python'], 'node': ['node'], @@ -116,7 +116,7 @@ LANG_RELEASE_MATRIX = { ('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])), ('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])), ('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.17.0', ReleaseInfo(skip_runtime=['go1.7', 'go1.8'])), + ('v1.17.0', ReleaseInfo(skip_runtime=['go1.8'])), ]), 'java': OrderedDict([ From bd5e15608d217e7a5efcaac73dc45f7d1be28a21 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Jan 2019 16:44:43 +0100 Subject: [PATCH 2/6] minor doc updates --- tools/interop_matrix/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 531bef82b8e..5a8965757ee 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -11,8 +11,8 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee - Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php` - Verify that the new docker image was built successfully and uploaded to GCR. For example, - - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. + - `gcloud container images list --repository gcr.io/grpc-testing` lists available images. + - `gcloud container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. - Verify the just-created docker client image would pass backward compatibility test (it should). For example, - `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 tools/interop_matrix/testcases/java__master` From 2a997c66eada4015912230a482d3b898ec526800 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Jan 2019 16:51:51 +0100 Subject: [PATCH 3/6] allow creating images for non-yet-listed releases --- tools/interop_matrix/client_matrix.py | 2 +- tools/interop_matrix/create_matrix_images.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 9a7c90acd32..ec499a98b61 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -36,7 +36,7 @@ def get_release_tags(lang): def get_runtimes_for_lang_release(lang, release): """Get list of valid runtimes for given release of lang.""" runtimes_to_skip = [] - release_info = LANG_RELEASE_MATRIX[lang][release] + release_info = LANG_RELEASE_MATRIX[lang].get(release) if release_info: runtimes_to_skip = release_info.skip_runtime return [ diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py index 31a0e1c7ba6..28dc4be0f4d 100755 --- a/tools/interop_matrix/create_matrix_images.py +++ b/tools/interop_matrix/create_matrix_images.py @@ -260,7 +260,7 @@ atexit.register(cleanup) def maybe_apply_patches_on_git_tag(stack_base, lang, release): files_to_patch = [] - release_info = client_matrix.LANG_RELEASE_MATRIX[lang][release] + release_info = client_matrix.LANG_RELEASE_MATRIX[lang].get(release) if release_info: files_to_patch = release_info.patch if not files_to_patch: From 0d1743b3cc2a24dc2a17fb8e87e2fc8795d44585 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Jan 2019 17:11:54 +0100 Subject: [PATCH 4/6] having GCR images with ":master" tag is an antipattern. --- tools/interop_matrix/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 5a8965757ee..a2c3189bbd6 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -42,10 +42,10 @@ For more details on each step, refer to sections below. - The output for all the test cases is recorded in a junit style xml file (default to 'report.xml'). ## Instructions for running test cases against a GCR image manually -- Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.8:master`. +- Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0`. - Run test cases by specifying `docker_image` variable inline with the test case script created above. For example: - - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.8:master ./testcases/go__master` will run go__master test cases against `go1.8` with gRPC release `master` docker image in GCR. + - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0 ./testcases/go__master` will run go__master test cases against `go1.8` with gRPC release `v1.16.0` docker image in GCR. Note: - File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`). From b2044fdd6f8a74cbd489affee7f195b82bb15aac Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Jan 2019 17:22:09 +0100 Subject: [PATCH 5/6] remove confusing remarks from interop_matrix README.md --- tools/interop_matrix/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index a2c3189bbd6..ecd71be7f87 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -13,6 +13,7 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee - Verify that the new docker image was built successfully and uploaded to GCR. For example, - `gcloud container images list --repository gcr.io/grpc-testing` lists available images. - `gcloud container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. + - images can also be viewed in https://pantheon.corp.google.com/gcr/images/grpc-testing?project=grpc-testing - Verify the just-created docker client image would pass backward compatibility test (it should). For example, - `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 tools/interop_matrix/testcases/java__master` @@ -20,13 +21,11 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee - (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`. For more details on each step, refer to sections below. -## Instructions for adding new language/runtimes* +## Instructions for adding new language/runtimes - Create new `Dockerfile.template`, `build_interop.sh.template` for the language/runtime under `template/tools/dockerfile/`. - Run `tools/buildgen/generate_projects.sh` to create corresponding files under `tools/dockerfile/`. - Add language/runtimes to `client_matrix.py` following existing language/runtimes examples. -- Run `tools/interop_matrix/create_matrix_images.py` which will build and upload images to GCR. Unless you are also building images for a gRPC release, make sure not to set `--release` (the default release 'master' is used for testing). - -*: Please delete your docker images at https://pantheon.corp.google.com/gcr/images/grpc-testing?project=grpc-testing afterwards. Permissions to access GrpcTesting project is required for this step. +- Run `tools/interop_matrix/create_matrix_images.py` which will build (and upload) images to GCR. ## Instructions for creating new test cases - Create test cases by running `LANG= [RELEASE=] ./create_testcases.sh`. For example, @@ -39,7 +38,7 @@ For more details on each step, refer to sections below. - `--release` specifies a git release tag. Defaults to `--release=all`. Make sure the GCR images with the tag have been created using `create_matrix_images.py` above. - `--language` specifies a language. Defaults to `--language=all`. For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`. -- The output for all the test cases is recorded in a junit style xml file (default to 'report.xml'). +- The output for all the test cases is recorded in a junit style xml file (defaults to 'report.xml'). ## Instructions for running test cases against a GCR image manually - Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0`. From 905cd6a3151269fd2c3ad8577fbce5431d8f791b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Jan 2019 18:10:06 +0100 Subject: [PATCH 6/6] invert the strategy test runtimes are being selected per release --- tools/interop_matrix/client_matrix.py | 53 ++++++++++++++------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index ec499a98b61..655c7c7b6bf 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -35,14 +35,15 @@ def get_release_tags(lang): def get_runtimes_for_lang_release(lang, release): """Get list of valid runtimes for given release of lang.""" - runtimes_to_skip = [] + runtimes = list(LANG_RUNTIME_MATRIX[lang]) release_info = LANG_RELEASE_MATRIX[lang].get(release) - if release_info: - runtimes_to_skip = release_info.skip_runtime - return [ - runtime for runtime in LANG_RUNTIME_MATRIX[lang] - if runtime not in runtimes_to_skip - ] + if release_info and release_info.runtime_subset: + runtimes = list(release_info.runtime_subset) + + # check that all selected runtimes are valid for given language + for runtime in runtimes: + assert runtime in LANG_RUNTIME_MATRIX[lang] + return runtimes def should_build_docker_interop_image_from_release_tag(lang): @@ -70,9 +71,9 @@ LANG_RUNTIME_MATRIX = { class ReleaseInfo: """Info about a single release of a language""" - def __init__(self, patch=[], skip_runtime=[], testcases_file=None): + def __init__(self, patch=[], runtime_subset=[], testcases_file=None): self.patch = patch - self.skip_runtime = skip_runtime + self.runtime_subset = runtime_subset self.testcases_file = None @@ -100,23 +101,23 @@ LANG_RELEASE_MATRIX = { ]), 'go': OrderedDict([ - ('v1.0.5', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.2.1', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.3.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.4.2', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.5.2', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.6.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.7.4', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.8.2', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.9.2', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.10.1', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.11.3', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.12.2', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.13.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])), - ('v1.17.0', ReleaseInfo(skip_runtime=['go1.8'])), + ('v1.0.5', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.2.1', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.3.0', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.4.2', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.5.2', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.6.0', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.7.4', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.8.2', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.9.2', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.10.1', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.11.3', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.12.2', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.13.0', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.14.0', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.15.0', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.16.0', ReleaseInfo(runtime_subset=['go1.8'])), + ('v1.17.0', ReleaseInfo(runtime_subset=['go1.11'])), ]), 'java': OrderedDict([