Merge pull request #17671 from jtattermusch/interop_matrix_noskip_runtime

interop_matrix: introduce runtime_subset and more cleanup.

tools/interop_matrix/README.md

@@ -11,8 +11,9 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee
 - Build new client docker image(s).  For example, for C and wrapper languages release `v1.9.9`, do
   - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php`
 - Verify that the new docker image was built successfully and uploaded to GCR.  For example,
-  - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos.
+  - `gcloud container images list --repository gcr.io/grpc-testing` lists available images.
-  - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`.
+  - `gcloud container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`.
+  - images can also be viewed in https://pantheon.corp.google.com/gcr/images/grpc-testing?project=grpc-testing
 - Verify the just-created docker client image would pass backward compatibility test (it should).  For example,
   - `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by
   - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 tools/interop_matrix/testcases/java__master`
@@ -20,13 +21,11 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee
 - (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`.
 For more details on each step, refer to sections below.
 
-## Instructions for adding new language/runtimes*
+## Instructions for adding new language/runtimes
 - Create new `Dockerfile.template`, `build_interop.sh.template` for the language/runtime under `template/tools/dockerfile/`.
 - Run `tools/buildgen/generate_projects.sh` to create corresponding files under `tools/dockerfile/`.
 - Add language/runtimes to `client_matrix.py` following existing language/runtimes examples.
-- Run `tools/interop_matrix/create_matrix_images.py` which will build and upload images to GCR.  Unless you are also building images for a gRPC release, make sure not to set `--release` (the default release 'master' is used for testing).
+- Run `tools/interop_matrix/create_matrix_images.py` which will build (and upload) images to GCR.
-
-*: Please delete your docker images at https://pantheon.corp.google.com/gcr/images/grpc-testing?project=grpc-testing afterwards.  Permissions to access GrpcTesting project is required for this step.
 
 ## Instructions for creating new test cases
 - Create test cases by running `LANG=<lang> [RELEASE=<release>] ./create_testcases.sh`.  For example,
@@ -39,13 +38,13 @@ For more details on each step, refer to sections below.
   - `--release` specifies a git release tag.  Defaults to `--release=all`.  Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
   - `--language` specifies a language.  Defaults to `--language=all`.
   For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`.
-- The output for all the test cases is recorded in a junit style xml file (default to 'report.xml').
+- The output for all the test cases is recorded in a junit style xml file (defaults to 'report.xml').
 
 ## Instructions for running test cases against a GCR image manually
-- Download docker image from GCR.  For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.7:master`.
+- Download docker image from GCR.  For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0`.
 - Run test cases by specifying `docker_image` variable inline with the test case script created above.
 For example:
-  - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.7:master ./testcases/go__master` will run go__master test cases against `go1.7` with gRPC release `master` docker image in GCR.
+  - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0 ./testcases/go__master` will run go__master test cases against `go1.8` with gRPC release `v1.16.0` docker image in GCR.
 
 Note:
 - File path starting with `tools/` or `template/` are relative to the grpc repo root dir.  File path starting with `./` are relative to current directory (`tools/interop_matrix`).

tools/interop_matrix/client_matrix.py

@@ -35,14 +35,15 @@ def get_release_tags(lang):
 
 def get_runtimes_for_lang_release(lang, release):
     """Get list of valid runtimes for given release of lang."""
-    runtimes_to_skip = []
+    runtimes = list(LANG_RUNTIME_MATRIX[lang])
-    release_info = LANG_RELEASE_MATRIX[lang][release]
+    release_info = LANG_RELEASE_MATRIX[lang].get(release)
-    if release_info:
+    if release_info and release_info.runtime_subset:
-        runtimes_to_skip = release_info.skip_runtime
+        runtimes = list(release_info.runtime_subset)
-    return [
+
-        runtime for runtime in LANG_RUNTIME_MATRIX[lang]
+    # check that all selected runtimes are valid for given language
-        if runtime not in runtimes_to_skip
+    for runtime in runtimes:
-    ]
+        assert runtime in LANG_RUNTIME_MATRIX[lang]
+    return runtimes
 
 
 def should_build_docker_interop_image_from_release_tag(lang):
@@ -57,7 +58,7 @@ def should_build_docker_interop_image_from_release_tag(lang):
 # Dictionary of runtimes per language
 LANG_RUNTIME_MATRIX = {
     'cxx': ['cxx'],  # This is actually debian8.
-    'go': ['go1.7', 'go1.8', 'go1.11'],
+    'go': ['go1.8', 'go1.11'],
     'java': ['java_oracle8'],
     'python': ['python'],
     'node': ['node'],
@@ -70,9 +71,9 @@ LANG_RUNTIME_MATRIX = {
 class ReleaseInfo:
     """Info about a single release of a language"""
 
-    def __init__(self, patch=[], skip_runtime=[], testcases_file=None):
+    def __init__(self, patch=[], runtime_subset=[], testcases_file=None):
         self.patch = patch
-        self.skip_runtime = skip_runtime
+        self.runtime_subset = runtime_subset
         self.testcases_file = None
 
 
@@ -100,23 +101,23 @@ LANG_RELEASE_MATRIX = {
     ]),
     'go':
     OrderedDict([
-        ('v1.0.5', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.0.5', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.2.1', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.2.1', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.3.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.3.0', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.4.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.4.2', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.5.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.5.2', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.6.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.6.0', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.7.4', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.7.4', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.8.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.8.2', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.9.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.9.2', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.10.1', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.10.1', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.11.3', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.11.3', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.12.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.12.2', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.13.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.13.0', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.14.0', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.15.0', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.16.0', ReleaseInfo(runtime_subset=['go1.8'])),
-        ('v1.17.0', ReleaseInfo(skip_runtime=['go1.7', 'go1.8'])),
+        ('v1.17.0', ReleaseInfo(runtime_subset=['go1.11'])),
     ]),
     'java':
     OrderedDict([

tools/interop_matrix/create_matrix_images.py

@@ -260,7 +260,7 @@ atexit.register(cleanup)
 def maybe_apply_patches_on_git_tag(stack_base, lang, release):
     files_to_patch = []
 
-    release_info = client_matrix.LANG_RELEASE_MATRIX[lang][release]
+    release_info = client_matrix.LANG_RELEASE_MATRIX[lang].get(release)
     if release_info:
         files_to_patch = release_info.patch
     if not files_to_patch: