Merge pull request #17671 from jtattermusch/interop_matrix_noskip_runtime

interop_matrix: introduce runtime_subset and more cleanup.
pull/17690/head
Jan Tattermusch 6 years ago committed by GitHub
commit 12bca9fa66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 17
      tools/interop_matrix/README.md
  2. 57
      tools/interop_matrix/client_matrix.py
  3. 2
      tools/interop_matrix/create_matrix_images.py

@ -11,8 +11,9 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee
- Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do
- `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php`
- Verify that the new docker image was built successfully and uploaded to GCR. For example,
- `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos.
- `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`.
- `gcloud container images list --repository gcr.io/grpc-testing` lists available images.
- `gcloud container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`.
- images can also be viewed in https://pantheon.corp.google.com/gcr/images/grpc-testing?project=grpc-testing
- Verify the just-created docker client image would pass backward compatibility test (it should). For example,
- `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by
- `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 tools/interop_matrix/testcases/java__master`
@ -20,13 +21,11 @@ We have continuous nightly test setup to test gRPC backward compatibility betwee
- (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`.
For more details on each step, refer to sections below.
## Instructions for adding new language/runtimes*
## Instructions for adding new language/runtimes
- Create new `Dockerfile.template`, `build_interop.sh.template` for the language/runtime under `template/tools/dockerfile/`.
- Run `tools/buildgen/generate_projects.sh` to create corresponding files under `tools/dockerfile/`.
- Add language/runtimes to `client_matrix.py` following existing language/runtimes examples.
- Run `tools/interop_matrix/create_matrix_images.py` which will build and upload images to GCR. Unless you are also building images for a gRPC release, make sure not to set `--release` (the default release 'master' is used for testing).
*: Please delete your docker images at https://pantheon.corp.google.com/gcr/images/grpc-testing?project=grpc-testing afterwards. Permissions to access GrpcTesting project is required for this step.
- Run `tools/interop_matrix/create_matrix_images.py` which will build (and upload) images to GCR.
## Instructions for creating new test cases
- Create test cases by running `LANG=<lang> [RELEASE=<release>] ./create_testcases.sh`. For example,
@ -39,13 +38,13 @@ For more details on each step, refer to sections below.
- `--release` specifies a git release tag. Defaults to `--release=all`. Make sure the GCR images with the tag have been created using `create_matrix_images.py` above.
- `--language` specifies a language. Defaults to `--language=all`.
For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`.
- The output for all the test cases is recorded in a junit style xml file (default to 'report.xml').
- The output for all the test cases is recorded in a junit style xml file (defaults to 'report.xml').
## Instructions for running test cases against a GCR image manually
- Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.7:master`.
- Download docker image from GCR. For example: `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0`.
- Run test cases by specifying `docker_image` variable inline with the test case script created above.
For example:
- `docker_image=gcr.io/grpc-testing/grpc_interop_go1.7:master ./testcases/go__master` will run go__master test cases against `go1.7` with gRPC release `master` docker image in GCR.
- `docker_image=gcr.io/grpc-testing/grpc_interop_go1.8:v1.16.0 ./testcases/go__master` will run go__master test cases against `go1.8` with gRPC release `v1.16.0` docker image in GCR.
Note:
- File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`).

@ -35,14 +35,15 @@ def get_release_tags(lang):
def get_runtimes_for_lang_release(lang, release):
"""Get list of valid runtimes for given release of lang."""
runtimes_to_skip = []
release_info = LANG_RELEASE_MATRIX[lang][release]
if release_info:
runtimes_to_skip = release_info.skip_runtime
return [
runtime for runtime in LANG_RUNTIME_MATRIX[lang]
if runtime not in runtimes_to_skip
]
runtimes = list(LANG_RUNTIME_MATRIX[lang])
release_info = LANG_RELEASE_MATRIX[lang].get(release)
if release_info and release_info.runtime_subset:
runtimes = list(release_info.runtime_subset)
# check that all selected runtimes are valid for given language
for runtime in runtimes:
assert runtime in LANG_RUNTIME_MATRIX[lang]
return runtimes
def should_build_docker_interop_image_from_release_tag(lang):
@ -57,7 +58,7 @@ def should_build_docker_interop_image_from_release_tag(lang):
# Dictionary of runtimes per language
LANG_RUNTIME_MATRIX = {
'cxx': ['cxx'], # This is actually debian8.
'go': ['go1.7', 'go1.8', 'go1.11'],
'go': ['go1.8', 'go1.11'],
'java': ['java_oracle8'],
'python': ['python'],
'node': ['node'],
@ -70,9 +71,9 @@ LANG_RUNTIME_MATRIX = {
class ReleaseInfo:
"""Info about a single release of a language"""
def __init__(self, patch=[], skip_runtime=[], testcases_file=None):
def __init__(self, patch=[], runtime_subset=[], testcases_file=None):
self.patch = patch
self.skip_runtime = skip_runtime
self.runtime_subset = runtime_subset
self.testcases_file = None
@ -100,23 +101,23 @@ LANG_RELEASE_MATRIX = {
]),
'go':
OrderedDict([
('v1.0.5', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.2.1', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.3.0', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.4.2', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.5.2', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.6.0', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.7.4', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.8.2', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.9.2', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.10.1', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.11.3', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.12.2', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.13.0', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])),
('v1.17.0', ReleaseInfo(skip_runtime=['go1.7', 'go1.8'])),
('v1.0.5', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.2.1', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.3.0', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.4.2', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.5.2', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.6.0', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.7.4', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.8.2', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.9.2', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.10.1', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.11.3', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.12.2', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.13.0', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.14.0', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.15.0', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.16.0', ReleaseInfo(runtime_subset=['go1.8'])),
('v1.17.0', ReleaseInfo(runtime_subset=['go1.11'])),
]),
'java':
OrderedDict([

@ -260,7 +260,7 @@ atexit.register(cleanup)
def maybe_apply_patches_on_git_tag(stack_base, lang, release):
files_to_patch = []
release_info = client_matrix.LANG_RELEASE_MATRIX[lang][release]
release_info = client_matrix.LANG_RELEASE_MATRIX[lang].get(release)
if release_info:
files_to_patch = release_info.patch
if not files_to_patch:

Loading…
Cancel
Save