|
|
|
#!/usr/bin/env python3
|
|
|
|
# Copyright 2015 gRPC authors.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
"""Run interop (cross-language) tests in parallel."""
|
|
|
|
|
|
|
|
from __future__ import print_function
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import atexit
|
|
|
|
import itertools
|
|
|
|
import json
|
|
|
|
import multiprocessing
|
|
|
|
import os
|
|
|
|
import re
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import time
|
|
|
|
import traceback
|
|
|
|
import uuid
|
|
|
|
|
|
|
|
import six
|
|
|
|
|
|
|
|
import python_utils.dockerjob as dockerjob
|
|
|
|
import python_utils.jobset as jobset
|
|
|
|
import python_utils.report_utils as report_utils
|
|
|
|
|
|
|
|
# It's ok to not import because this is only necessary to upload results to BQ.
|
|
|
|
try:
|
|
|
|
from python_utils.upload_test_results import upload_interop_results_to_bq
|
|
|
|
except ImportError as e:
|
|
|
|
print(e)
|
|
|
|
|
|
|
|
# Docker doesn't clean up after itself, so we do it on exit.
|
|
|
|
atexit.register(lambda: subprocess.call(["stty", "echo"]))
|
|
|
|
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../.."))
|
|
|
|
os.chdir(ROOT)
|
|
|
|
|
|
|
|
_DEFAULT_SERVER_PORT = 8080
|
|
|
|
|
|
|
|
_SKIP_CLIENT_COMPRESSION = [
|
|
|
|
"client_compressed_unary",
|
|
|
|
"client_compressed_streaming",
|
|
|
|
]
|
|
|
|
|
|
|
|
_SKIP_SERVER_COMPRESSION = [
|
|
|
|
"server_compressed_unary",
|
|
|
|
"server_compressed_streaming",
|
|
|
|
]
|
|
|
|
|
|
|
|
_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION
|
|
|
|
|
|
|
|
_SKIP_ADVANCED = [
|
|
|
|
"status_code_and_message",
|
|
|
|
"custom_metadata",
|
|
|
|
"unimplemented_method",
|
|
|
|
"unimplemented_service",
|
|
|
|
]
|
|
|
|
|
|
|
|
_SKIP_SPECIAL_STATUS_MESSAGE = ["special_status_message"]
|
|
|
|
|
|
|
|
_ORCA_TEST_CASES = ["orca_per_rpc", "orca_oob"]
|
|
|
|
|
|
|
|
_GOOGLE_DEFAULT_CREDS_TEST_CASE = "google_default_credentials"
|
|
|
|
|
|
|
|
_SKIP_GOOGLE_DEFAULT_CREDS = [
|
|
|
|
_GOOGLE_DEFAULT_CREDS_TEST_CASE,
|
|
|
|
]
|
|
|
|
|
|
|
|
_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = "compute_engine_channel_credentials"
|
|
|
|
|
|
|
|
_SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [
|
|
|
|
_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE,
|
|
|
|
]
|
|
|
|
|
|
|
|
_TEST_TIMEOUT = 3 * 60
|
|
|
|
|
|
|
|
# disable this test on core-based languages,
|
|
|
|
# see https://github.com/grpc/grpc/issues/9779
|
|
|
|
_SKIP_DATA_FRAME_PADDING = ["data_frame_padding"]
|
|
|
|
|
|
|
|
# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI
|
|
|
|
_DOCKER_BUILD_XML_REPORT = "interop_docker_build/sponge_log.xml"
|
|
|
|
_TESTS_XML_REPORT = "interop_test/sponge_log.xml"
|
|
|
|
|
|
|
|
|
|
|
|
class CXXLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = None
|
|
|
|
self.server_cwd = None
|
|
|
|
self.http2_cwd = None
|
|
|
|
self.safename = "cxx"
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["cmake/build/interop_client"] + args
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args):
|
|
|
|
return ["cmake/build/http2_client"] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["cmake/build/interop_server"] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return []
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "c++"
|
|
|
|
|
|
|
|
|
|
|
|
class AspNetCoreLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "../grpc-dotnet/output/InteropTestsClient"
|
|
|
|
self.server_cwd = "../grpc-dotnet/output/InteropTestsWebsite"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["dotnet", "exec", "InteropTestsClient.dll"] + args
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["dotnet", "exec", "InteropTestsWebsite.dll"] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "aspnetcore"
|
|
|
|
|
|
|
|
|
|
|
|
class DartLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "../grpc-dart/interop"
|
|
|
|
self.server_cwd = "../grpc-dart/interop"
|
|
|
|
self.http2_cwd = "../grpc-dart/interop"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["dart", "bin/client.dart"] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["dart", "bin/server.dart"] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_COMPRESSION
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return (
|
|
|
|
_SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE + _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "dart"
|
|
|
|
|
|
|
|
|
|
|
|
class JavaLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "../grpc-java"
|
|
|
|
self.server_cwd = "../grpc-java"
|
|
|
|
self.http2_cwd = "../grpc-java"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["./run-test-client.sh"] + args
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args):
|
|
|
|
return [
|
|
|
|
"./interop-testing/build/install/grpc-interop-testing/bin/http2-client"
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["./run-test-server.sh"] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return []
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
# Does not support CompressedRequest feature.
|
|
|
|
# Only supports CompressedResponse feature for unary.
|
|
|
|
return _SKIP_CLIENT_COMPRESSION + ["server_compressed_streaming"]
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "java"
|
|
|
|
|
|
|
|
|
|
|
|
class JavaOkHttpClient:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "../grpc-java"
|
|
|
|
self.safename = "java"
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["./run-test-client.sh", "--use_okhttp=true"] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return _SKIP_DATA_FRAME_PADDING
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "javaokhttp"
|
|
|
|
|
|
|
|
|
|
|
|
class GoLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
# TODO: this relies on running inside docker
|
|
|
|
self.client_cwd = "/go/src/google.golang.org/grpc/interop/client"
|
|
|
|
self.server_cwd = "/go/src/google.golang.org/grpc/interop/server"
|
|
|
|
self.http2_cwd = "/go/src/google.golang.org/grpc/interop/http2"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["go", "run", "client.go"] + args
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args):
|
|
|
|
return ["go", "run", "negative_http2_client.go"] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["go", "run", "server.go"] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {"GO111MODULE": "on"}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return _SKIP_COMPRESSION
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _SKIP_COMPRESSION
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "go"
|
|
|
|
|
|
|
|
|
|
|
|
class Http2Server:
|
|
|
|
"""Represents the HTTP/2 Interop Test server
|
|
|
|
|
|
|
|
This pretends to be a language in order to be built and run, but really it
|
|
|
|
isn't.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self.server_cwd = None
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["python test/http2_test/http2_test_server.py"]
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_TEST_CASES
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "http2"
|
|
|
|
|
|
|
|
|
|
|
|
class Http2Client:
|
|
|
|
"""Represents the HTTP/2 Interop Test
|
|
|
|
|
|
|
|
This pretends to be a language in order to be built and run, but really it
|
|
|
|
isn't.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = None
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["tools/http2_interop/http2_interop.test", "-test.v"] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_TEST_CASES
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "http2"
|
|
|
|
|
|
|
|
|
|
|
|
class NodeLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "../../../../home/appuser/grpc-node"
|
|
|
|
self.server_cwd = "../../../../home/appuser/grpc-node"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return [
|
|
|
|
"packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh",
|
|
|
|
"node",
|
|
|
|
"--require",
|
|
|
|
"./test/fixtures/native_native",
|
|
|
|
"test/interop/interop_client.js",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return [
|
|
|
|
"packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh",
|
|
|
|
"node",
|
|
|
|
"--require",
|
|
|
|
"./test/fixtures/native_native",
|
|
|
|
"test/interop/interop_server.js",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _SKIP_COMPRESSION + _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "node"
|
|
|
|
|
|
|
|
|
|
|
|
class NodePureJSLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "../../../../home/appuser/grpc-node"
|
|
|
|
self.server_cwd = "../../../../home/appuser/grpc-node"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return [
|
|
|
|
"packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh",
|
|
|
|
"node",
|
|
|
|
"--require",
|
|
|
|
"./test/fixtures/js_js",
|
|
|
|
"test/interop/interop_client.js",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "nodepurejs"
|
|
|
|
|
|
|
|
|
|
|
|
class PHP7Language:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = None
|
|
|
|
self.server_cwd = None
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return ["src/php/bin/interop_client.sh"] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return ["src/php/bin/interop_server.sh"] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_SERVER_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _SKIP_COMPRESSION + _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "php7"
|
|
|
|
|
|
|
|
|
|
|
|
class ObjcLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = "src/objective-c/tests"
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
# from args, extract the server port and craft xcodebuild command out of it
|
|
|
|
for arg in args:
|
|
|
|
port = re.search("--server_port=(\d+)", arg)
|
|
|
|
if port:
|
|
|
|
portnum = port.group(1)
|
|
|
|
cmdline = (
|
|
|
|
"pod install && xcodebuild -workspace Tests.xcworkspace"
|
|
|
|
' -scheme InteropTestsLocalSSL -destination name="iPhone 6"'
|
|
|
|
" HOST_PORT_LOCALSSL=localhost:%s test" % portnum
|
|
|
|
)
|
|
|
|
return [cmdline]
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
# ObjC test runs all cases with the same command. It ignores the testcase
|
|
|
|
# cmdline argument. Here we return all but one test cases as unimplemented,
|
|
|
|
# and depend upon ObjC test's behavior that it runs all cases even when
|
|
|
|
# we tell it to run just one.
|
|
|
|
return (
|
|
|
|
_TEST_CASES[1:]
|
|
|
|
+ _SKIP_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _SKIP_COMPRESSION + _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "objc"
|
|
|
|
|
|
|
|
|
|
|
|
class RubyLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = None
|
|
|
|
self.server_cwd = None
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return [
|
|
|
|
"tools/run_tests/interop/with_rvm.sh",
|
|
|
|
"ruby",
|
|
|
|
"src/ruby/pb/test/client.rb",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return [
|
|
|
|
"tools/run_tests/interop/with_rvm.sh",
|
|
|
|
"ruby",
|
|
|
|
"src/ruby/pb/test/server.rb",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_SERVER_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _SKIP_COMPRESSION + _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "ruby"
|
|
|
|
|
|
|
|
|
|
|
|
_PYTHON_BINARY = "py39/bin/python"
|
|
|
|
|
|
|
|
|
|
|
|
class PythonLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = None
|
|
|
|
self.server_cwd = None
|
|
|
|
self.http2_cwd = None
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return [
|
|
|
|
_PYTHON_BINARY,
|
|
|
|
"src/python/grpcio_tests/setup.py",
|
|
|
|
"run_interop",
|
|
|
|
"--client",
|
|
|
|
'--args="{}"'.format(" ".join(args)),
|
|
|
|
]
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args):
|
|
|
|
return [
|
|
|
|
_PYTHON_BINARY,
|
|
|
|
"src/python/grpcio_tests/tests/http2/negative_http2_client.py",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return [
|
|
|
|
_PYTHON_BINARY,
|
|
|
|
"src/python/grpcio_tests/setup.py",
|
|
|
|
"run_interop",
|
|
|
|
"--server",
|
|
|
|
'--args="{}"'.format(" ".join(args)),
|
|
|
|
]
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {
|
|
|
|
"LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT),
|
|
|
|
"PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT),
|
|
|
|
}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
return (
|
|
|
|
_SKIP_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _SKIP_GOOGLE_DEFAULT_CREDS
|
|
|
|
+ _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
return _SKIP_COMPRESSION + _ORCA_TEST_CASES
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "python"
|
|
|
|
|
|
|
|
|
|
|
|
class PythonAsyncIOLanguage:
|
|
|
|
def __init__(self):
|
|
|
|
self.client_cwd = None
|
|
|
|
self.server_cwd = None
|
|
|
|
self.http2_cwd = None
|
|
|
|
self.safename = str(self)
|
|
|
|
|
|
|
|
def client_cmd(self, args):
|
|
|
|
return [
|
|
|
|
_PYTHON_BINARY,
|
|
|
|
"src/python/grpcio_tests/setup.py",
|
|
|
|
"run_interop",
|
|
|
|
"--use-asyncio",
|
|
|
|
"--client",
|
|
|
|
'--args="{}"'.format(" ".join(args)),
|
|
|
|
]
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args):
|
|
|
|
return [
|
|
|
|
_PYTHON_BINARY,
|
|
|
|
"src/python/grpcio_tests/tests/http2/negative_http2_client.py",
|
|
|
|
] + args
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self):
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def server_cmd(self, args):
|
|
|
|
return [
|
|
|
|
_PYTHON_BINARY,
|
|
|
|
"src/python/grpcio_tests/setup.py",
|
|
|
|
"run_interop",
|
|
|
|
"--use-asyncio",
|
|
|
|
"--server",
|
|
|
|
'--args="{}"'.format(" ".join(args)),
|
|
|
|
]
|
|
|
|
|
|
|
|
def global_env(self):
|
|
|
|
return {
|
|
|
|
"LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT),
|
|
|
|
"PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT),
|
|
|
|
}
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self):
|
|
|
|
# TODO(https://github.com/grpc/grpc/issues/21707)
|
|
|
|
return (
|
|
|
|
_SKIP_COMPRESSION
|
|
|
|
+ _SKIP_DATA_FRAME_PADDING
|
|
|
|
+ _AUTH_TEST_CASES
|
|
|
|
+ ["timeout_on_sleeping_server"]
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def unimplemented_test_cases_server(self):
|
|
|
|
# TODO(https://github.com/grpc/grpc/issues/21749)
|
|
|
|
return (
|
|
|
|
_TEST_CASES
|
|
|
|
+ _AUTH_TEST_CASES
|
|
|
|
+ _HTTP2_TEST_CASES
|
|
|
|
+ _HTTP2_SERVER_TEST_CASES
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "pythonasyncio"
|
|
|
|
|
|
|
|
|
|
|
|
_LANGUAGES = {
|
|
|
|
"c++": CXXLanguage(),
|
|
|
|
"aspnetcore": AspNetCoreLanguage(),
|
|
|
|
"dart": DartLanguage(),
|
|
|
|
"go": GoLanguage(),
|
|
|
|
"java": JavaLanguage(),
|
|
|
|
"javaokhttp": JavaOkHttpClient(),
|
|
|
|
"node": NodeLanguage(),
|
|
|
|
"nodepurejs": NodePureJSLanguage(),
|
|
|
|
"php7": PHP7Language(),
|
|
|
|
"objc": ObjcLanguage(),
|
|
|
|
"ruby": RubyLanguage(),
|
|
|
|
"python": PythonLanguage(),
|
|
|
|
"pythonasyncio": PythonAsyncIOLanguage(),
|
|
|
|
}
|
|
|
|
|
|
|
|
# languages supported as cloud_to_cloud servers
|
|
|
|
_SERVERS = [
|
|
|
|
"c++",
|
|
|
|
"node",
|
|
|
|
"aspnetcore",
|
|
|
|
"java",
|
|
|
|
"go",
|
|
|
|
"ruby",
|
|
|
|
"python",
|
|
|
|
"dart",
|
|
|
|
"pythonasyncio",
|
|
|
|
"php7",
|
|
|
|
]
|
|
|
|
|
|
|
|
_TEST_CASES = [
|
|
|
|
"large_unary",
|
|
|
|
"empty_unary",
|
|
|
|
"ping_pong",
|
|
|
|
"empty_stream",
|
|
|
|
"client_streaming",
|
|
|
|
"server_streaming",
|
|
|
|
"cancel_after_begin",
|
|
|
|
"cancel_after_first_response",
|
|
|
|
"timeout_on_sleeping_server",
|
|
|
|
"custom_metadata",
|
|
|
|
"status_code_and_message",
|
|
|
|
"unimplemented_method",
|
|
|
|
"client_compressed_unary",
|
|
|
|
"server_compressed_unary",
|
|
|
|
"client_compressed_streaming",
|
|
|
|
"server_compressed_streaming",
|
|
|
|
"unimplemented_service",
|
|
|
|
"special_status_message",
|
|
|
|
"orca_per_rpc",
|
|
|
|
"orca_oob",
|
|
|
|
]
|
|
|
|
|
|
|
|
_AUTH_TEST_CASES = [
|
|
|
|
"compute_engine_creds",
|
|
|
|
"jwt_token_creds",
|
|
|
|
"oauth2_auth_token",
|
|
|
|
"per_rpc_creds",
|
|
|
|
_GOOGLE_DEFAULT_CREDS_TEST_CASE,
|
|
|
|
_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE,
|
|
|
|
]
|
|
|
|
|
|
|
|
_HTTP2_TEST_CASES = ["tls", "framing"]
|
|
|
|
|
|
|
|
_HTTP2_SERVER_TEST_CASES = [
|
|
|
|
"rst_after_header",
|
|
|
|
"rst_after_data",
|
|
|
|
"rst_during_data",
|
|
|
|
"goaway",
|
|
|
|
"ping",
|
|
|
|
"max_streams",
|
|
|
|
"data_frame_padding",
|
|
|
|
"no_df_padding_sanity_test",
|
|
|
|
]
|
|
|
|
|
|
|
|
_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = {
|
|
|
|
"data_frame_padding": "large_unary",
|
|
|
|
"no_df_padding_sanity_test": "large_unary",
|
|
|
|
}
|
|
|
|
|
|
|
|
_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = list(
|
|
|
|
_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys()
|
|
|
|
)
|
|
|
|
|
|
|
|
_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [
|
|
|
|
"java",
|
|
|
|
"go",
|
|
|
|
"python",
|
|
|
|
"c++",
|
|
|
|
]
|
|
|
|
|
|
|
|
_LANGUAGES_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"]
|
|
|
|
|
|
|
|
_SERVERS_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"]
|
|
|
|
|
|
|
|
_TRANSPORT_SECURITY_OPTIONS = ["tls", "alts", "insecure"]
|
|
|
|
|
|
|
|
_CUSTOM_CREDENTIALS_TYPE_OPTIONS = [
|
|
|
|
"tls",
|
|
|
|
"google_default_credentials",
|
|
|
|
"compute_engine_channel_creds",
|
|
|
|
]
|
|
|
|
|
|
|
|
DOCKER_WORKDIR_ROOT = "/var/local/git/grpc"
|
|
|
|
|
|
|
|
|
|
|
|
def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
|
|
|
|
"""Wraps given cmdline array to create 'docker run' cmdline from it."""
|
|
|
|
|
|
|
|
# don't use '-t' even when TTY is available, since that would break
|
|
|
|
# the testcases generated by tools/interop_matrix/create_testcases.sh
|
|
|
|
docker_cmdline = ["docker", "run", "-i", "--rm=true"]
|
|
|
|
|
|
|
|
# turn environ into -e docker args
|
|
|
|
if environ:
|
|
|
|
for k, v in list(environ.items()):
|
|
|
|
docker_cmdline += ["-e", "%s=%s" % (k, v)]
|
|
|
|
|
|
|
|
# set working directory
|
|
|
|
workdir = DOCKER_WORKDIR_ROOT
|
|
|
|
if cwd:
|
|
|
|
workdir = os.path.join(workdir, cwd)
|
|
|
|
docker_cmdline += ["-w", workdir]
|
|
|
|
|
|
|
|
docker_cmdline += docker_args + [image] + cmdline
|
|
|
|
return docker_cmdline
|
|
|
|
|
|
|
|
|
|
|
|
def manual_cmdline(docker_cmdline, docker_image):
|
|
|
|
"""Returns docker cmdline adjusted for manual invocation."""
|
|
|
|
print_cmdline = []
|
|
|
|
for item in docker_cmdline:
|
|
|
|
if item.startswith("--name="):
|
|
|
|
continue
|
|
|
|
if item == docker_image:
|
|
|
|
item = "$docker_image"
|
|
|
|
item = item.replace('"', '\\"')
|
|
|
|
# add quotes when necessary
|
|
|
|
if any(character.isspace() for character in item):
|
|
|
|
item = '"%s"' % item
|
|
|
|
print_cmdline.append(item)
|
|
|
|
return " ".join(print_cmdline)
|
|
|
|
|
|
|
|
|
|
|
|
def write_cmdlog_maybe(cmdlog, filename):
|
|
|
|
"""Returns docker cmdline adjusted for manual invocation."""
|
|
|
|
if cmdlog:
|
|
|
|
with open(filename, "w") as logfile:
|
|
|
|
logfile.write("#!/bin/bash\n")
|
|
|
|
logfile.write("# DO NOT MODIFY\n")
|
|
|
|
logfile.write(
|
|
|
|
"# This file is generated by"
|
|
|
|
" run_interop_tests.py/create_testcases.sh\n"
|
|
|
|
)
|
|
|
|
logfile.writelines("%s\n" % line for line in cmdlog)
|
|
|
|
print("Command log written to file %s" % filename)
|
|
|
|
|
|
|
|
|
|
|
|
def bash_cmdline(cmdline):
|
|
|
|
"""Creates bash -c cmdline from args list."""
|
|
|
|
# Use login shell:
|
|
|
|
# * makes error messages clearer if executables are missing
|
|
|
|
return ["bash", "-c", " ".join(cmdline)]
|
|
|
|
|
|
|
|
|
|
|
|
def compute_engine_creds_required(language, test_case):
|
|
|
|
"""Returns True if given test requires access to compute engine creds."""
|
|
|
|
language = str(language)
|
|
|
|
if test_case == "compute_engine_creds":
|
|
|
|
return True
|
|
|
|
if test_case == "oauth2_auth_token" and language == "c++":
|
|
|
|
# C++ oauth2 test uses GCE creds because C++ only supports JWT
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
def auth_options(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
google_default_creds_use_key_file,
|
|
|
|
service_account_key_file,
|
|
|
|
default_service_account,
|
|
|
|
):
|
|
|
|
"""Returns (cmdline, env) tuple with cloud_to_prod_auth test options."""
|
|
|
|
|
|
|
|
language = str(language)
|
|
|
|
cmdargs = []
|
|
|
|
env = {}
|
|
|
|
|
|
|
|
oauth_scope_arg = "--oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
|
|
|
|
key_file_arg = "--service_account_key_file=%s" % service_account_key_file
|
|
|
|
default_account_arg = (
|
|
|
|
"--default_service_account=%s" % default_service_account
|
|
|
|
)
|
|
|
|
|
|
|
|
if test_case in ["jwt_token_creds", "per_rpc_creds", "oauth2_auth_token"]:
|
|
|
|
if language in [
|
|
|
|
"aspnetcore",
|
|
|
|
"node",
|
|
|
|
"php7",
|
|
|
|
"python",
|
|
|
|
"ruby",
|
|
|
|
"nodepurejs",
|
|
|
|
]:
|
|
|
|
env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file
|
|
|
|
else:
|
|
|
|
cmdargs += [key_file_arg]
|
|
|
|
|
|
|
|
if test_case in ["per_rpc_creds", "oauth2_auth_token"]:
|
|
|
|
cmdargs += [oauth_scope_arg]
|
|
|
|
|
|
|
|
if test_case == "oauth2_auth_token" and language == "c++":
|
|
|
|
# C++ oauth2 test uses GCE creds and thus needs to know the default account
|
|
|
|
cmdargs += [default_account_arg]
|
|
|
|
|
|
|
|
if test_case == "compute_engine_creds":
|
|
|
|
cmdargs += [oauth_scope_arg, default_account_arg]
|
|
|
|
|
|
|
|
if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE:
|
|
|
|
if google_default_creds_use_key_file:
|
|
|
|
env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file
|
|
|
|
cmdargs += [default_account_arg]
|
|
|
|
|
|
|
|
if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE:
|
|
|
|
cmdargs += [default_account_arg]
|
|
|
|
|
|
|
|
return (cmdargs, env)
|
|
|
|
|
|
|
|
|
|
|
|
def _job_kill_handler(job):
|
|
|
|
if job._spec.container_name:
|
|
|
|
dockerjob.docker_kill(job._spec.container_name)
|
|
|
|
# When the job times out and we decide to kill it,
|
|
|
|
# we need to wait a before restarting the job
|
|
|
|
# to prevent "container name already in use" error.
|
|
|
|
# TODO(jtattermusch): figure out a cleaner way to this.
|
|
|
|
time.sleep(2)
|
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_prod_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
server_host_nickname,
|
|
|
|
server_host,
|
|
|
|
google_default_creds_use_key_file,
|
|
|
|
docker_image=None,
|
|
|
|
auth=False,
|
|
|
|
manual_cmd_log=None,
|
|
|
|
service_account_key_file=None,
|
|
|
|
default_service_account=None,
|
|
|
|
transport_security="tls",
|
|
|
|
):
|
|
|
|
"""Creates jobspec for cloud-to-prod interop test"""
|
|
|
|
container_name = None
|
|
|
|
cmdargs = [
|
|
|
|
"--server_host=%s" % server_host,
|
|
|
|
"--server_port=443",
|
|
|
|
"--test_case=%s" % test_case,
|
|
|
|
]
|
|
|
|
if transport_security == "tls":
|
|
|
|
transport_security_options = ["--use_tls=true"]
|
|
|
|
elif transport_security == "google_default_credentials" and str(
|
|
|
|
language
|
|
|
|
) in ["c++", "go", "java", "javaokhttp"]:
|
|
|
|
transport_security_options = [
|
|
|
|
"--custom_credentials_type=google_default_credentials"
|
|
|
|
]
|
|
|
|
elif transport_security == "compute_engine_channel_creds" and str(
|
|
|
|
language
|
|
|
|
) in ["go", "java", "javaokhttp"]:
|
|
|
|
transport_security_options = [
|
|
|
|
"--custom_credentials_type=compute_engine_channel_creds"
|
|
|
|
]
|
|
|
|
else:
|
|
|
|
print(
|
|
|
|
"Invalid transport security option %s in cloud_to_prod_jobspec."
|
|
|
|
" Lang: %s" % (str(language), transport_security)
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
cmdargs = cmdargs + transport_security_options
|
|
|
|
environ = dict(language.cloud_to_prod_env(), **language.global_env())
|
|
|
|
if auth:
|
|
|
|
auth_cmdargs, auth_env = auth_options(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
google_default_creds_use_key_file,
|
|
|
|
service_account_key_file,
|
|
|
|
default_service_account,
|
|
|
|
)
|
|
|
|
cmdargs += auth_cmdargs
|
|
|
|
environ.update(auth_env)
|
|
|
|
cmdline = bash_cmdline(language.client_cmd(cmdargs))
|
|
|
|
cwd = language.client_cwd
|
|
|
|
|
|
|
|
if docker_image:
|
|
|
|
container_name = dockerjob.random_name(
|
|
|
|
"interop_client_%s" % language.safename
|
|
|
|
)
|
|
|
|
cmdline = docker_run_cmdline(
|
|
|
|
cmdline,
|
|
|
|
image=docker_image,
|
|
|
|
cwd=cwd,
|
|
|
|
environ=environ,
|
|
|
|
docker_args=["--net=host", "--name=%s" % container_name],
|
|
|
|
)
|
|
|
|
if manual_cmd_log is not None:
|
|
|
|
if manual_cmd_log == []:
|
|
|
|
manual_cmd_log.append(
|
|
|
|
'echo "Testing ${docker_image:=%s}"' % docker_image
|
|
|
|
)
|
|
|
|
manual_cmd_log.append(manual_cmdline(cmdline, docker_image))
|
|
|
|
cwd = None
|
|
|
|
environ = None
|
|
|
|
|
|
|
|
suite_name = "cloud_to_prod_auth" if auth else "cloud_to_prod"
|
|
|
|
test_job = jobset.JobSpec(
|
|
|
|
cmdline=cmdline,
|
|
|
|
cwd=cwd,
|
|
|
|
environ=environ,
|
|
|
|
shortname="%s:%s:%s:%s:%s"
|
|
|
|
% (
|
|
|
|
suite_name,
|
|
|
|
language,
|
|
|
|
server_host_nickname,
|
|
|
|
test_case,
|
|
|
|
transport_security,
|
|
|
|
),
|
|
|
|
timeout_seconds=_TEST_TIMEOUT,
|
|
|
|
flake_retries=4 if args.allow_flakes else 0,
|
|
|
|
timeout_retries=2 if args.allow_flakes else 0,
|
|
|
|
kill_handler=_job_kill_handler,
|
|
|
|
)
|
|
|
|
if docker_image:
|
|
|
|
test_job.container_name = container_name
|
|
|
|
return test_job
|
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_cloud_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
server_name,
|
|
|
|
server_host,
|
|
|
|
server_port,
|
|
|
|
docker_image=None,
|
|
|
|
transport_security="tls",
|
|
|
|
manual_cmd_log=None,
|
|
|
|
):
|
|
|
|
"""Creates jobspec for cloud-to-cloud interop test"""
|
|
|
|
interop_only_options = [
|
|
|
|
"--server_host_override=foo.test.google.fr",
|
|
|
|
"--use_test_ca=true",
|
|
|
|
]
|
|
|
|
if transport_security == "tls":
|
|
|
|
interop_only_options += ["--use_tls=true"]
|
|
|
|
elif transport_security == "alts":
|
|
|
|
interop_only_options += ["--use_tls=false", "--use_alts=true"]
|
|
|
|
elif transport_security == "insecure":
|
|
|
|
interop_only_options += ["--use_tls=false"]
|
|
|
|
else:
|
|
|
|
print(
|
|
|
|
"Invalid transport security option %s in cloud_to_cloud_jobspec."
|
|
|
|
% transport_security
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
client_test_case = test_case
|
|
|
|
if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS:
|
|
|
|
client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[
|
|
|
|
test_case
|
|
|
|
]
|
|
|
|
if client_test_case in language.unimplemented_test_cases():
|
|
|
|
print(
|
|
|
|
"asking client %s to run unimplemented test case %s"
|
|
|
|
% (repr(language), client_test_case)
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
if test_case in _ORCA_TEST_CASES:
|
|
|
|
interop_only_options += [
|
|
|
|
'--service_config_json=\'{"loadBalancingConfig":[{"test_backend_metrics_load_balancer":{}}]}\''
|
|
|
|
]
|
|
|
|
|
|
|
|
common_options = [
|
|
|
|
"--test_case=%s" % client_test_case,
|
|
|
|
"--server_host=%s" % server_host,
|
|
|
|
"--server_port=%s" % server_port,
|
|
|
|
]
|
|
|
|
|
|
|
|
if test_case in _HTTP2_SERVER_TEST_CASES:
|
|
|
|
if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS:
|
|
|
|
client_options = interop_only_options + common_options
|
|
|
|
cmdline = bash_cmdline(language.client_cmd(client_options))
|
|
|
|
cwd = language.client_cwd
|
|
|
|
else:
|
|
|
|
cmdline = bash_cmdline(
|
|
|
|
language.client_cmd_http2interop(common_options)
|
|
|
|
)
|
|
|
|
cwd = language.http2_cwd
|
|
|
|
else:
|
|
|
|
cmdline = bash_cmdline(
|
|
|
|
language.client_cmd(common_options + interop_only_options)
|
|
|
|
)
|
|
|
|
cwd = language.client_cwd
|
|
|
|
|
|
|
|
environ = language.global_env()
|
|
|
|
if docker_image and language.safename != "objc":
|
|
|
|
# we can't run client in docker for objc.
|
|
|
|
container_name = dockerjob.random_name(
|
|
|
|
"interop_client_%s" % language.safename
|
|
|
|
)
|
|
|
|
cmdline = docker_run_cmdline(
|
|
|
|
cmdline,
|
|
|
|
image=docker_image,
|
|
|
|
environ=environ,
|
|
|
|
cwd=cwd,
|
|
|
|
docker_args=["--net=host", "--name=%s" % container_name],
|
|
|
|
)
|
|
|
|
if manual_cmd_log is not None:
|
|
|
|
if manual_cmd_log == []:
|
|
|
|
manual_cmd_log.append(
|
|
|
|
'echo "Testing ${docker_image:=%s}"' % docker_image
|
|
|
|
)
|
|
|
|
manual_cmd_log.append(manual_cmdline(cmdline, docker_image))
|
|
|
|
cwd = None
|
|
|
|
|
|
|
|
test_job = jobset.JobSpec(
|
|
|
|
cmdline=cmdline,
|
|
|
|
cwd=cwd,
|
|
|
|
environ=environ,
|
|
|
|
shortname="cloud_to_cloud:%s:%s_server:%s:%s"
|
|
|
|
% (language, server_name, test_case, transport_security),
|
|
|
|
timeout_seconds=_TEST_TIMEOUT,
|
|
|
|
flake_retries=4 if args.allow_flakes else 0,
|
|
|
|
timeout_retries=2 if args.allow_flakes else 0,
|
|
|
|
kill_handler=_job_kill_handler,
|
|
|
|
)
|
|
|
|
if docker_image:
|
|
|
|
test_job.container_name = container_name
|
|
|
|
return test_job
|
|
|
|
|
|
|
|
|
|
|
|
def server_jobspec(
|
|
|
|
language, docker_image, transport_security="tls", manual_cmd_log=None
|
|
|
|
):
|
|
|
|
"""Create jobspec for running a server"""
|
|
|
|
container_name = dockerjob.random_name(
|
|
|
|
"interop_server_%s" % language.safename
|
|
|
|
)
|
|
|
|
server_cmd = ["--port=%s" % _DEFAULT_SERVER_PORT]
|
|
|
|
if transport_security == "tls":
|
|
|
|
server_cmd += ["--use_tls=true"]
|
|
|
|
elif transport_security == "alts":
|
|
|
|
server_cmd += ["--use_tls=false", "--use_alts=true"]
|
|
|
|
elif transport_security == "insecure":
|
|
|
|
server_cmd += ["--use_tls=false"]
|
|
|
|
else:
|
|
|
|
print(
|
|
|
|
"Invalid transport security option %s in server_jobspec."
|
|
|
|
% transport_security
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
cmdline = bash_cmdline(language.server_cmd(server_cmd))
|
|
|
|
environ = language.global_env()
|
|
|
|
docker_args = ["--name=%s" % container_name]
|
|
|
|
if language.safename == "http2":
|
|
|
|
# we are running the http2 interop server. Open next N ports beginning
|
|
|
|
# with the server port. These ports are used for http2 interop test
|
|
|
|
# (one test case per port).
|
|
|
|
docker_args += list(
|
|
|
|
itertools.chain.from_iterable(
|
|
|
|
("-p", str(_DEFAULT_SERVER_PORT + i))
|
|
|
|
for i in range(len(_HTTP2_SERVER_TEST_CASES))
|
|
|
|
)
|
|
|
|
)
|
|
|
|
# Enable docker's healthcheck mechanism.
|
|
|
|
# This runs a Python script inside the container every second. The script
|
|
|
|
# pings the http2 server to verify it is ready. The 'health-retries' flag
|
|
|
|
# specifies the number of consecutive failures before docker will report
|
|
|
|
# the container's status as 'unhealthy'. Prior to the first 'health_retries'
|
|
|
|
# failures or the first success, the status will be 'starting'. 'docker ps'
|
|
|
|
# or 'docker inspect' can be used to see the health of the container on the
|
|
|
|
# command line.
|
|
|
|
docker_args += [
|
|
|
|
"--health-cmd=python test/http2_test/http2_server_health_check.py "
|
|
|
|
"--server_host=%s --server_port=%d"
|
|
|
|
% ("localhost", _DEFAULT_SERVER_PORT),
|
|
|
|
"--health-interval=1s",
|
|
|
|
"--health-retries=5",
|
|
|
|
"--health-timeout=10s",
|
|
|
|
]
|
|
|
|
|
|
|
|
else:
|
|
|
|
docker_args += ["-p", str(_DEFAULT_SERVER_PORT)]
|
|
|
|
|
|
|
|
docker_cmdline = docker_run_cmdline(
|
|
|
|
cmdline,
|
|
|
|
image=docker_image,
|
|
|
|
cwd=language.server_cwd,
|
|
|
|
environ=environ,
|
|
|
|
docker_args=docker_args,
|
|
|
|
)
|
|
|
|
if manual_cmd_log is not None:
|
|
|
|
if manual_cmd_log == []:
|
|
|
|
manual_cmd_log.append(
|
|
|
|
'echo "Testing ${docker_image:=%s}"' % docker_image
|
|
|
|
)
|
|
|
|
manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image))
|
|
|
|
server_job = jobset.JobSpec(
|
|
|
|
cmdline=docker_cmdline,
|
|
|
|
environ=environ,
|
|
|
|
shortname="interop_server_%s" % language,
|
|
|
|
timeout_seconds=30 * 60,
|
|
|
|
)
|
|
|
|
server_job.container_name = container_name
|
|
|
|
return server_job
|
|
|
|
|
|
|
|
|
|
|
|
def build_interop_image_jobspec(language, tag=None):
|
|
|
|
"""Creates jobspec for building interop docker image for a language"""
|
|
|
|
if not tag:
|
|
|
|
tag = "grpc_interop_%s:%s" % (language.safename, uuid.uuid4())
|
|
|
|
env = {
|
|
|
|
"INTEROP_IMAGE": tag,
|
|
|
|
"BASE_NAME": "grpc_interop_%s" % language.safename,
|
|
|
|
}
|
|
|
|
build_job = jobset.JobSpec(
|
|
|
|
cmdline=["tools/run_tests/dockerize/build_interop_image.sh"],
|
|
|
|
environ=env,
|
|
|
|
shortname="build_docker_%s" % (language),
|
|
|
|
timeout_seconds=30 * 60,
|
|
|
|
)
|
|
|
|
build_job.tag = tag
|
|
|
|
return build_job
|
|
|
|
|
|
|
|
|
|
|
|
def aggregate_http2_results(stdout):
|
|
|
|
match = re.search(r'\{"cases[^\]]*\]\}', stdout)
|
|
|
|
if not match:
|
|
|
|
return None
|
|
|
|
|
|
|
|
results = json.loads(match.group(0))
|
|
|
|
skipped = 0
|
|
|
|
passed = 0
|
|
|
|
failed = 0
|
|
|
|
failed_cases = []
|
|
|
|
for case in results["cases"]:
|
|
|
|
if case.get("skipped", False):
|
|
|
|
skipped += 1
|
|
|
|
else:
|
|
|
|
if case.get("passed", False):
|
|
|
|
passed += 1
|
|
|
|
else:
|
|
|
|
failed += 1
|
|
|
|
failed_cases.append(case.get("name", "NONAME"))
|
|
|
|
return {
|
|
|
|
"passed": passed,
|
|
|
|
"failed": failed,
|
|
|
|
"skipped": skipped,
|
|
|
|
"failed_cases": ", ".join(failed_cases),
|
|
|
|
"percent": 1.0 * passed / (passed + failed),
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
# A dictionary of prod servers to test against.
|
|
|
|
# See go/grpc-interop-tests (internal-only) for details.
|
|
|
|
prod_servers = {
|
|
|
|
"default": "grpc-test.sandbox.googleapis.com",
|
|
|
|
"gateway_v4": "grpc-test4.sandbox.googleapis.com",
|
|
|
|
}
|
|
|
|
|
|
|
|
argp = argparse.ArgumentParser(description="Run interop tests.")
|
|
|
|
argp.add_argument(
|
|
|
|
"-l",
|
|
|
|
"--language",
|
|
|
|
choices=["all"] + sorted(_LANGUAGES),
|
|
|
|
nargs="+",
|
|
|
|
default=["all"],
|
|
|
|
help="Clients to run. Objc client can be only run on OSX.",
|
|
|
|
)
|
|
|
|
argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int)
|
|
|
|
argp.add_argument(
|
|
|
|
"--cloud_to_prod",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help="Run cloud_to_prod tests.",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--cloud_to_prod_auth",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help="Run cloud_to_prod_auth tests.",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--google_default_creds_use_key_file",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help=(
|
|
|
|
"Whether or not we should use a key file for the "
|
|
|
|
"google_default_credentials test case, e.g. by "
|
|
|
|
"setting env var GOOGLE_APPLICATION_CREDENTIALS."
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--prod_servers",
|
|
|
|
choices=list(prod_servers.keys()),
|
|
|
|
default=["default"],
|
|
|
|
nargs="+",
|
|
|
|
help=(
|
|
|
|
"The servers to run cloud_to_prod and cloud_to_prod_auth tests against."
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"-s",
|
|
|
|
"--server",
|
|
|
|
choices=["all"] + sorted(_SERVERS),
|
|
|
|
nargs="+",
|
|
|
|
help="Run cloud_to_cloud servers in a separate docker "
|
|
|
|
+ "image. Servers can only be started automatically if "
|
|
|
|
+ "--use_docker option is enabled.",
|
|
|
|
default=[],
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--override_server",
|
|
|
|
action="append",
|
|
|
|
type=lambda kv: kv.split("="),
|
|
|
|
help=(
|
|
|
|
"Use servername=HOST:PORT to explicitly specify a server. E.g."
|
|
|
|
" csharp=localhost:50000"
|
|
|
|
),
|
|
|
|
default=[],
|
|
|
|
)
|
|
|
|
# TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used.
|
|
|
|
argp.add_argument(
|
|
|
|
"--service_account_key_file",
|
|
|
|
type=str,
|
|
|
|
help="The service account key file to use for some auth interop tests.",
|
|
|
|
default="/root/service_account/grpc-testing-ebe7c1ac7381.json",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--default_service_account",
|
|
|
|
type=str,
|
|
|
|
help=(
|
|
|
|
"Default GCE service account email to use for some auth interop tests."
|
|
|
|
),
|
|
|
|
default="830293263384-compute@developer.gserviceaccount.com",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"-t",
|
|
|
|
"--travis",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help=(
|
|
|
|
"When set, indicates that the script is running on CI (= not locally)."
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"-v", "--verbose", default=False, action="store_const", const=True
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--use_docker",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help="Run all the interop tests under docker. That provides "
|
|
|
|
+ "additional isolation and prevents the need to install "
|
|
|
|
+ "language specific prerequisites. Only available on Linux.",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--allow_flakes",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help=(
|
|
|
|
"Allow flaky tests to show as passing (re-runs failed tests up to five"
|
|
|
|
" times)"
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--manual_run",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help="Prepare things for running interop tests manually. "
|
|
|
|
+ "Preserve docker images after building them and skip "
|
|
|
|
"actually running the tests. Only print commands to run by " + "hand.",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--http2_interop",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help="Enable HTTP/2 client edge case testing. (Bad client, good server)",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--http2_server_interop",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help=(
|
|
|
|
"Enable HTTP/2 server edge case testing. (Includes positive and"
|
|
|
|
" negative tests"
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--transport_security",
|
|
|
|
choices=_TRANSPORT_SECURITY_OPTIONS,
|
|
|
|
default="tls",
|
|
|
|
type=str,
|
|
|
|
nargs="?",
|
|
|
|
const=True,
|
|
|
|
help="Which transport security mechanism to use.",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--custom_credentials_type",
|
|
|
|
choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS,
|
|
|
|
default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS,
|
|
|
|
nargs="+",
|
|
|
|
help=(
|
|
|
|
"Credential types to test in the cloud_to_prod setup. Default is to"
|
|
|
|
" test with all creds types possible."
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--skip_compute_engine_creds",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help="Skip auth tests requiring access to compute engine credentials.",
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--internal_ci",
|
|
|
|
default=False,
|
|
|
|
action="store_const",
|
|
|
|
const=True,
|
|
|
|
help=(
|
|
|
|
"(Deprecated, has no effect) Put reports into subdirectories to improve"
|
|
|
|
" presentation of results by Internal CI."
|
|
|
|
),
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
"--bq_result_table",
|
|
|
|
default="",
|
|
|
|
type=str,
|
|
|
|
nargs="?",
|
|
|
|
help="Upload test results to a specified BQ table.",
|
|
|
|
)
|
|
|
|
args = argp.parse_args()
|
|
|
|
|
|
|
|
servers = set(
|
|
|
|
s
|
|
|
|
for s in itertools.chain.from_iterable(
|
|
|
|
_SERVERS if x == "all" else [x] for x in args.server
|
|
|
|
)
|
|
|
|
)
|
|
|
|
# ALTS servers are only available for certain languages.
|
|
|
|
if args.transport_security == "alts":
|
|
|
|
servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES)
|
|
|
|
|
|
|
|
if args.use_docker:
|
|
|
|
if not args.travis:
|
|
|
|
print("Seen --use_docker flag, will run interop tests under docker.")
|
|
|
|
print("")
|
|
|
|
print(
|
|
|
|
"IMPORTANT: The changes you are testing need to be locally"
|
|
|
|
" committed"
|
|
|
|
)
|
|
|
|
print(
|
|
|
|
"because only the committed changes in the current branch will be"
|
|
|
|
)
|
|
|
|
print("copied to the docker environment.")
|
|
|
|
time.sleep(5)
|
|
|
|
|
|
|
|
if args.manual_run and not args.use_docker:
|
|
|
|
print("--manual_run is only supported with --use_docker option enabled.")
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
if not args.use_docker and servers:
|
|
|
|
print(
|
|
|
|
"Running interop servers is only supported with --use_docker option"
|
|
|
|
" enabled."
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
# we want to include everything but objc in 'all'
|
|
|
|
# because objc won't run on non-mac platforms
|
|
|
|
all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(["objc"])
|
|
|
|
languages = set(
|
|
|
|
_LANGUAGES[l]
|
|
|
|
for l in itertools.chain.from_iterable(
|
|
|
|
all_but_objc if x == "all" else [x] for x in args.language
|
|
|
|
)
|
|
|
|
)
|
|
|
|
# ALTS interop clients are only available for certain languages.
|
|
|
|
if args.transport_security == "alts":
|
|
|
|
alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES)
|
|
|
|
languages = languages.intersection(alts_languages)
|
|
|
|
|
|
|
|
languages_http2_clients_for_http2_server_interop = set()
|
|
|
|
if args.http2_server_interop:
|
|
|
|
languages_http2_clients_for_http2_server_interop = set(
|
|
|
|
_LANGUAGES[l]
|
|
|
|
for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES
|
|
|
|
if "all" in args.language or l in args.language
|
|
|
|
)
|
|
|
|
|
|
|
|
http2Interop = Http2Client() if args.http2_interop else None
|
|
|
|
http2InteropServer = Http2Server() if args.http2_server_interop else None
|
|
|
|
|
|
|
|
docker_images = {}
|
|
|
|
if args.use_docker:
|
|
|
|
# languages for which to build docker images
|
|
|
|
languages_to_build = set(
|
|
|
|
_LANGUAGES[k]
|
|
|
|
for k in set([str(l) for l in languages] + [s for s in servers])
|
|
|
|
)
|
|
|
|
languages_to_build = (
|
|
|
|
languages_to_build | languages_http2_clients_for_http2_server_interop
|
|
|
|
)
|
|
|
|
|
|
|
|
if args.http2_interop:
|
|
|
|
languages_to_build.add(http2Interop)
|
|
|
|
|
|
|
|
if args.http2_server_interop:
|
|
|
|
languages_to_build.add(http2InteropServer)
|
|
|
|
|
|
|
|
build_jobs = []
|
|
|
|
for l in languages_to_build:
|
|
|
|
if str(l) == "objc":
|
|
|
|
# we don't need to build a docker image for objc
|
|
|
|
continue
|
|
|
|
job = build_interop_image_jobspec(l)
|
|
|
|
docker_images[str(l)] = job.tag
|
|
|
|
build_jobs.append(job)
|
|
|
|
|
|
|
|
if build_jobs:
|
|
|
|
jobset.message(
|
|
|
|
"START", "Building interop docker images.", do_newline=True
|
|
|
|
)
|
|
|
|
if args.verbose:
|
|
|
|
print("Jobs to run: \n%s\n" % "\n".join(str(j) for j in build_jobs))
|
|
|
|
|
|
|
|
num_failures, build_resultset = jobset.run(
|
|
|
|
build_jobs, newline_on_success=True, maxjobs=args.jobs
|
|
|
|
)
|
|
|
|
|
|
|
|
report_utils.render_junit_xml_report(
|
|
|
|
build_resultset, _DOCKER_BUILD_XML_REPORT
|
|
|
|
)
|
|
|
|
|
|
|
|
if num_failures == 0:
|
|
|
|
jobset.message(
|
|
|
|
"SUCCESS",
|
|
|
|
"All docker images built successfully.",
|
|
|
|
do_newline=True,
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
jobset.message(
|
|
|
|
"FAILED",
|
|
|
|
"Failed to build interop docker images.",
|
|
|
|
do_newline=True,
|
|
|
|
)
|
|
|
|
for image in six.itervalues(docker_images):
|
|
|
|
dockerjob.remove_image(image, skip_nonexistent=True)
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
server_manual_cmd_log = [] if args.manual_run else None
|
|
|
|
client_manual_cmd_log = [] if args.manual_run else None
|
|
|
|
|
|
|
|
# Start interop servers.
|
|
|
|
server_jobs = {}
|
|
|
|
server_addresses = {}
|
|
|
|
try:
|
|
|
|
for s in servers:
|
|
|
|
lang = str(s)
|
|
|
|
spec = server_jobspec(
|
|
|
|
_LANGUAGES[lang],
|
|
|
|
docker_images.get(lang),
|
|
|
|
args.transport_security,
|
|
|
|
manual_cmd_log=server_manual_cmd_log,
|
|
|
|
)
|
|
|
|
if not args.manual_run:
|
|
|
|
job = dockerjob.DockerJob(spec)
|
|
|
|
server_jobs[lang] = job
|
|
|
|
server_addresses[lang] = (
|
|
|
|
"localhost",
|
|
|
|
job.mapped_port(_DEFAULT_SERVER_PORT),
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
# don't run the server, set server port to a placeholder value
|
|
|
|
server_addresses[lang] = ("localhost", "${SERVER_PORT}")
|
|
|
|
|
|
|
|
http2_server_job = None
|
|
|
|
if args.http2_server_interop:
|
|
|
|
# launch a HTTP2 server emulator that creates edge cases
|
|
|
|
lang = str(http2InteropServer)
|
|
|
|
spec = server_jobspec(
|
|
|
|
http2InteropServer,
|
|
|
|
docker_images.get(lang),
|
|
|
|
manual_cmd_log=server_manual_cmd_log,
|
|
|
|
)
|
|
|
|
if not args.manual_run:
|
|
|
|
http2_server_job = dockerjob.DockerJob(spec)
|
|
|
|
server_jobs[lang] = http2_server_job
|
|
|
|
else:
|
|
|
|
# don't run the server, set server port to a placeholder value
|
|
|
|
server_addresses[lang] = ("localhost", "${SERVER_PORT}")
|
|
|
|
|
|
|
|
jobs = []
|
|
|
|
if args.cloud_to_prod:
|
|
|
|
if args.transport_security not in ["tls"]:
|
|
|
|
print("TLS is always enabled for cloud_to_prod scenarios.")
|
|
|
|
for server_host_nickname in args.prod_servers:
|
|
|
|
for language in languages:
|
|
|
|
for test_case in _TEST_CASES:
|
|
|
|
if not test_case in language.unimplemented_test_cases():
|
|
|
|
if (
|
|
|
|
not test_case
|
|
|
|
in _SKIP_ADVANCED
|
|
|
|
+ _SKIP_COMPRESSION
|
|
|
|
+ _SKIP_SPECIAL_STATUS_MESSAGE
|
|
|
|
+ _ORCA_TEST_CASES
|
|
|
|
):
|
|
|
|
for (
|
|
|
|
transport_security
|
|
|
|
) in args.custom_credentials_type:
|
|
|
|
# google_default_credentials not yet supported by all languages
|
|
|
|
if (
|
|
|
|
transport_security
|
|
|
|
== "google_default_credentials"
|
|
|
|
and str(language)
|
|
|
|
not in ["c++", "go", "java", "javaokhttp"]
|
|
|
|
):
|
|
|
|
continue
|
|
|
|
# compute_engine_channel_creds not yet supported by all languages
|
|
|
|
if (
|
|
|
|
transport_security
|
|
|
|
== "compute_engine_channel_creds"
|
|
|
|
and str(language)
|
|
|
|
not in ["go", "java", "javaokhttp"]
|
|
|
|
):
|
|
|
|
continue
|
|
|
|
test_job = cloud_to_prod_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
server_host_nickname,
|
|
|
|
prod_servers[server_host_nickname],
|
|
|
|
google_default_creds_use_key_file=args.google_default_creds_use_key_file,
|
|
|
|
docker_image=docker_images.get(
|
|
|
|
str(language)
|
|
|
|
),
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
service_account_key_file=args.service_account_key_file,
|
|
|
|
default_service_account=args.default_service_account,
|
|
|
|
transport_security=transport_security,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
if args.http2_interop:
|
|
|
|
for test_case in _HTTP2_TEST_CASES:
|
|
|
|
test_job = cloud_to_prod_jobspec(
|
|
|
|
http2Interop,
|
|
|
|
test_case,
|
|
|
|
server_host_nickname,
|
|
|
|
prod_servers[server_host_nickname],
|
|
|
|
google_default_creds_use_key_file=args.google_default_creds_use_key_file,
|
|
|
|
docker_image=docker_images.get(str(http2Interop)),
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
service_account_key_file=args.service_account_key_file,
|
|
|
|
default_service_account=args.default_service_account,
|
|
|
|
transport_security=args.transport_security,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
|
|
|
|
if args.cloud_to_prod_auth:
|
|
|
|
if args.transport_security not in ["tls"]:
|
|
|
|
print("TLS is always enabled for cloud_to_prod scenarios.")
|
|
|
|
for server_host_nickname in args.prod_servers:
|
|
|
|
for language in languages:
|
|
|
|
for test_case in _AUTH_TEST_CASES:
|
|
|
|
if (
|
|
|
|
not args.skip_compute_engine_creds
|
|
|
|
or not compute_engine_creds_required(
|
|
|
|
language, test_case
|
|
|
|
)
|
|
|
|
):
|
|
|
|
if not test_case in language.unimplemented_test_cases():
|
|
|
|
if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE:
|
|
|
|
transport_security = (
|
|
|
|
"google_default_credentials"
|
|
|
|
)
|
|
|
|
elif (
|
|
|
|
test_case
|
|
|
|
== _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE
|
|
|
|
):
|
|
|
|
transport_security = (
|
|
|
|
"compute_engine_channel_creds"
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
transport_security = "tls"
|
|
|
|
if (
|
|
|
|
transport_security
|
|
|
|
not in args.custom_credentials_type
|
|
|
|
):
|
|
|
|
continue
|
|
|
|
test_job = cloud_to_prod_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
server_host_nickname,
|
|
|
|
prod_servers[server_host_nickname],
|
|
|
|
google_default_creds_use_key_file=args.google_default_creds_use_key_file,
|
|
|
|
docker_image=docker_images.get(str(language)),
|
|
|
|
auth=True,
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
service_account_key_file=args.service_account_key_file,
|
|
|
|
default_service_account=args.default_service_account,
|
|
|
|
transport_security=transport_security,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
for server in args.override_server:
|
|
|
|
server_name = server[0]
|
|
|
|
(server_host, server_port) = server[1].split(":")
|
|
|
|
server_addresses[server_name] = (server_host, server_port)
|
|
|
|
|
|
|
|
for server_name, server_address in list(server_addresses.items()):
|
|
|
|
(server_host, server_port) = server_address
|
|
|
|
server_language = _LANGUAGES.get(server_name, None)
|
|
|
|
skip_server = [] # test cases unimplemented by server
|
|
|
|
if server_language:
|
|
|
|
skip_server = server_language.unimplemented_test_cases_server()
|
|
|
|
for language in languages:
|
|
|
|
for test_case in _TEST_CASES:
|
|
|
|
if not test_case in language.unimplemented_test_cases():
|
|
|
|
if not test_case in skip_server:
|
|
|
|
test_job = cloud_to_cloud_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
server_name,
|
|
|
|
server_host,
|
|
|
|
server_port,
|
|
|
|
docker_image=docker_images.get(str(language)),
|
|
|
|
transport_security=args.transport_security,
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
|
|
|
|
if args.http2_interop:
|
|
|
|
for test_case in _HTTP2_TEST_CASES:
|
|
|
|
if server_name == "go":
|
|
|
|
# TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434
|
|
|
|
continue
|
|
|
|
test_job = cloud_to_cloud_jobspec(
|
|
|
|
http2Interop,
|
|
|
|
test_case,
|
|
|
|
server_name,
|
|
|
|
server_host,
|
|
|
|
server_port,
|
|
|
|
docker_image=docker_images.get(str(http2Interop)),
|
|
|
|
transport_security=args.transport_security,
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
|
|
|
|
if args.http2_server_interop:
|
|
|
|
if not args.manual_run:
|
|
|
|
http2_server_job.wait_for_healthy(timeout_seconds=600)
|
|
|
|
for language in languages_http2_clients_for_http2_server_interop:
|
|
|
|
for test_case in set(_HTTP2_SERVER_TEST_CASES) - set(
|
|
|
|
_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS
|
|
|
|
):
|
|
|
|
offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case)
|
|
|
|
server_port = _DEFAULT_SERVER_PORT + offset
|
|
|
|
if not args.manual_run:
|
|
|
|
server_port = http2_server_job.mapped_port(server_port)
|
|
|
|
test_job = cloud_to_cloud_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
str(http2InteropServer),
|
|
|
|
"localhost",
|
|
|
|
server_port,
|
|
|
|
docker_image=docker_images.get(str(language)),
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
for language in languages:
|
|
|
|
# HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of
|
|
|
|
# HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather
|
|
|
|
# than specialized http2 clients, reusing existing test implementations.
|
|
|
|
# For example, in the "data_frame_padding" test, use language's gRPC
|
|
|
|
# interop clients and make them think that they're running "large_unary"
|
|
|
|
# test case. This avoids implementing a new test case in each language.
|
|
|
|
for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS:
|
|
|
|
if test_case not in language.unimplemented_test_cases():
|
|
|
|
offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case)
|
|
|
|
server_port = _DEFAULT_SERVER_PORT + offset
|
|
|
|
if not args.manual_run:
|
|
|
|
server_port = http2_server_job.mapped_port(server_port)
|
|
|
|
if args.transport_security != "insecure":
|
|
|
|
print(
|
|
|
|
"Creating grpc client to http2 server test case "
|
|
|
|
"with insecure connection, even though "
|
|
|
|
"args.transport_security is not insecure. Http2 "
|
|
|
|
"test server only supports insecure connections."
|
|
|
|
)
|
|
|
|
test_job = cloud_to_cloud_jobspec(
|
|
|
|
language,
|
|
|
|
test_case,
|
|
|
|
str(http2InteropServer),
|
|
|
|
"localhost",
|
|
|
|
server_port,
|
|
|
|
docker_image=docker_images.get(str(language)),
|
|
|
|
transport_security="insecure",
|
|
|
|
manual_cmd_log=client_manual_cmd_log,
|
|
|
|
)
|
|
|
|
jobs.append(test_job)
|
|
|
|
|
|
|
|
if not jobs:
|
|
|
|
print("No jobs to run.")
|
|
|
|
for image in six.itervalues(docker_images):
|
|
|
|
dockerjob.remove_image(image, skip_nonexistent=True)
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
if args.manual_run:
|
|
|
|
print("All tests will skipped --manual_run option is active.")
|
|
|
|
|
|
|
|
if args.verbose:
|
|
|
|
print("Jobs to run: \n%s\n" % "\n".join(str(job) for job in jobs))
|
|
|
|
|
|
|
|
num_failures, resultset = jobset.run(
|
|
|
|
jobs,
|
|
|
|
newline_on_success=True,
|
|
|
|
maxjobs=args.jobs,
|
|
|
|
skip_jobs=args.manual_run,
|
|
|
|
)
|
|
|
|
if args.bq_result_table and resultset:
|
|
|
|
upload_interop_results_to_bq(resultset, args.bq_result_table)
|
|
|
|
if num_failures:
|
|
|
|
jobset.message("FAILED", "Some tests failed", do_newline=True)
|
|
|
|
else:
|
|
|
|
jobset.message("SUCCESS", "All tests passed", do_newline=True)
|
|
|
|
|
|
|
|
write_cmdlog_maybe(server_manual_cmd_log, "interop_server_cmds.sh")
|
|
|
|
write_cmdlog_maybe(client_manual_cmd_log, "interop_client_cmds.sh")
|
|
|
|
|
|
|
|
report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT)
|
|
|
|
|
|
|
|
for name, job in list(resultset.items()):
|
|
|
|
if "http2" in name:
|
|
|
|
job[0].http2results = aggregate_http2_results(job[0].message)
|
|
|
|
|
|
|
|
http2_server_test_cases = (
|
|
|
|
_HTTP2_SERVER_TEST_CASES if args.http2_server_interop else []
|
|
|
|
)
|
|
|
|
|
|
|
|
if num_failures:
|
|
|
|
sys.exit(1)
|
|
|
|
else:
|
|
|
|
sys.exit(0)
|
|
|
|
finally:
|
|
|
|
# Check if servers are still running.
|
|
|
|
for server, job in list(server_jobs.items()):
|
|
|
|
if not job.is_running():
|
|
|
|
print('Server "%s" has exited prematurely.' % server)
|
|
|
|
|
|
|
|
dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)])
|
|
|
|
|
|
|
|
for image in six.itervalues(docker_images):
|
|
|
|
if not args.manual_run:
|
|
|
|
print("Removing docker image %s" % image)
|
|
|
|
dockerjob.remove_image(image)
|
|
|
|
else:
|
|
|
|
print("Preserving docker image: %s" % image)
|