|
|
|
@ -12,7 +12,6 @@ |
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
|
|
# See the License for the specific language governing permissions and |
|
|
|
# See the License for the specific language governing permissions and |
|
|
|
# limitations under the License. |
|
|
|
# limitations under the License. |
|
|
|
|
|
|
|
|
|
|
|
"""Run interop (cross-language) tests in parallel.""" |
|
|
|
"""Run interop (cross-language) tests in parallel.""" |
|
|
|
|
|
|
|
|
|
|
|
from __future__ import print_function |
|
|
|
from __future__ import print_function |
|
|
|
@ -47,22 +46,24 @@ atexit.register(lambda: subprocess.call(['stty', 'echo'])) |
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) |
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) |
|
|
|
os.chdir(ROOT) |
|
|
|
os.chdir(ROOT) |
|
|
|
|
|
|
|
|
|
|
|
_DEFAULT_SERVER_PORT=8080 |
|
|
|
_DEFAULT_SERVER_PORT = 8080 |
|
|
|
|
|
|
|
|
|
|
|
_SKIP_CLIENT_COMPRESSION = ['client_compressed_unary', |
|
|
|
_SKIP_CLIENT_COMPRESSION = [ |
|
|
|
'client_compressed_streaming'] |
|
|
|
'client_compressed_unary', 'client_compressed_streaming' |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
_SKIP_SERVER_COMPRESSION = ['server_compressed_unary', |
|
|
|
_SKIP_SERVER_COMPRESSION = [ |
|
|
|
'server_compressed_streaming'] |
|
|
|
'server_compressed_unary', 'server_compressed_streaming' |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION |
|
|
|
_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION |
|
|
|
|
|
|
|
|
|
|
|
_SKIP_ADVANCED = ['status_code_and_message', |
|
|
|
_SKIP_ADVANCED = [ |
|
|
|
'custom_metadata', |
|
|
|
'status_code_and_message', 'custom_metadata', 'unimplemented_method', |
|
|
|
'unimplemented_method', |
|
|
|
'unimplemented_service' |
|
|
|
'unimplemented_service'] |
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
_TEST_TIMEOUT = 3*60 |
|
|
|
_TEST_TIMEOUT = 3 * 60 |
|
|
|
|
|
|
|
|
|
|
|
# disable this test on core-based languages, |
|
|
|
# disable this test on core-based languages, |
|
|
|
# see https://github.com/grpc/grpc/issues/9779 |
|
|
|
# see https://github.com/grpc/grpc/issues/9779 |
|
|
|
@ -178,7 +179,9 @@ class JavaLanguage: |
|
|
|
return ['./run-test-client.sh'] + args |
|
|
|
return ['./run-test-client.sh'] + args |
|
|
|
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args): |
|
|
|
def client_cmd_http2interop(self, args): |
|
|
|
return ['./interop-testing/build/install/grpc-interop-testing/bin/http2-client'] + args |
|
|
|
return [ |
|
|
|
|
|
|
|
'./interop-testing/build/install/grpc-interop-testing/bin/http2-client' |
|
|
|
|
|
|
|
] + args |
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self): |
|
|
|
def cloud_to_prod_env(self): |
|
|
|
return {} |
|
|
|
return {} |
|
|
|
@ -254,12 +257,14 @@ class GoLanguage: |
|
|
|
def __str__(self): |
|
|
|
def __str__(self): |
|
|
|
return 'go' |
|
|
|
return 'go' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Http2Server: |
|
|
|
class Http2Server: |
|
|
|
"""Represents the HTTP/2 Interop Test server |
|
|
|
"""Represents the HTTP/2 Interop Test server |
|
|
|
|
|
|
|
|
|
|
|
This pretends to be a language in order to be built and run, but really it |
|
|
|
This pretends to be a language in order to be built and run, but really it |
|
|
|
isn't. |
|
|
|
isn't. |
|
|
|
""" |
|
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
def __init__(self): |
|
|
|
self.server_cwd = None |
|
|
|
self.server_cwd = None |
|
|
|
self.safename = str(self) |
|
|
|
self.safename = str(self) |
|
|
|
@ -282,12 +287,14 @@ class Http2Server: |
|
|
|
def __str__(self): |
|
|
|
def __str__(self): |
|
|
|
return 'http2' |
|
|
|
return 'http2' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Http2Client: |
|
|
|
class Http2Client: |
|
|
|
"""Represents the HTTP/2 Interop Test |
|
|
|
"""Represents the HTTP/2 Interop Test |
|
|
|
|
|
|
|
|
|
|
|
This pretends to be a language in order to be built and run, but really it |
|
|
|
This pretends to be a language in order to be built and run, but really it |
|
|
|
isn't. |
|
|
|
isn't. |
|
|
|
""" |
|
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
def __init__(self): |
|
|
|
self.client_cwd = None |
|
|
|
self.client_cwd = None |
|
|
|
self.safename = str(self) |
|
|
|
self.safename = str(self) |
|
|
|
@ -310,6 +317,7 @@ class Http2Client: |
|
|
|
def __str__(self): |
|
|
|
def __str__(self): |
|
|
|
return 'http2' |
|
|
|
return 'http2' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class NodeLanguage: |
|
|
|
class NodeLanguage: |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
def __init__(self): |
|
|
|
@ -318,17 +326,21 @@ class NodeLanguage: |
|
|
|
self.safename = str(self) |
|
|
|
self.safename = str(self) |
|
|
|
|
|
|
|
|
|
|
|
def client_cmd(self, args): |
|
|
|
def client_cmd(self, args): |
|
|
|
return ['packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
|
|
|
return [ |
|
|
|
|
|
|
|
'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
|
|
|
'node', '--require', './test/fixtures/native_native', |
|
|
|
'node', '--require', './test/fixtures/native_native', |
|
|
|
'test/interop/interop_client.js'] + args |
|
|
|
'test/interop/interop_client.js' |
|
|
|
|
|
|
|
] + args |
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self): |
|
|
|
def cloud_to_prod_env(self): |
|
|
|
return {} |
|
|
|
return {} |
|
|
|
|
|
|
|
|
|
|
|
def server_cmd(self, args): |
|
|
|
def server_cmd(self, args): |
|
|
|
return ['packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
|
|
|
return [ |
|
|
|
|
|
|
|
'packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh', |
|
|
|
'node', '--require', './test/fixtures/native_native', |
|
|
|
'node', '--require', './test/fixtures/native_native', |
|
|
|
'test/interop/interop_server.js'] + args |
|
|
|
'test/interop/interop_server.js' |
|
|
|
|
|
|
|
] + args |
|
|
|
|
|
|
|
|
|
|
|
def global_env(self): |
|
|
|
def global_env(self): |
|
|
|
return {} |
|
|
|
return {} |
|
|
|
@ -392,6 +404,7 @@ class PHP7Language: |
|
|
|
def __str__(self): |
|
|
|
def __str__(self): |
|
|
|
return 'php7' |
|
|
|
return 'php7' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ObjcLanguage: |
|
|
|
class ObjcLanguage: |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
def __init__(self): |
|
|
|
@ -404,7 +417,7 @@ class ObjcLanguage: |
|
|
|
port = re.search('--server_port=(\d+)', arg) |
|
|
|
port = re.search('--server_port=(\d+)', arg) |
|
|
|
if port: |
|
|
|
if port: |
|
|
|
portnum = port.group(1) |
|
|
|
portnum = port.group(1) |
|
|
|
cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test'%portnum |
|
|
|
cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test' % portnum |
|
|
|
return [cmdline] |
|
|
|
return [cmdline] |
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self): |
|
|
|
def cloud_to_prod_env(self): |
|
|
|
@ -426,6 +439,7 @@ class ObjcLanguage: |
|
|
|
def __str__(self): |
|
|
|
def __str__(self): |
|
|
|
return 'objc' |
|
|
|
return 'objc' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class RubyLanguage: |
|
|
|
class RubyLanguage: |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
def __init__(self): |
|
|
|
@ -434,15 +448,19 @@ class RubyLanguage: |
|
|
|
self.safename = str(self) |
|
|
|
self.safename = str(self) |
|
|
|
|
|
|
|
|
|
|
|
def client_cmd(self, args): |
|
|
|
def client_cmd(self, args): |
|
|
|
return ['tools/run_tests/interop/with_rvm.sh', |
|
|
|
return [ |
|
|
|
'ruby', 'src/ruby/pb/test/client.rb'] + args |
|
|
|
'tools/run_tests/interop/with_rvm.sh', 'ruby', |
|
|
|
|
|
|
|
'src/ruby/pb/test/client.rb' |
|
|
|
|
|
|
|
] + args |
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_prod_env(self): |
|
|
|
def cloud_to_prod_env(self): |
|
|
|
return {} |
|
|
|
return {} |
|
|
|
|
|
|
|
|
|
|
|
def server_cmd(self, args): |
|
|
|
def server_cmd(self, args): |
|
|
|
return ['tools/run_tests/interop/with_rvm.sh', |
|
|
|
return [ |
|
|
|
'ruby', 'src/ruby/pb/test/server.rb'] + args |
|
|
|
'tools/run_tests/interop/with_rvm.sh', 'ruby', |
|
|
|
|
|
|
|
'src/ruby/pb/test/server.rb' |
|
|
|
|
|
|
|
] + args |
|
|
|
|
|
|
|
|
|
|
|
def global_env(self): |
|
|
|
def global_env(self): |
|
|
|
return {} |
|
|
|
return {} |
|
|
|
@ -456,6 +474,7 @@ class RubyLanguage: |
|
|
|
def __str__(self): |
|
|
|
def __str__(self): |
|
|
|
return 'ruby' |
|
|
|
return 'ruby' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PythonLanguage: |
|
|
|
class PythonLanguage: |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
def __init__(self): |
|
|
|
@ -466,15 +485,13 @@ class PythonLanguage: |
|
|
|
|
|
|
|
|
|
|
|
def client_cmd(self, args): |
|
|
|
def client_cmd(self, args): |
|
|
|
return [ |
|
|
|
return [ |
|
|
|
'py27/bin/python', |
|
|
|
'py27/bin/python', 'src/python/grpcio_tests/setup.py', |
|
|
|
'src/python/grpcio_tests/setup.py', |
|
|
|
'run_interop', '--client', '--args="{}"'.format(' '.join(args)) |
|
|
|
'run_interop', |
|
|
|
|
|
|
|
'--client', |
|
|
|
|
|
|
|
'--args="{}"'.format(' '.join(args)) |
|
|
|
|
|
|
|
] |
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
def client_cmd_http2interop(self, args): |
|
|
|
def client_cmd_http2interop(self, args): |
|
|
|
return [ 'py27/bin/python', |
|
|
|
return [ |
|
|
|
|
|
|
|
'py27/bin/python', |
|
|
|
'src/python/grpcio_tests/tests/http2/negative_http2_client.py', |
|
|
|
'src/python/grpcio_tests/tests/http2/negative_http2_client.py', |
|
|
|
] + args |
|
|
|
] + args |
|
|
|
|
|
|
|
|
|
|
|
@ -483,16 +500,15 @@ class PythonLanguage: |
|
|
|
|
|
|
|
|
|
|
|
def server_cmd(self, args): |
|
|
|
def server_cmd(self, args): |
|
|
|
return [ |
|
|
|
return [ |
|
|
|
'py27/bin/python', |
|
|
|
'py27/bin/python', 'src/python/grpcio_tests/setup.py', |
|
|
|
'src/python/grpcio_tests/setup.py', |
|
|
|
'run_interop', '--server', '--args="{}"'.format(' '.join(args)) |
|
|
|
'run_interop', |
|
|
|
|
|
|
|
'--server', |
|
|
|
|
|
|
|
'--args="{}"'.format(' '.join(args)) |
|
|
|
|
|
|
|
] |
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
def global_env(self): |
|
|
|
def global_env(self): |
|
|
|
return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), |
|
|
|
return { |
|
|
|
'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT)} |
|
|
|
'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT), |
|
|
|
|
|
|
|
'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
def unimplemented_test_cases(self): |
|
|
|
def unimplemented_test_cases(self): |
|
|
|
return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING |
|
|
|
return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING |
|
|
|
@ -505,56 +521,70 @@ class PythonLanguage: |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_LANGUAGES = { |
|
|
|
_LANGUAGES = { |
|
|
|
'c++' : CXXLanguage(), |
|
|
|
'c++': CXXLanguage(), |
|
|
|
'csharp' : CSharpLanguage(), |
|
|
|
'csharp': CSharpLanguage(), |
|
|
|
'csharpcoreclr' : CSharpCoreCLRLanguage(), |
|
|
|
'csharpcoreclr': CSharpCoreCLRLanguage(), |
|
|
|
'go' : GoLanguage(), |
|
|
|
'go': GoLanguage(), |
|
|
|
'java' : JavaLanguage(), |
|
|
|
'java': JavaLanguage(), |
|
|
|
'javaokhttp' : JavaOkHttpClient(), |
|
|
|
'javaokhttp': JavaOkHttpClient(), |
|
|
|
'node' : NodeLanguage(), |
|
|
|
'node': NodeLanguage(), |
|
|
|
'php' : PHPLanguage(), |
|
|
|
'php': PHPLanguage(), |
|
|
|
'php7' : PHP7Language(), |
|
|
|
'php7': PHP7Language(), |
|
|
|
'objc' : ObjcLanguage(), |
|
|
|
'objc': ObjcLanguage(), |
|
|
|
'ruby' : RubyLanguage(), |
|
|
|
'ruby': RubyLanguage(), |
|
|
|
'python' : PythonLanguage(), |
|
|
|
'python': PythonLanguage(), |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
# languages supported as cloud_to_cloud servers |
|
|
|
# languages supported as cloud_to_cloud servers |
|
|
|
_SERVERS = ['c++', 'node', 'csharp', 'csharpcoreclr', 'java', 'go', 'ruby', 'python'] |
|
|
|
_SERVERS = [ |
|
|
|
|
|
|
|
'c++', 'node', 'csharp', 'csharpcoreclr', 'java', 'go', 'ruby', 'python' |
|
|
|
_TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', |
|
|
|
] |
|
|
|
'empty_stream', 'client_streaming', 'server_streaming', |
|
|
|
|
|
|
|
'cancel_after_begin', 'cancel_after_first_response', |
|
|
|
_TEST_CASES = [ |
|
|
|
'timeout_on_sleeping_server', 'custom_metadata', |
|
|
|
'large_unary', 'empty_unary', 'ping_pong', 'empty_stream', |
|
|
|
'status_code_and_message', 'unimplemented_method', |
|
|
|
'client_streaming', 'server_streaming', 'cancel_after_begin', |
|
|
|
|
|
|
|
'cancel_after_first_response', 'timeout_on_sleeping_server', |
|
|
|
|
|
|
|
'custom_metadata', 'status_code_and_message', 'unimplemented_method', |
|
|
|
'client_compressed_unary', 'server_compressed_unary', |
|
|
|
'client_compressed_unary', 'server_compressed_unary', |
|
|
|
'client_compressed_streaming', 'server_compressed_streaming', |
|
|
|
'client_compressed_streaming', 'server_compressed_streaming', |
|
|
|
'unimplemented_service'] |
|
|
|
'unimplemented_service' |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
_AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', |
|
|
|
_AUTH_TEST_CASES = [ |
|
|
|
'oauth2_auth_token', 'per_rpc_creds'] |
|
|
|
'compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', |
|
|
|
|
|
|
|
'per_rpc_creds' |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
_HTTP2_TEST_CASES = ['tls', 'framing'] |
|
|
|
_HTTP2_TEST_CASES = ['tls', 'framing'] |
|
|
|
|
|
|
|
|
|
|
|
_HTTP2_SERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data', |
|
|
|
_HTTP2_SERVER_TEST_CASES = [ |
|
|
|
'goaway', 'ping', 'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test'] |
|
|
|
'rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', |
|
|
|
|
|
|
|
'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test' |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 'data_frame_padding': 'large_unary', 'no_df_padding_sanity_test': 'large_unary' } |
|
|
|
_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { |
|
|
|
|
|
|
|
'data_frame_padding': 'large_unary', |
|
|
|
|
|
|
|
'no_df_padding_sanity_test': 'large_unary' |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys() |
|
|
|
_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys( |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = ['java', 'go', 'python', 'c++'] |
|
|
|
_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ |
|
|
|
|
|
|
|
'java', 'go', 'python', 'c++' |
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
|
|
DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' |
|
|
|
DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): |
|
|
|
def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): |
|
|
|
"""Wraps given cmdline array to create 'docker run' cmdline from it.""" |
|
|
|
"""Wraps given cmdline array to create 'docker run' cmdline from it.""" |
|
|
|
docker_cmdline = ['docker', 'run', '-i', '--rm=true'] |
|
|
|
docker_cmdline = ['docker', 'run', '-i', '--rm=true'] |
|
|
|
|
|
|
|
|
|
|
|
# turn environ into -e docker args |
|
|
|
# turn environ into -e docker args |
|
|
|
if environ: |
|
|
|
if environ: |
|
|
|
for k,v in environ.items(): |
|
|
|
for k, v in environ.items(): |
|
|
|
docker_cmdline += ['-e', '%s=%s' % (k,v)] |
|
|
|
docker_cmdline += ['-e', '%s=%s' % (k, v)] |
|
|
|
|
|
|
|
|
|
|
|
# set working directory |
|
|
|
# set working directory |
|
|
|
workdir = DOCKER_WORKDIR_ROOT |
|
|
|
workdir = DOCKER_WORKDIR_ROOT |
|
|
|
@ -612,7 +642,10 @@ def auth_options(language, test_case): |
|
|
|
default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' |
|
|
|
default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' |
|
|
|
|
|
|
|
|
|
|
|
if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: |
|
|
|
if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: |
|
|
|
if language in ['csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', 'ruby']: |
|
|
|
if language in [ |
|
|
|
|
|
|
|
'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', |
|
|
|
|
|
|
|
'ruby' |
|
|
|
|
|
|
|
]: |
|
|
|
env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath |
|
|
|
env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath |
|
|
|
else: |
|
|
|
else: |
|
|
|
cmdargs += [key_file_arg] |
|
|
|
cmdargs += [key_file_arg] |
|
|
|
@ -640,17 +673,20 @@ def _job_kill_handler(job): |
|
|
|
time.sleep(2) |
|
|
|
time.sleep(2) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_prod_jobspec(language, test_case, server_host_name, |
|
|
|
def cloud_to_prod_jobspec(language, |
|
|
|
server_host_detail, docker_image=None, auth=False, |
|
|
|
test_case, |
|
|
|
|
|
|
|
server_host_name, |
|
|
|
|
|
|
|
server_host_detail, |
|
|
|
|
|
|
|
docker_image=None, |
|
|
|
|
|
|
|
auth=False, |
|
|
|
manual_cmd_log=None): |
|
|
|
manual_cmd_log=None): |
|
|
|
"""Creates jobspec for cloud-to-prod interop test""" |
|
|
|
"""Creates jobspec for cloud-to-prod interop test""" |
|
|
|
container_name = None |
|
|
|
container_name = None |
|
|
|
cmdargs = [ |
|
|
|
cmdargs = [ |
|
|
|
'--server_host=%s' % server_host_detail[0], |
|
|
|
'--server_host=%s' % server_host_detail[0], |
|
|
|
'--server_host_override=%s' % server_host_detail[1], |
|
|
|
'--server_host_override=%s' % server_host_detail[1], |
|
|
|
'--server_port=443', |
|
|
|
'--server_port=443', '--use_tls=true', '--test_case=%s' % test_case |
|
|
|
'--use_tls=true', |
|
|
|
] |
|
|
|
'--test_case=%s' % test_case] |
|
|
|
|
|
|
|
environ = dict(language.cloud_to_prod_env(), **language.global_env()) |
|
|
|
environ = dict(language.cloud_to_prod_env(), **language.global_env()) |
|
|
|
if auth: |
|
|
|
if auth: |
|
|
|
auth_cmdargs, auth_env = auth_options(language, test_case) |
|
|
|
auth_cmdargs, auth_env = auth_options(language, test_case) |
|
|
|
@ -662,20 +698,21 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, |
|
|
|
if docker_image: |
|
|
|
if docker_image: |
|
|
|
container_name = dockerjob.random_name('interop_client_%s' % |
|
|
|
container_name = dockerjob.random_name('interop_client_%s' % |
|
|
|
language.safename) |
|
|
|
language.safename) |
|
|
|
cmdline = docker_run_cmdline(cmdline, |
|
|
|
cmdline = docker_run_cmdline( |
|
|
|
|
|
|
|
cmdline, |
|
|
|
image=docker_image, |
|
|
|
image=docker_image, |
|
|
|
cwd=cwd, |
|
|
|
cwd=cwd, |
|
|
|
environ=environ, |
|
|
|
environ=environ, |
|
|
|
docker_args=['--net=host', |
|
|
|
docker_args=['--net=host', '--name=%s' % container_name]) |
|
|
|
'--name=%s' % container_name]) |
|
|
|
|
|
|
|
if manual_cmd_log is not None: |
|
|
|
if manual_cmd_log is not None: |
|
|
|
if manual_cmd_log == []: |
|
|
|
if manual_cmd_log == []: |
|
|
|
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image) |
|
|
|
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % |
|
|
|
|
|
|
|
docker_image) |
|
|
|
manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
|
|
|
manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
|
|
|
cwd = None |
|
|
|
cwd = None |
|
|
|
environ = None |
|
|
|
environ = None |
|
|
|
|
|
|
|
|
|
|
|
suite_name='cloud_to_prod_auth' if auth else 'cloud_to_prod' |
|
|
|
suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod' |
|
|
|
test_job = jobset.JobSpec( |
|
|
|
test_job = jobset.JobSpec( |
|
|
|
cmdline=cmdline, |
|
|
|
cmdline=cmdline, |
|
|
|
cwd=cwd, |
|
|
|
cwd=cwd, |
|
|
|
@ -691,8 +728,13 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, |
|
|
|
return test_job |
|
|
|
return test_job |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, |
|
|
|
def cloud_to_cloud_jobspec(language, |
|
|
|
server_port, docker_image=None, insecure=False, |
|
|
|
test_case, |
|
|
|
|
|
|
|
server_name, |
|
|
|
|
|
|
|
server_host, |
|
|
|
|
|
|
|
server_port, |
|
|
|
|
|
|
|
docker_image=None, |
|
|
|
|
|
|
|
insecure=False, |
|
|
|
manual_cmd_log=None): |
|
|
|
manual_cmd_log=None): |
|
|
|
"""Creates jobspec for cloud-to-cloud interop test""" |
|
|
|
"""Creates jobspec for cloud-to-cloud interop test""" |
|
|
|
interop_only_options = [ |
|
|
|
interop_only_options = [ |
|
|
|
@ -703,9 +745,11 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, |
|
|
|
|
|
|
|
|
|
|
|
client_test_case = test_case |
|
|
|
client_test_case = test_case |
|
|
|
if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
|
|
|
if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
|
|
|
client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[test_case] |
|
|
|
client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ |
|
|
|
|
|
|
|
test_case] |
|
|
|
if client_test_case in language.unimplemented_test_cases(): |
|
|
|
if client_test_case in language.unimplemented_test_cases(): |
|
|
|
print('asking client %s to run unimplemented test case %s' % (repr(language), client_test_case)) |
|
|
|
print('asking client %s to run unimplemented test case %s' % |
|
|
|
|
|
|
|
(repr(language), client_test_case)) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
common_options = [ |
|
|
|
common_options = [ |
|
|
|
@ -720,25 +764,29 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, |
|
|
|
cmdline = bash_cmdline(language.client_cmd(client_options)) |
|
|
|
cmdline = bash_cmdline(language.client_cmd(client_options)) |
|
|
|
cwd = language.client_cwd |
|
|
|
cwd = language.client_cwd |
|
|
|
else: |
|
|
|
else: |
|
|
|
cmdline = bash_cmdline(language.client_cmd_http2interop(common_options)) |
|
|
|
cmdline = bash_cmdline( |
|
|
|
|
|
|
|
language.client_cmd_http2interop(common_options)) |
|
|
|
cwd = language.http2_cwd |
|
|
|
cwd = language.http2_cwd |
|
|
|
else: |
|
|
|
else: |
|
|
|
cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options)) |
|
|
|
cmdline = bash_cmdline( |
|
|
|
|
|
|
|
language.client_cmd(common_options + interop_only_options)) |
|
|
|
cwd = language.client_cwd |
|
|
|
cwd = language.client_cwd |
|
|
|
|
|
|
|
|
|
|
|
environ = language.global_env() |
|
|
|
environ = language.global_env() |
|
|
|
if docker_image and language.safename != 'objc': |
|
|
|
if docker_image and language.safename != 'objc': |
|
|
|
# we can't run client in docker for objc. |
|
|
|
# we can't run client in docker for objc. |
|
|
|
container_name = dockerjob.random_name('interop_client_%s' % language.safename) |
|
|
|
container_name = dockerjob.random_name('interop_client_%s' % |
|
|
|
cmdline = docker_run_cmdline(cmdline, |
|
|
|
language.safename) |
|
|
|
|
|
|
|
cmdline = docker_run_cmdline( |
|
|
|
|
|
|
|
cmdline, |
|
|
|
image=docker_image, |
|
|
|
image=docker_image, |
|
|
|
environ=environ, |
|
|
|
environ=environ, |
|
|
|
cwd=cwd, |
|
|
|
cwd=cwd, |
|
|
|
docker_args=['--net=host', |
|
|
|
docker_args=['--net=host', '--name=%s' % container_name]) |
|
|
|
'--name=%s' % container_name]) |
|
|
|
|
|
|
|
if manual_cmd_log is not None: |
|
|
|
if manual_cmd_log is not None: |
|
|
|
if manual_cmd_log == []: |
|
|
|
if manual_cmd_log == []: |
|
|
|
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image) |
|
|
|
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % |
|
|
|
|
|
|
|
docker_image) |
|
|
|
manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
|
|
|
manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
|
|
|
cwd = None |
|
|
|
cwd = None |
|
|
|
|
|
|
|
|
|
|
|
@ -759,10 +807,13 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, |
|
|
|
|
|
|
|
|
|
|
|
def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): |
|
|
|
def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): |
|
|
|
"""Create jobspec for running a server""" |
|
|
|
"""Create jobspec for running a server""" |
|
|
|
container_name = dockerjob.random_name('interop_server_%s' % language.safename) |
|
|
|
container_name = dockerjob.random_name('interop_server_%s' % |
|
|
|
|
|
|
|
language.safename) |
|
|
|
cmdline = bash_cmdline( |
|
|
|
cmdline = bash_cmdline( |
|
|
|
language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT, |
|
|
|
language.server_cmd([ |
|
|
|
'--use_tls=%s' % ('false' if insecure else 'true')])) |
|
|
|
'--port=%s' % _DEFAULT_SERVER_PORT, '--use_tls=%s' % ( |
|
|
|
|
|
|
|
'false' if insecure else 'true') |
|
|
|
|
|
|
|
])) |
|
|
|
environ = language.global_env() |
|
|
|
environ = language.global_env() |
|
|
|
docker_args = ['--name=%s' % container_name] |
|
|
|
docker_args = ['--name=%s' % container_name] |
|
|
|
if language.safename == 'http2': |
|
|
|
if language.safename == 'http2': |
|
|
|
@ -783,8 +834,8 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): |
|
|
|
# command line. |
|
|
|
# command line. |
|
|
|
docker_args += [ |
|
|
|
docker_args += [ |
|
|
|
'--health-cmd=python test/http2_test/http2_server_health_check.py ' |
|
|
|
'--health-cmd=python test/http2_test/http2_server_health_check.py ' |
|
|
|
'--server_host=%s --server_port=%d' |
|
|
|
'--server_host=%s --server_port=%d' % |
|
|
|
% ('localhost', _DEFAULT_SERVER_PORT), |
|
|
|
('localhost', _DEFAULT_SERVER_PORT), |
|
|
|
'--health-interval=1s', |
|
|
|
'--health-interval=1s', |
|
|
|
'--health-retries=5', |
|
|
|
'--health-retries=5', |
|
|
|
'--health-timeout=10s', |
|
|
|
'--health-timeout=10s', |
|
|
|
@ -793,20 +844,22 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): |
|
|
|
else: |
|
|
|
else: |
|
|
|
docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] |
|
|
|
docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] |
|
|
|
|
|
|
|
|
|
|
|
docker_cmdline = docker_run_cmdline(cmdline, |
|
|
|
docker_cmdline = docker_run_cmdline( |
|
|
|
|
|
|
|
cmdline, |
|
|
|
image=docker_image, |
|
|
|
image=docker_image, |
|
|
|
cwd=language.server_cwd, |
|
|
|
cwd=language.server_cwd, |
|
|
|
environ=environ, |
|
|
|
environ=environ, |
|
|
|
docker_args=docker_args) |
|
|
|
docker_args=docker_args) |
|
|
|
if manual_cmd_log is not None: |
|
|
|
if manual_cmd_log is not None: |
|
|
|
if manual_cmd_log == []: |
|
|
|
if manual_cmd_log == []: |
|
|
|
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % docker_image) |
|
|
|
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' % |
|
|
|
|
|
|
|
docker_image) |
|
|
|
manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) |
|
|
|
manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) |
|
|
|
server_job = jobset.JobSpec( |
|
|
|
server_job = jobset.JobSpec( |
|
|
|
cmdline=docker_cmdline, |
|
|
|
cmdline=docker_cmdline, |
|
|
|
environ=environ, |
|
|
|
environ=environ, |
|
|
|
shortname='interop_server_%s' % language, |
|
|
|
shortname='interop_server_%s' % language, |
|
|
|
timeout_seconds=30*60) |
|
|
|
timeout_seconds=30 * 60) |
|
|
|
server_job.container_name = container_name |
|
|
|
server_job.container_name = container_name |
|
|
|
return server_job |
|
|
|
return server_job |
|
|
|
|
|
|
|
|
|
|
|
@ -815,8 +868,10 @@ def build_interop_image_jobspec(language, tag=None): |
|
|
|
"""Creates jobspec for building interop docker image for a language""" |
|
|
|
"""Creates jobspec for building interop docker image for a language""" |
|
|
|
if not tag: |
|
|
|
if not tag: |
|
|
|
tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) |
|
|
|
tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) |
|
|
|
env = {'INTEROP_IMAGE': tag, |
|
|
|
env = { |
|
|
|
'BASE_NAME': 'grpc_interop_%s' % language.safename} |
|
|
|
'INTEROP_IMAGE': tag, |
|
|
|
|
|
|
|
'BASE_NAME': 'grpc_interop_%s' % language.safename |
|
|
|
|
|
|
|
} |
|
|
|
if not args.travis: |
|
|
|
if not args.travis: |
|
|
|
env['TTY_FLAG'] = '-t' |
|
|
|
env['TTY_FLAG'] = '-t' |
|
|
|
# This env variable is used to get around the github rate limit |
|
|
|
# This env variable is used to get around the github rate limit |
|
|
|
@ -829,7 +884,7 @@ def build_interop_image_jobspec(language, tag=None): |
|
|
|
cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], |
|
|
|
cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], |
|
|
|
environ=env, |
|
|
|
environ=env, |
|
|
|
shortname='build_docker_%s' % (language), |
|
|
|
shortname='build_docker_%s' % (language), |
|
|
|
timeout_seconds=30*60) |
|
|
|
timeout_seconds=30 * 60) |
|
|
|
build_job.tag = tag |
|
|
|
build_job.tag = tag |
|
|
|
return build_job |
|
|
|
return build_job |
|
|
|
|
|
|
|
|
|
|
|
@ -861,126 +916,141 @@ def aggregate_http2_results(stdout): |
|
|
|
'percent': 1.0 * passed / (passed + failed) |
|
|
|
'percent': 1.0 * passed / (passed + failed) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# A dictionary of prod servers to test. |
|
|
|
# A dictionary of prod servers to test. |
|
|
|
# Format: server_name: (server_host, server_host_override, errors_allowed) |
|
|
|
# Format: server_name: (server_host, server_host_override, errors_allowed) |
|
|
|
# TODO(adelez): implement logic for errors_allowed where if the indicated tests |
|
|
|
# TODO(adelez): implement logic for errors_allowed where if the indicated tests |
|
|
|
# fail, they don't impact the overall test result. |
|
|
|
# fail, they don't impact the overall test result. |
|
|
|
prod_servers = { |
|
|
|
prod_servers = { |
|
|
|
'default': ('216.239.32.254', |
|
|
|
'default': ('216.239.32.254', 'grpc-test.sandbox.googleapis.com', False), |
|
|
|
'grpc-test.sandbox.googleapis.com', False), |
|
|
|
'gateway_v2': ('216.239.32.254', 'grpc-test2.sandbox.googleapis.com', True), |
|
|
|
'gateway_v2': ('216.239.32.254', |
|
|
|
|
|
|
|
'grpc-test2.sandbox.googleapis.com', True), |
|
|
|
|
|
|
|
'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com', |
|
|
|
'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com', |
|
|
|
False), |
|
|
|
False), |
|
|
|
'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com', |
|
|
|
'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com', |
|
|
|
True), |
|
|
|
True), |
|
|
|
'gateway_v4': ('216.239.32.254', |
|
|
|
'gateway_v4': ('216.239.32.254', 'grpc-test4.sandbox.googleapis.com', True), |
|
|
|
'grpc-test4.sandbox.googleapis.com', True), |
|
|
|
|
|
|
|
'cloud_gateway_v4': ('216.239.32.255', 'grpc-test4.sandbox.googleapis.com', |
|
|
|
'cloud_gateway_v4': ('216.239.32.255', 'grpc-test4.sandbox.googleapis.com', |
|
|
|
True), |
|
|
|
True), |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
argp = argparse.ArgumentParser(description='Run interop tests.') |
|
|
|
argp = argparse.ArgumentParser(description='Run interop tests.') |
|
|
|
argp.add_argument('-l', '--language', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'-l', |
|
|
|
|
|
|
|
'--language', |
|
|
|
choices=['all'] + sorted(_LANGUAGES), |
|
|
|
choices=['all'] + sorted(_LANGUAGES), |
|
|
|
nargs='+', |
|
|
|
nargs='+', |
|
|
|
default=['all'], |
|
|
|
default=['all'], |
|
|
|
help='Clients to run. Objc client can be only run on OSX.') |
|
|
|
help='Clients to run. Objc client can be only run on OSX.') |
|
|
|
argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) |
|
|
|
argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) |
|
|
|
argp.add_argument('--cloud_to_prod', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--cloud_to_prod', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Run cloud_to_prod tests.') |
|
|
|
help='Run cloud_to_prod tests.') |
|
|
|
argp.add_argument('--cloud_to_prod_auth', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--cloud_to_prod_auth', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Run cloud_to_prod_auth tests.') |
|
|
|
help='Run cloud_to_prod_auth tests.') |
|
|
|
argp.add_argument('--prod_servers', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--prod_servers', |
|
|
|
choices=prod_servers.keys(), |
|
|
|
choices=prod_servers.keys(), |
|
|
|
default=['default'], |
|
|
|
default=['default'], |
|
|
|
nargs='+', |
|
|
|
nargs='+', |
|
|
|
help=('The servers to run cloud_to_prod and ' |
|
|
|
help=('The servers to run cloud_to_prod and ' |
|
|
|
'cloud_to_prod_auth tests against.')) |
|
|
|
'cloud_to_prod_auth tests against.')) |
|
|
|
argp.add_argument('-s', '--server', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'-s', |
|
|
|
|
|
|
|
'--server', |
|
|
|
choices=['all'] + sorted(_SERVERS), |
|
|
|
choices=['all'] + sorted(_SERVERS), |
|
|
|
nargs='+', |
|
|
|
nargs='+', |
|
|
|
help='Run cloud_to_cloud servers in a separate docker ' + |
|
|
|
help='Run cloud_to_cloud servers in a separate docker ' + |
|
|
|
'image. Servers can only be started automatically if ' + |
|
|
|
'image. Servers can only be started automatically if ' + |
|
|
|
'--use_docker option is enabled.', |
|
|
|
'--use_docker option is enabled.', |
|
|
|
default=[]) |
|
|
|
default=[]) |
|
|
|
argp.add_argument('--override_server', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--override_server', |
|
|
|
action='append', |
|
|
|
action='append', |
|
|
|
type=lambda kv: kv.split('='), |
|
|
|
type=lambda kv: kv.split('='), |
|
|
|
help='Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', |
|
|
|
help='Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', |
|
|
|
default=[]) |
|
|
|
default=[]) |
|
|
|
argp.add_argument('-t', '--travis', |
|
|
|
argp.add_argument( |
|
|
|
default=False, |
|
|
|
'-t', '--travis', default=False, action='store_const', const=True) |
|
|
|
action='store_const', |
|
|
|
argp.add_argument( |
|
|
|
const=True) |
|
|
|
'-v', '--verbose', default=False, action='store_const', const=True) |
|
|
|
argp.add_argument('-v', '--verbose', |
|
|
|
argp.add_argument( |
|
|
|
default=False, |
|
|
|
'--use_docker', |
|
|
|
action='store_const', |
|
|
|
|
|
|
|
const=True) |
|
|
|
|
|
|
|
argp.add_argument('--use_docker', |
|
|
|
|
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Run all the interop tests under docker. That provides ' + |
|
|
|
help='Run all the interop tests under docker. That provides ' + |
|
|
|
'additional isolation and prevents the need to install ' + |
|
|
|
'additional isolation and prevents the need to install ' + |
|
|
|
'language specific prerequisites. Only available on Linux.') |
|
|
|
'language specific prerequisites. Only available on Linux.') |
|
|
|
argp.add_argument('--allow_flakes', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--allow_flakes', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Allow flaky tests to show as passing (re-runs failed tests up to five times)') |
|
|
|
help='Allow flaky tests to show as passing (re-runs failed tests up to five times)' |
|
|
|
argp.add_argument('--manual_run', |
|
|
|
) |
|
|
|
|
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--manual_run', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Prepare things for running interop tests manually. ' + |
|
|
|
help='Prepare things for running interop tests manually. ' + |
|
|
|
'Preserve docker images after building them and skip ' |
|
|
|
'Preserve docker images after building them and skip ' |
|
|
|
'actually running the tests. Only print commands to run by ' + |
|
|
|
'actually running the tests. Only print commands to run by ' + 'hand.') |
|
|
|
'hand.') |
|
|
|
argp.add_argument( |
|
|
|
argp.add_argument('--http2_interop', |
|
|
|
'--http2_interop', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Enable HTTP/2 client edge case testing. (Bad client, good server)') |
|
|
|
help='Enable HTTP/2 client edge case testing. (Bad client, good server)') |
|
|
|
argp.add_argument('--http2_server_interop', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--http2_server_interop', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Enable HTTP/2 server edge case testing. (Includes positive and negative tests') |
|
|
|
help='Enable HTTP/2 server edge case testing. (Includes positive and negative tests' |
|
|
|
argp.add_argument('--insecure', |
|
|
|
) |
|
|
|
|
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--insecure', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Whether to use secure channel.') |
|
|
|
help='Whether to use secure channel.') |
|
|
|
argp.add_argument('--internal_ci', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--internal_ci', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help=('Put reports into subdirectories to improve ' |
|
|
|
help=('Put reports into subdirectories to improve ' |
|
|
|
'presentation of results by Internal CI.')) |
|
|
|
'presentation of results by Internal CI.')) |
|
|
|
argp.add_argument('--bq_result_table', |
|
|
|
argp.add_argument( |
|
|
|
|
|
|
|
'--bq_result_table', |
|
|
|
default='', |
|
|
|
default='', |
|
|
|
type=str, |
|
|
|
type=str, |
|
|
|
nargs='?', |
|
|
|
nargs='?', |
|
|
|
help='Upload test results to a specified BQ table.') |
|
|
|
help='Upload test results to a specified BQ table.') |
|
|
|
args = argp.parse_args() |
|
|
|
args = argp.parse_args() |
|
|
|
|
|
|
|
|
|
|
|
servers = set(s for s in itertools.chain.from_iterable(_SERVERS |
|
|
|
servers = set( |
|
|
|
if x == 'all' else [x] |
|
|
|
s |
|
|
|
|
|
|
|
for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] |
|
|
|
for x in args.server)) |
|
|
|
for x in args.server)) |
|
|
|
|
|
|
|
|
|
|
|
if args.use_docker: |
|
|
|
if args.use_docker: |
|
|
|
if not args.travis: |
|
|
|
if not args.travis: |
|
|
|
print('Seen --use_docker flag, will run interop tests under docker.') |
|
|
|
print('Seen --use_docker flag, will run interop tests under docker.') |
|
|
|
print('') |
|
|
|
print('') |
|
|
|
print('IMPORTANT: The changes you are testing need to be locally committed') |
|
|
|
print( |
|
|
|
print('because only the committed changes in the current branch will be') |
|
|
|
'IMPORTANT: The changes you are testing need to be locally committed' |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
print( |
|
|
|
|
|
|
|
'because only the committed changes in the current branch will be') |
|
|
|
print('copied to the docker environment.') |
|
|
|
print('copied to the docker environment.') |
|
|
|
time.sleep(5) |
|
|
|
time.sleep(5) |
|
|
|
|
|
|
|
|
|
|
|
@ -989,32 +1059,35 @@ if args.manual_run and not args.use_docker: |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
if not args.use_docker and servers: |
|
|
|
if not args.use_docker and servers: |
|
|
|
print('Running interop servers is only supported with --use_docker option enabled.') |
|
|
|
print( |
|
|
|
|
|
|
|
'Running interop servers is only supported with --use_docker option enabled.' |
|
|
|
|
|
|
|
) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# we want to include everything but objc in 'all' |
|
|
|
# we want to include everything but objc in 'all' |
|
|
|
# because objc won't run on non-mac platforms |
|
|
|
# because objc won't run on non-mac platforms |
|
|
|
all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) |
|
|
|
all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) |
|
|
|
languages = set(_LANGUAGES[l] |
|
|
|
languages = set( |
|
|
|
for l in itertools.chain.from_iterable( |
|
|
|
_LANGUAGES[l] |
|
|
|
all_but_objc if x == 'all' else [x] |
|
|
|
for l in itertools.chain.from_iterable(all_but_objc if x == 'all' else [x] |
|
|
|
for x in args.language)) |
|
|
|
for x in args.language)) |
|
|
|
|
|
|
|
|
|
|
|
languages_http2_clients_for_http2_server_interop = set() |
|
|
|
languages_http2_clients_for_http2_server_interop = set() |
|
|
|
if args.http2_server_interop: |
|
|
|
if args.http2_server_interop: |
|
|
|
languages_http2_clients_for_http2_server_interop = set( |
|
|
|
languages_http2_clients_for_http2_server_interop = set( |
|
|
|
_LANGUAGES[l] for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES |
|
|
|
_LANGUAGES[l] |
|
|
|
|
|
|
|
for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES |
|
|
|
if 'all' in args.language or l in args.language) |
|
|
|
if 'all' in args.language or l in args.language) |
|
|
|
|
|
|
|
|
|
|
|
http2Interop = Http2Client() if args.http2_interop else None |
|
|
|
http2Interop = Http2Client() if args.http2_interop else None |
|
|
|
http2InteropServer = Http2Server() if args.http2_server_interop else None |
|
|
|
http2InteropServer = Http2Server() if args.http2_server_interop else None |
|
|
|
|
|
|
|
|
|
|
|
docker_images={} |
|
|
|
docker_images = {} |
|
|
|
if args.use_docker: |
|
|
|
if args.use_docker: |
|
|
|
# languages for which to build docker images |
|
|
|
# languages for which to build docker images |
|
|
|
languages_to_build = set( |
|
|
|
languages_to_build = set( |
|
|
|
_LANGUAGES[k] for k in set([str(l) for l in languages] + [s for s in servers])) |
|
|
|
_LANGUAGES[k] |
|
|
|
|
|
|
|
for k in set([str(l) for l in languages] + [s for s in servers])) |
|
|
|
languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop |
|
|
|
languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop |
|
|
|
|
|
|
|
|
|
|
|
if args.http2_interop: |
|
|
|
if args.http2_interop: |
|
|
|
@ -1033,17 +1106,22 @@ if args.use_docker: |
|
|
|
build_jobs.append(job) |
|
|
|
build_jobs.append(job) |
|
|
|
|
|
|
|
|
|
|
|
if build_jobs: |
|
|
|
if build_jobs: |
|
|
|
jobset.message('START', 'Building interop docker images.', do_newline=True) |
|
|
|
jobset.message( |
|
|
|
|
|
|
|
'START', 'Building interop docker images.', do_newline=True) |
|
|
|
if args.verbose: |
|
|
|
if args.verbose: |
|
|
|
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) |
|
|
|
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) |
|
|
|
|
|
|
|
|
|
|
|
num_failures, _ = jobset.run( |
|
|
|
num_failures, _ = jobset.run( |
|
|
|
build_jobs, newline_on_success=True, maxjobs=args.jobs) |
|
|
|
build_jobs, newline_on_success=True, maxjobs=args.jobs) |
|
|
|
if num_failures == 0: |
|
|
|
if num_failures == 0: |
|
|
|
jobset.message('SUCCESS', 'All docker images built successfully.', |
|
|
|
jobset.message( |
|
|
|
|
|
|
|
'SUCCESS', |
|
|
|
|
|
|
|
'All docker images built successfully.', |
|
|
|
do_newline=True) |
|
|
|
do_newline=True) |
|
|
|
else: |
|
|
|
else: |
|
|
|
jobset.message('FAILED', 'Failed to build interop docker images.', |
|
|
|
jobset.message( |
|
|
|
|
|
|
|
'FAILED', |
|
|
|
|
|
|
|
'Failed to build interop docker images.', |
|
|
|
do_newline=True) |
|
|
|
do_newline=True) |
|
|
|
for image in six.itervalues(docker_images): |
|
|
|
for image in six.itervalues(docker_images): |
|
|
|
dockerjob.remove_image(image, skip_nonexistent=True) |
|
|
|
dockerjob.remove_image(image, skip_nonexistent=True) |
|
|
|
@ -1058,12 +1136,16 @@ server_addresses = {} |
|
|
|
try: |
|
|
|
try: |
|
|
|
for s in servers: |
|
|
|
for s in servers: |
|
|
|
lang = str(s) |
|
|
|
lang = str(s) |
|
|
|
spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), |
|
|
|
spec = server_jobspec( |
|
|
|
args.insecure, manual_cmd_log=server_manual_cmd_log) |
|
|
|
_LANGUAGES[lang], |
|
|
|
|
|
|
|
docker_images.get(lang), |
|
|
|
|
|
|
|
args.insecure, |
|
|
|
|
|
|
|
manual_cmd_log=server_manual_cmd_log) |
|
|
|
if not args.manual_run: |
|
|
|
if not args.manual_run: |
|
|
|
job = dockerjob.DockerJob(spec) |
|
|
|
job = dockerjob.DockerJob(spec) |
|
|
|
server_jobs[lang] = job |
|
|
|
server_jobs[lang] = job |
|
|
|
server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT)) |
|
|
|
server_addresses[lang] = ('localhost', |
|
|
|
|
|
|
|
job.mapped_port(_DEFAULT_SERVER_PORT)) |
|
|
|
else: |
|
|
|
else: |
|
|
|
# don't run the server, set server port to a placeholder value |
|
|
|
# don't run the server, set server port to a placeholder value |
|
|
|
server_addresses[lang] = ('localhost', '${SERVER_PORT}') |
|
|
|
server_addresses[lang] = ('localhost', '${SERVER_PORT}') |
|
|
|
@ -1072,7 +1154,9 @@ try: |
|
|
|
if args.http2_server_interop: |
|
|
|
if args.http2_server_interop: |
|
|
|
# launch a HTTP2 server emulator that creates edge cases |
|
|
|
# launch a HTTP2 server emulator that creates edge cases |
|
|
|
lang = str(http2InteropServer) |
|
|
|
lang = str(http2InteropServer) |
|
|
|
spec = server_jobspec(http2InteropServer, docker_images.get(lang), |
|
|
|
spec = server_jobspec( |
|
|
|
|
|
|
|
http2InteropServer, |
|
|
|
|
|
|
|
docker_images.get(lang), |
|
|
|
manual_cmd_log=server_manual_cmd_log) |
|
|
|
manual_cmd_log=server_manual_cmd_log) |
|
|
|
if not args.manual_run: |
|
|
|
if not args.manual_run: |
|
|
|
http2_server_job = dockerjob.DockerJob(spec) |
|
|
|
http2_server_job = dockerjob.DockerJob(spec) |
|
|
|
@ -1091,7 +1175,9 @@ try: |
|
|
|
if not test_case in language.unimplemented_test_cases(): |
|
|
|
if not test_case in language.unimplemented_test_cases(): |
|
|
|
if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: |
|
|
|
if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: |
|
|
|
test_job = cloud_to_prod_jobspec( |
|
|
|
test_job = cloud_to_prod_jobspec( |
|
|
|
language, test_case, server_host_name, |
|
|
|
language, |
|
|
|
|
|
|
|
test_case, |
|
|
|
|
|
|
|
server_host_name, |
|
|
|
prod_servers[server_host_name], |
|
|
|
prod_servers[server_host_name], |
|
|
|
docker_image=docker_images.get(str(language)), |
|
|
|
docker_image=docker_images.get(str(language)), |
|
|
|
manual_cmd_log=client_manual_cmd_log) |
|
|
|
manual_cmd_log=client_manual_cmd_log) |
|
|
|
@ -1100,7 +1186,9 @@ try: |
|
|
|
if args.http2_interop: |
|
|
|
if args.http2_interop: |
|
|
|
for test_case in _HTTP2_TEST_CASES: |
|
|
|
for test_case in _HTTP2_TEST_CASES: |
|
|
|
test_job = cloud_to_prod_jobspec( |
|
|
|
test_job = cloud_to_prod_jobspec( |
|
|
|
http2Interop, test_case, server_host_name, |
|
|
|
http2Interop, |
|
|
|
|
|
|
|
test_case, |
|
|
|
|
|
|
|
server_host_name, |
|
|
|
prod_servers[server_host_name], |
|
|
|
prod_servers[server_host_name], |
|
|
|
docker_image=docker_images.get(str(http2Interop)), |
|
|
|
docker_image=docker_images.get(str(http2Interop)), |
|
|
|
manual_cmd_log=client_manual_cmd_log) |
|
|
|
manual_cmd_log=client_manual_cmd_log) |
|
|
|
@ -1114,9 +1202,12 @@ try: |
|
|
|
for test_case in _AUTH_TEST_CASES: |
|
|
|
for test_case in _AUTH_TEST_CASES: |
|
|
|
if not test_case in language.unimplemented_test_cases(): |
|
|
|
if not test_case in language.unimplemented_test_cases(): |
|
|
|
test_job = cloud_to_prod_jobspec( |
|
|
|
test_job = cloud_to_prod_jobspec( |
|
|
|
language, test_case, server_host_name, |
|
|
|
language, |
|
|
|
|
|
|
|
test_case, |
|
|
|
|
|
|
|
server_host_name, |
|
|
|
prod_servers[server_host_name], |
|
|
|
prod_servers[server_host_name], |
|
|
|
docker_image=docker_images.get(str(language)), auth=True, |
|
|
|
docker_image=docker_images.get(str(language)), |
|
|
|
|
|
|
|
auth=True, |
|
|
|
manual_cmd_log=client_manual_cmd_log) |
|
|
|
manual_cmd_log=client_manual_cmd_log) |
|
|
|
jobs.append(test_job) |
|
|
|
jobs.append(test_job) |
|
|
|
|
|
|
|
|
|
|
|
@ -1135,7 +1226,8 @@ try: |
|
|
|
for test_case in _TEST_CASES: |
|
|
|
for test_case in _TEST_CASES: |
|
|
|
if not test_case in language.unimplemented_test_cases(): |
|
|
|
if not test_case in language.unimplemented_test_cases(): |
|
|
|
if not test_case in skip_server: |
|
|
|
if not test_case in skip_server: |
|
|
|
test_job = cloud_to_cloud_jobspec(language, |
|
|
|
test_job = cloud_to_cloud_jobspec( |
|
|
|
|
|
|
|
language, |
|
|
|
test_case, |
|
|
|
test_case, |
|
|
|
server_name, |
|
|
|
server_name, |
|
|
|
server_host, |
|
|
|
server_host, |
|
|
|
@ -1150,7 +1242,8 @@ try: |
|
|
|
if server_name == "go": |
|
|
|
if server_name == "go": |
|
|
|
# TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 |
|
|
|
# TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 |
|
|
|
continue |
|
|
|
continue |
|
|
|
test_job = cloud_to_cloud_jobspec(http2Interop, |
|
|
|
test_job = cloud_to_cloud_jobspec( |
|
|
|
|
|
|
|
http2Interop, |
|
|
|
test_case, |
|
|
|
test_case, |
|
|
|
server_name, |
|
|
|
server_name, |
|
|
|
server_host, |
|
|
|
server_host, |
|
|
|
@ -1164,12 +1257,14 @@ try: |
|
|
|
if not args.manual_run: |
|
|
|
if not args.manual_run: |
|
|
|
http2_server_job.wait_for_healthy(timeout_seconds=600) |
|
|
|
http2_server_job.wait_for_healthy(timeout_seconds=600) |
|
|
|
for language in languages_http2_clients_for_http2_server_interop: |
|
|
|
for language in languages_http2_clients_for_http2_server_interop: |
|
|
|
for test_case in set(_HTTP2_SERVER_TEST_CASES) - set(_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): |
|
|
|
for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( |
|
|
|
|
|
|
|
_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): |
|
|
|
offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
|
|
|
offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
|
|
|
server_port = _DEFAULT_SERVER_PORT+offset |
|
|
|
server_port = _DEFAULT_SERVER_PORT + offset |
|
|
|
if not args.manual_run: |
|
|
|
if not args.manual_run: |
|
|
|
server_port = http2_server_job.mapped_port(server_port) |
|
|
|
server_port = http2_server_job.mapped_port(server_port) |
|
|
|
test_job = cloud_to_cloud_jobspec(language, |
|
|
|
test_job = cloud_to_cloud_jobspec( |
|
|
|
|
|
|
|
language, |
|
|
|
test_case, |
|
|
|
test_case, |
|
|
|
str(http2InteropServer), |
|
|
|
str(http2InteropServer), |
|
|
|
'localhost', |
|
|
|
'localhost', |
|
|
|
@ -1187,13 +1282,16 @@ try: |
|
|
|
for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
|
|
|
for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
|
|
|
if test_case not in language.unimplemented_test_cases(): |
|
|
|
if test_case not in language.unimplemented_test_cases(): |
|
|
|
offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
|
|
|
offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
|
|
|
server_port = _DEFAULT_SERVER_PORT+offset |
|
|
|
server_port = _DEFAULT_SERVER_PORT + offset |
|
|
|
if not args.manual_run: |
|
|
|
if not args.manual_run: |
|
|
|
server_port = http2_server_job.mapped_port(server_port) |
|
|
|
server_port = http2_server_job.mapped_port(server_port) |
|
|
|
if not args.insecure: |
|
|
|
if not args.insecure: |
|
|
|
print(('Creating grpc cient to http2 server test case with insecure connection, even though' |
|
|
|
print(( |
|
|
|
' args.insecure is False. Http2 test server only supports insecure connections.')) |
|
|
|
'Creating grpc cient to http2 server test case with insecure connection, even though' |
|
|
|
test_job = cloud_to_cloud_jobspec(language, |
|
|
|
' args.insecure is False. Http2 test server only supports insecure connections.' |
|
|
|
|
|
|
|
)) |
|
|
|
|
|
|
|
test_job = cloud_to_cloud_jobspec( |
|
|
|
|
|
|
|
language, |
|
|
|
test_case, |
|
|
|
test_case, |
|
|
|
str(http2InteropServer), |
|
|
|
str(http2InteropServer), |
|
|
|
'localhost', |
|
|
|
'localhost', |
|
|
|
@ -1215,7 +1313,9 @@ try: |
|
|
|
if args.verbose: |
|
|
|
if args.verbose: |
|
|
|
print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) |
|
|
|
print('Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) |
|
|
|
|
|
|
|
|
|
|
|
num_failures, resultset = jobset.run(jobs, newline_on_success=True, |
|
|
|
num_failures, resultset = jobset.run( |
|
|
|
|
|
|
|
jobs, |
|
|
|
|
|
|
|
newline_on_success=True, |
|
|
|
maxjobs=args.jobs, |
|
|
|
maxjobs=args.jobs, |
|
|
|
skip_jobs=args.manual_run) |
|
|
|
skip_jobs=args.manual_run) |
|
|
|
if args.bq_result_table and resultset: |
|
|
|
if args.bq_result_table and resultset: |
|
|
|
@ -1237,14 +1337,14 @@ try: |
|
|
|
if "http2" in name: |
|
|
|
if "http2" in name: |
|
|
|
job[0].http2results = aggregate_http2_results(job[0].message) |
|
|
|
job[0].http2results = aggregate_http2_results(job[0].message) |
|
|
|
|
|
|
|
|
|
|
|
http2_server_test_cases = ( |
|
|
|
http2_server_test_cases = (_HTTP2_SERVER_TEST_CASES |
|
|
|
_HTTP2_SERVER_TEST_CASES if args.http2_server_interop else []) |
|
|
|
if args.http2_server_interop else []) |
|
|
|
|
|
|
|
|
|
|
|
report_utils.render_interop_html_report( |
|
|
|
report_utils.render_interop_html_report( |
|
|
|
set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, |
|
|
|
set([str(l) for l in languages]), servers, _TEST_CASES, |
|
|
|
_HTTP2_TEST_CASES, http2_server_test_cases, resultset, num_failures, |
|
|
|
_AUTH_TEST_CASES, _HTTP2_TEST_CASES, http2_server_test_cases, resultset, |
|
|
|
args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers, |
|
|
|
num_failures, args.cloud_to_prod_auth or args.cloud_to_prod, |
|
|
|
args.http2_interop) |
|
|
|
args.prod_servers, args.http2_interop) |
|
|
|
|
|
|
|
|
|
|
|
if num_failures: |
|
|
|
if num_failures: |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
ncteisen
7 years ago