Add a grpclb-in-DNS interop test suite

templates/tools/dockerfile/java_build_interop.sh.include

@@ -25,3 +25,11 @@ cp -r /var/local/jenkins/service_account $HOME || true
 cd /var/local/git/grpc-java
 
 ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true
+
+# enable extra java logging
+mkdir -p /var/local/grpc_java_logging
+echo "handlers = java.util.logging.ConsoleHandler
+java.util.logging.ConsoleHandler.level = ALL
+.level = FINE
+io.grpc.netty.NettyClientHandler = ALL
+io.grpc.netty.NettyServerHandler = ALL" > /var/local/grpc_java_logging/logconf.txt

templates/tools/run_tests/generated/lb_interop_test_scenarios.json.template

@@ -0,0 +1,6 @@
+%YAML 1.2
+--- |
+  <%!
+  import json
+  %>
+  ${json.dumps(lb_interop_test_scenarios, indent=4, sort_keys=True)}

test/cpp/naming/utils/dns_server.py

@@ -93,6 +93,10 @@ def start_local_dns_server(args):
           _push_record(record_full_name, dns.Record_SRV(p, w, port, target_full_name, ttl=r_ttl))
         if r_type == 'TXT':
           _maybe_split_up_txt_data(record_full_name, r_data, r_ttl)
+  # Add an optional IPv4 record is specified
+  if args.add_a_record:
+    extra_host, extra_host_ipv4 = args.add_a_record.split(':')
+    _push_record(extra_host, dns.Record_A(extra_host_ipv4, ttl=0))
   # Server health check record
   _push_record(_SERVER_HEALTH_CHECK_RECORD_NAME, dns.Record_A(_SERVER_HEALTH_CHECK_RECORD_DATA, ttl=0))
   soa_record = dns.Record_SOA(mname = common_zone_name)
@@ -122,7 +126,7 @@ def flush_stdout_loop():
   num_timeouts_so_far = 0
   sleep_time = 1
   # Prevent zombies. Tests that use this server are short-lived.
-  max_timeouts = 60 * 2
+  max_timeouts = 60 * 10
   while num_timeouts_so_far < max_timeouts:
     sys.stdout.flush()
     time.sleep(sleep_time)
@@ -136,7 +140,14 @@ def main():
                     help='Port for DNS server to listen on for TCP and UDP.')
   argp.add_argument('-r', '--records_config_path', default=None, type=str,
                     help=('Directory of resolver_test_record_groups.yaml file. '
-                          'Defauls to path needed when the test is invoked as part of run_tests.py.'))
+                          'Defaults to path needed when the test is invoked as part '
+                          'of run_tests.py.'))
+  argp.add_argument('--add_a_record', default=None, type=str,
+                    help=('Add an A record via the command line. Useful for when we '
+                          'need to serve a one-off A record that is under a '
+                          'different domain then the rest the records configured in '
+                          '--records_config_path (which all need to be under the '
+                          'same domain). Format: <name>:<ipv4 address>'))
   args = argp.parse_args()
   signal.signal(signal.SIGTERM, _quit_on_signal)
   signal.signal(signal.SIGINT, _quit_on_signal)

test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python2.7
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import subprocess
+import os
+import tempfile
+import sys
+import time
+import signal
+import yaml
+
+argp = argparse.ArgumentParser(description='Runs a DNS server for LB interop tests')
+argp.add_argument('-l', '--grpclb_ips', default=None, type=str,
+                  help='Comma-separated list of IP addresses of balancers')
+argp.add_argument('-f', '--fallback_ips', default=None, type=str,
+                  help='Comma-separated list of IP addresses of fallback servers')
+argp.add_argument('-c', '--cause_no_error_no_data_for_balancer_a_record',
+                  default=False, action='store_const', const=True,
+                  help=('Used for testing the case in which the grpclb '
+                        'balancer A record lookup results in a DNS NOERROR response '
+                        'but with no ANSWER section i.e. no addresses'))
+args = argp.parse_args()
+
+balancer_records = []
+grpclb_ips = args.grpclb_ips.split(',')
+if grpclb_ips[0]:
+    for ip in grpclb_ips:
+        balancer_records.append({
+            'TTL': '2100',
+            'data': ip,
+            'type': 'A',
+        })
+fallback_records = []
+fallback_ips = args.fallback_ips.split(',')
+if fallback_ips[0]:
+    for ip in fallback_ips:
+        fallback_records.append({
+            'TTL': '2100',
+            'data': ip,
+            'type': 'A',
+        })
+records_config_yaml = {
+    'resolver_tests_common_zone_name':
+    'test.google.fr.',
+    'resolver_component_tests': [{
+        'records': {
+            '_grpclb._tcp.server': [
+                {
+                    'TTL': '2100',
+                    'data': '0 0 12000 balancer',
+                    'type': 'SRV'
+                },
+            ],
+            'balancer':
+            balancer_records,
+            'server':
+            fallback_records,
+        }
+    }]
+}
+if args.cause_no_error_no_data_for_balancer_a_record:
+    balancer_records = records_config_yaml[
+        'resolver_component_tests'][0]['records']['balancer']
+    assert not balancer_records
+    # Insert a TXT record at the balancer.test.google.fr. domain.
+    # This TXT record won't actually be resolved or used by gRPC clients;
+    # inserting this record is just a way get the balancer.test.google.fr.
+    # A record queries to return NOERROR DNS responses that also have no
+    # ANSWER section, in order to simulate this failure case.
+    balancer_records.append({
+        'TTL': '2100',
+        'data': 'arbitrary string that wont actually be resolved',
+        'type': 'TXT',
+    })
+# Generate the actual DNS server records config file
+records_config_path = tempfile.mktemp()
+with open(records_config_path, 'w') as records_config_generated:
+    records_config_generated.write(yaml.dump(records_config_yaml))
+
+with open(records_config_path, 'r') as records_config_generated:
+    sys.stderr.write('===== DNS server records config: =====\n')
+    sys.stderr.write(records_config_generated.read())
+    sys.stderr.write('======================================\n')
+
+# Run the DNS server
+# Note that we need to add the extra
+# A record for metadata.google.internal in order for compute engine
+# OAuth creds and ALTS creds to work.
+# TODO(apolcyn): should metadata.google.internal always resolve
+# to 169.254.169.254?
+subprocess.check_output([
+    '/var/local/git/grpc/test/cpp/naming/utils/dns_server.py', '--port=53',
+    '--records_config_path', records_config_path,
+    '--add_a_record=metadata.google.internal:169.254.169.254',
+])

test/cpp/naming/utils/tcp_connect.py

@@ -31,7 +31,8 @@ def main():
   argp.add_argument('-t', '--timeout', default=1, type=int,
                     help='Force process exit after this number of seconds.')
   args = argp.parse_args()
-  socket.create_connection([args.server_host, args.server_port])
+  socket.create_connection([args.server_host, args.server_port],
+                           timeout=args.timeout)
 
 if __name__ == '__main__':
   main()

tools/buildgen/generate_build_additions.sh

@@ -25,7 +25,8 @@ gen_build_yaml_dirs="  \
   test/core/bad_ssl    \
   test/core/end2end    \
   test/cpp/naming \
-  test/cpp/qps"
+  test/cpp/qps \
+  tools/run_tests/lb_interop_tests"
 gen_build_files=""
 for gen_build_yaml in $gen_build_yaml_dirs
 do

tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh

@@ -31,7 +31,7 @@ cd /var/local/git/grpc
 make install-certs
 
 # build C++ interop client & server
-make interop_client interop_server
+make interop_client interop_server -j2
 
 # build C++ http2 client
 make http2_client

tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh

@@ -26,3 +26,11 @@ cd /var/local/git/grpc-java
 
 ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true
 
+# enable extra java logging
+mkdir -p /var/local/grpc_java_logging
+echo "handlers = java.util.logging.ConsoleHandler
+java.util.logging.ConsoleHandler.level = ALL
+.level = FINE
+io.grpc.netty.NettyClientHandler = ALL
+io.grpc.netty.NettyServerHandler = ALL" > /var/local/grpc_java_logging/logconf.txt
+

tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh

@@ -26,3 +26,11 @@ cd /var/local/git/grpc-java
 
 ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true
 
+# enable extra java logging
+mkdir -p /var/local/grpc_java_logging
+echo "handlers = java.util.logging.ConsoleHandler
+java.util.logging.ConsoleHandler.level = ALL
+.level = FINE
+io.grpc.netty.NettyClientHandler = ALL
+io.grpc.netty.NettyServerHandler = ALL" > /var/local/grpc_java_logging/logconf.txt
+  

tools/dockerfile/interoptest/lb_interop_fake_servers/Dockerfile

@@ -0,0 +1,34 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.10
+
+RUN apt-get update && apt-get install -y \
+  dnsutils \
+  git \
+  vim \
+  curl \
+  python-pip \
+  python-yaml \
+  make && apt-get clean
+
+RUN ln -s /usr/local/go/bin/go /usr/local/bin
+
+# Install Python packages from PyPI
+RUN pip install --upgrade pip==10.0.1
+RUN pip install virtualenv
+RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/interoptest/lb_interop_fake_servers/build_interop.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Gets a built Go interop server, fake balancer server, and python
+# DNS server into a base image.
+set -e
+
+# Clone just the grpc-go source code without any dependencies.
+# We are cloning from a local git repo that contains the right revision
+# to test instead of using "go get" to download from Github directly.
+git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
+
+# Get all gRPC Go dependencies
+(cd src/google.golang.org/grpc && make deps && make testdeps)
+
+# Build the interop server and fake balancer
+(cd src/google.golang.org/grpc/interop/server && go install)
+(cd src/google.golang.org/grpc/interop/fake_grpclb && go install)
+  
+# Clone the grpc/grpc repo to get the python DNS server.
+# Hack: we don't need to init submodules for the scripts we need.
+mkdir -p /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc

tools/internal_ci/helper_scripts/prepare_build_grpclb_interop_rc

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Source this rc script to prepare the environment for interop builds
+# This rc script must be used in the root directory of gRPC
+
+export LANG=en_US.UTF-8
+
+# Download Docker images from DockerHub
+export DOCKERHUB_ORGANIZATION=grpctesting
+
+git submodule update --init
+
+# Set up gRPC-Go and gRPC-Java to test
+git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go
+git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java
+
+# TODO(apolcyn): move to kokoro image?
+virtualenv env
+source env/bin/activate
+pip install twisted

tools/internal_ci/linux/grpc_run_grpclb_interop_tests.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+export LANG=en_US.UTF-8
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+source tools/internal_ci/helper_scripts/prepare_build_grpclb_interop_rc
+
+tools/run_tests/run_grpclb_interop_tests.py -l all --scenarios_file=tools/run_tests/generated/lb_interop_test_scenarios.json

tools/internal_ci/linux/grpclb_in_dns_interop.cfg

@@ -0,0 +1,25 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_run_grpclb_interop_tests.sh"
+timeout_mins: 60
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+    regex: "github/grpc/reports/**"
+  }
+}

tools/run_tests/lb_interop_tests/gen_build_yaml.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python2.7
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Generates the appropriate JSON data for LB interop test scenarios."""
+
+import json
+import os
+import yaml
+
+all_scenarios = []
+
+# TODO(https://github.com/grpc/grpc-go/issues/2347): enable
+# client_falls_back_because_no_backends_* scenarios for Java/Go.
+
+# TODO(https://github.com/grpc/grpc-java/issues/4887): enable
+# *short_stream* scenarios for Java.
+
+# TODO(https://github.com/grpc/grpc-java/issues/4912): enable
+# Java TLS tests involving TLS to the balancer.
+
+
+def server_sec(transport_sec):
+    if transport_sec == 'google_default_credentials':
+        return 'alts', 'alts', 'tls'
+    return transport_sec, transport_sec, transport_sec
+
+
+def generate_no_balancer_because_lb_a_record_returns_nx_domain():
+    all_configs = []
+    for transport_sec in [
+            'insecure', 'alts', 'tls', 'google_default_credentials'
+    ]:
+        balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+        config = {
+            'name':
+            'no_balancer_because_lb_a_record_returns_nx_domain_%s' %
+            transport_sec,
+            'skip_langs': [],
+            'transport_sec':
+            transport_sec,
+            'balancer_configs': [],
+            'backend_configs': [],
+            'fallback_configs': [{
+                'transport_sec': fallback_sec,
+            }],
+            'cause_no_error_no_data_for_balancer_a_record':
+            False,
+        }
+        all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_no_balancer_because_lb_a_record_returns_nx_domain()
+
+
+def generate_no_balancer_because_lb_a_record_returns_no_data():
+    all_configs = []
+    for transport_sec in [
+            'insecure', 'alts', 'tls', 'google_default_credentials'
+    ]:
+        balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+        config = {
+            'name':
+            'no_balancer_because_lb_a_record_returns_no_data_%s' %
+            transport_sec,
+            'skip_langs': [],
+            'transport_sec':
+            transport_sec,
+            'balancer_configs': [],
+            'backend_configs': [],
+            'fallback_configs': [{
+                'transport_sec': fallback_sec,
+            }],
+            'cause_no_error_no_data_for_balancer_a_record':
+            True,
+        }
+        all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_no_balancer_because_lb_a_record_returns_no_data()
+
+
+def generate_client_referred_to_backend():
+    all_configs = []
+    for balancer_short_stream in [True, False]:
+        for transport_sec in [
+                'insecure', 'alts', 'tls', 'google_default_credentials'
+        ]:
+            balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+            skip_langs = []
+            if transport_sec == 'tls':
+                skip_langs += ['java']
+            if balancer_short_stream:
+                skip_langs += ['java']
+            config = {
+                'name':
+                'client_referred_to_backend_%s_short_stream_%s' %
+                (transport_sec, balancer_short_stream),
+                'skip_langs':
+                skip_langs,
+                'transport_sec':
+                transport_sec,
+                'balancer_configs': [{
+                    'transport_sec': balancer_sec,
+                    'short_stream': balancer_short_stream,
+                }],
+                'backend_configs': [{
+                    'transport_sec': backend_sec,
+                }],
+                'fallback_configs': [],
+                'cause_no_error_no_data_for_balancer_a_record':
+                False,
+            }
+            all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_client_referred_to_backend()
+
+
+def generate_client_referred_to_backend_fallback_broken():
+    all_configs = []
+    for balancer_short_stream in [True, False]:
+        for transport_sec in ['alts', 'tls', 'google_default_credentials']:
+            balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+            skip_langs = []
+            if transport_sec == 'tls':
+                skip_langs += ['java']
+            if balancer_short_stream:
+                skip_langs += ['java']
+            config = {
+                'name':
+                'client_referred_to_backend_fallback_broken_%s_short_stream_%s'
+                % (transport_sec, balancer_short_stream),
+                'skip_langs':
+                skip_langs,
+                'transport_sec':
+                transport_sec,
+                'balancer_configs': [{
+                    'transport_sec': balancer_sec,
+                    'short_stream': balancer_short_stream,
+                }],
+                'backend_configs': [{
+                    'transport_sec': backend_sec,
+                }],
+                'fallback_configs': [{
+                    'transport_sec': 'insecure',
+                }],
+                'cause_no_error_no_data_for_balancer_a_record':
+                False,
+            }
+            all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_client_referred_to_backend_fallback_broken()
+
+
+def generate_client_referred_to_backend_multiple_backends():
+    all_configs = []
+    for balancer_short_stream in [True, False]:
+        for transport_sec in [
+                'insecure', 'alts', 'tls', 'google_default_credentials'
+        ]:
+            balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+            skip_langs = []
+            if transport_sec == 'tls':
+                skip_langs += ['java']
+            if balancer_short_stream:
+                skip_langs += ['java']
+            config = {
+                'name':
+                'client_referred_to_backend_multiple_backends_%s_short_stream_%s'
+                % (transport_sec, balancer_short_stream),
+                'skip_langs':
+                skip_langs,
+                'transport_sec':
+                transport_sec,
+                'balancer_configs': [{
+                    'transport_sec': balancer_sec,
+                    'short_stream': balancer_short_stream,
+                }],
+                'backend_configs': [{
+                    'transport_sec': backend_sec,
+                }, {
+                    'transport_sec': backend_sec,
+                }, {
+                    'transport_sec': backend_sec,
+                }, {
+                    'transport_sec': backend_sec,
+                }, {
+                    'transport_sec': backend_sec,
+                }],
+                'fallback_configs': [],
+                'cause_no_error_no_data_for_balancer_a_record':
+                False,
+            }
+            all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_client_referred_to_backend_multiple_backends()
+
+
+def generate_client_falls_back_because_no_backends():
+    all_configs = []
+    for balancer_short_stream in [True, False]:
+        for transport_sec in [
+                'insecure', 'alts', 'tls', 'google_default_credentials'
+        ]:
+            balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+            skip_langs = ['go', 'java']
+            if transport_sec == 'tls':
+                skip_langs += ['java']
+            if balancer_short_stream:
+                skip_langs += ['java']
+            config = {
+                'name':
+                'client_falls_back_because_no_backends_%s_short_stream_%s' %
+                (transport_sec, balancer_short_stream),
+                'skip_langs':
+                skip_langs,
+                'transport_sec':
+                transport_sec,
+                'balancer_configs': [{
+                    'transport_sec': balancer_sec,
+                    'short_stream': balancer_short_stream,
+                }],
+                'backend_configs': [],
+                'fallback_configs': [{
+                    'transport_sec': fallback_sec,
+                }],
+                'cause_no_error_no_data_for_balancer_a_record':
+                False,
+            }
+            all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_client_falls_back_because_no_backends()
+
+
+def generate_client_falls_back_because_balancer_connection_broken():
+    all_configs = []
+    for transport_sec in ['alts', 'tls', 'google_default_credentials']:
+        balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+        skip_langs = []
+        if transport_sec == 'tls':
+            skip_langs = ['java']
+        config = {
+            'name':
+            'client_falls_back_because_balancer_connection_broken_%s' %
+            transport_sec,
+            'skip_langs':
+            skip_langs,
+            'transport_sec':
+            transport_sec,
+            'balancer_configs': [{
+                'transport_sec': 'insecure',
+                'short_stream': False,
+            }],
+            'backend_configs': [],
+            'fallback_configs': [{
+                'transport_sec': fallback_sec,
+            }],
+            'cause_no_error_no_data_for_balancer_a_record':
+            False,
+        }
+        all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_client_falls_back_because_balancer_connection_broken()
+
+
+def generate_client_referred_to_backend_multiple_balancers():
+    all_configs = []
+    for balancer_short_stream in [True, False]:
+        for transport_sec in [
+                'insecure', 'alts', 'tls', 'google_default_credentials'
+        ]:
+            balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec)
+            skip_langs = []
+            if transport_sec == 'tls':
+                skip_langs += ['java']
+            if balancer_short_stream:
+                skip_langs += ['java']
+            config = {
+                'name':
+                'client_referred_to_backend_multiple_balancers_%s_short_stream_%s'
+                % (transport_sec, balancer_short_stream),
+                'skip_langs':
+                skip_langs,
+                'transport_sec':
+                transport_sec,
+                'balancer_configs': [
+                    {
+                        'transport_sec': balancer_sec,
+                        'short_stream': balancer_short_stream,
+                    },
+                    {
+                        'transport_sec': balancer_sec,
+                        'short_stream': balancer_short_stream,
+                    },
+                    {
+                        'transport_sec': balancer_sec,
+                        'short_stream': balancer_short_stream,
+                    },
+                    {
+                        'transport_sec': balancer_sec,
+                        'short_stream': balancer_short_stream,
+                    },
+                    {
+                        'transport_sec': balancer_sec,
+                        'short_stream': balancer_short_stream,
+                    },
+                ],
+                'backend_configs': [
+                    {
+                        'transport_sec': backend_sec,
+                    },
+                ],
+                'fallback_configs': [],
+                'cause_no_error_no_data_for_balancer_a_record':
+                False,
+            }
+            all_configs.append(config)
+    return all_configs
+
+
+all_scenarios += generate_client_referred_to_backend_multiple_balancers()
+
+print(yaml.dump({
+    'lb_interop_test_scenarios': all_scenarios,
+}))

tools/run_tests/python_utils/dockerjob.py

@@ -20,6 +20,7 @@ import time
 import uuid
 import os
 import subprocess
+import json
 
 import jobset
 
@@ -54,6 +55,25 @@ def docker_mapped_port(cid, port, timeout_seconds=15):
                                                                          cid))
 
 
+def docker_ip_address(cid, timeout_seconds=15):
+    """Get port mapped to internal given internal port for given container."""
+    started = time.time()
+    while time.time() - started < timeout_seconds:
+        cmd = 'docker inspect %s' % cid
+        try:
+            output = subprocess.check_output(cmd, stderr=_DEVNULL, shell=True)
+            json_info = json.loads(output)
+            assert len(json_info) == 1
+            out = json_info[0]['NetworkSettings']['IPAddress']
+            if not out:
+                continue
+            return out
+        except subprocess.CalledProcessError as e:
+            pass
+    raise Exception(
+        'Non-retryable error: Failed to get ip address of container %s.' % cid)
+
+
 def wait_for_healthy(cid, shortname, timeout_seconds):
     """Wait timeout_seconds for the container to become healthy"""
     started = time.time()
@@ -74,10 +94,10 @@ def wait_for_healthy(cid, shortname, timeout_seconds):
                     (shortname, cid))
 
 
-def finish_jobs(jobs):
+def finish_jobs(jobs, suppress_failure=True):
     """Kills given docker containers and waits for corresponding jobs to finish"""
     for job in jobs:
-        job.kill(suppress_failure=True)
+        job.kill(suppress_failure=suppress_failure)
 
     while any(job.is_running() for job in jobs):
         time.sleep(1)
@@ -120,6 +140,9 @@ class DockerJob:
     def mapped_port(self, port):
         return docker_mapped_port(self._container_name, port)
 
+    def ip_address(self):
+        return docker_ip_address(self._container_name)
+
     def wait_for_healthy(self, timeout_seconds):
         wait_for_healthy(self._container_name, self._spec.shortname,
                          timeout_seconds)

tools/run_tests/run_grpclb_interop_tests.py

@@ -0,0 +1,609 @@
+#!/usr/bin/env python
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Run interop (cross-language) tests in parallel."""
+
+from __future__ import print_function
+
+import argparse
+import atexit
+import itertools
+import json
+import multiprocessing
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+import six
+import traceback
+
+import python_utils.dockerjob as dockerjob
+import python_utils.jobset as jobset
+import python_utils.report_utils as report_utils
+
+# Docker doesn't clean up after itself, so we do it on exit.
+atexit.register(lambda: subprocess.call(['stty', 'echo']))
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+os.chdir(ROOT)
+
+_FALLBACK_SERVER_PORT = 443
+_BALANCER_SERVER_PORT = 12000
+_BACKEND_SERVER_PORT = 8080
+
+_TEST_TIMEOUT = 30
+
+_FAKE_SERVERS_SAFENAME = 'fake_servers'
+
+# Use a name that's verified by the test certs
+_SERVICE_NAME = 'server.test.google.fr'
+
+
+class CXXLanguage:
+
+    def __init__(self):
+        self.client_cwd = '/var/local/git/grpc'
+        self.safename = 'cxx'
+
+    def client_cmd(self, args):
+        return ['bins/opt/interop_client'] + args
+
+    def global_env(self):
+        # 1) Set c-ares as the resolver, to
+        #    enable grpclb.
+        # 2) Turn on verbose logging.
+        # 3) Set the ROOTS_PATH env variable
+        #    to the test CA in order for
+        #    GoogleDefaultCredentials to be
+        #    able to use the test CA.
+        return {
+            'GRPC_DNS_RESOLVER':
+            'ares',
+            'GRPC_VERBOSITY':
+            'DEBUG',
+            'GRPC_TRACE':
+            'client_channel,glb',
+            'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
+            '/var/local/git/grpc/src/core/tsi/test_creds/ca.pem',
+        }
+
+    def __str__(self):
+        return 'c++'
+
+
+class JavaLanguage:
+
+    def __init__(self):
+        self.client_cwd = '/var/local/git/grpc-java'
+        self.safename = str(self)
+
+    def client_cmd(self, args):
+        # Take necessary steps to import our test CA into
+        # the set of test CA's that the Java runtime of the
+        # docker container will pick up, so that
+        # Java GoogleDefaultCreds can use it.
+        pem_to_der_cmd = ('openssl x509 -outform der '
+                          '-in /external_mount/src/core/tsi/test_creds/ca.pem '
+                          '-out /tmp/test_ca.der')
+        keystore_import_cmd = (
+            'keytool -import '
+            '-keystore /usr/lib/jvm/java-8-oracle/jre/lib/security/cacerts '
+            '-file /tmp/test_ca.der '
+            '-deststorepass changeit '
+            '-noprompt')
+        return [
+            'bash', '-c', ('{pem_to_der_cmd} && '
+                           '{keystore_import_cmd} && '
+                           './run-test-client.sh {java_client_args}').format(
+                               pem_to_der_cmd=pem_to_der_cmd,
+                               keystore_import_cmd=keystore_import_cmd,
+                               java_client_args=' '.join(args))
+        ]
+
+    def global_env(self):
+        # 1) Enable grpclb
+        # 2) Enable verbose logging
+        return {
+            'JAVA_OPTS':
+            ('-Dio.grpc.internal.DnsNameResolverProvider.enable_grpclb=true '
+             '-Djava.util.logging.config.file=/var/local/grpc_java_logging/logconf.txt'
+            )
+        }
+
+    def __str__(self):
+        return 'java'
+
+
+class GoLanguage:
+
+    def __init__(self):
+        self.client_cwd = '/go/src/google.golang.org/grpc/interop/client'
+        self.safename = str(self)
+
+    def client_cmd(self, args):
+        # Copy the test CA file into the path that
+        # the Go runtime in the docker container will use, so
+        # that Go's GoogleDefaultCredentials can use it.
+        # See https://golang.org/src/crypto/x509/root_linux.go.
+        return [
+            'bash', '-c', ('cp /external_mount/src/core/tsi/test_creds/ca.pem '
+                           '/etc/ssl/certs/ca-certificates.crt && '
+                           '/go/bin/client {go_client_args}'
+                          ).format(go_client_args=' '.join(args))
+        ]
+
+    def global_env(self):
+        return {
+            'GRPC_GO_LOG_VERBOSITY_LEVEL': '3',
+            'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO'
+        }
+
+    def __str__(self):
+        return 'go'
+
+
+_LANGUAGES = {
+    'c++': CXXLanguage(),
+    'go': GoLanguage(),
+    'java': JavaLanguage(),
+}
+
+
+def docker_run_cmdline(cmdline, image, docker_args, cwd, environ=None):
+    """Wraps given cmdline array to create 'docker run' cmdline from it."""
+    # turn environ into -e docker args
+    docker_cmdline = 'docker run -i --rm=true'.split()
+    if environ:
+        for k, v in environ.items():
+            docker_cmdline += ['-e', '%s=%s' % (k, v)]
+    return docker_cmdline + ['-w', cwd] + docker_args + [image] + cmdline
+
+
+def _job_kill_handler(job):
+    assert job._spec.container_name
+    dockerjob.docker_kill(job._spec.container_name)
+
+
+def transport_security_to_args(transport_security):
+    args = []
+    if transport_security == 'tls':
+        args += ['--use_tls=true']
+    elif transport_security == 'alts':
+        args += ['--use_tls=false', '--use_alts=true']
+    elif transport_security == 'insecure':
+        args += ['--use_tls=false']
+    elif transport_security == 'google_default_credentials':
+        args += ['--custom_credentials_type=google_default_credentials']
+    else:
+        print('Invalid transport security option.')
+        sys.exit(1)
+    return args
+
+
+def lb_client_interop_jobspec(language,
+                              dns_server_ip,
+                              docker_image,
+                              transport_security='tls'):
+    """Runs a gRPC client under test in a docker container"""
+    interop_only_options = [
+        '--server_host=%s' % _SERVICE_NAME,
+        '--server_port=%d' % _FALLBACK_SERVER_PORT
+    ] + transport_security_to_args(transport_security)
+    # Don't set the server host override in any client;
+    # Go and Java default to no override.
+    # We're using a DNS server so there's no need.
+    if language.safename == 'c++':
+        interop_only_options += ['--server_host_override=""']
+    # Don't set --use_test_ca; we're configuring
+    # clients to use test CA's via alternate means.
+    interop_only_options += ['--use_test_ca=false']
+    client_args = language.client_cmd(interop_only_options)
+    container_name = dockerjob.random_name(
+        'lb_interop_client_%s' % language.safename)
+    docker_cmdline = docker_run_cmdline(
+        client_args,
+        environ=language.global_env(),
+        image=docker_image,
+        cwd=language.client_cwd,
+        docker_args=[
+            '--dns=%s' % dns_server_ip,
+            '--net=host',
+            '--name=%s' % container_name,
+            '-v',
+            '{grpc_grpc_root_dir}:/external_mount:ro'.format(
+                grpc_grpc_root_dir=ROOT),
+        ])
+    jobset.message(
+        'IDLE',
+        'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
+        do_newline=True)
+    test_job = jobset.JobSpec(
+        cmdline=docker_cmdline,
+        shortname=('lb_interop_client:%s' % language),
+        timeout_seconds=_TEST_TIMEOUT,
+        kill_handler=_job_kill_handler)
+    test_job.container_name = container_name
+    return test_job
+
+
+def fallback_server_jobspec(transport_security, shortname):
+    """Create jobspec for running a fallback server"""
+    cmdline = [
+        'bin/server',
+        '--port=%d' % _FALLBACK_SERVER_PORT,
+    ] + transport_security_to_args(transport_security)
+    return grpc_server_in_docker_jobspec(
+        server_cmdline=cmdline, shortname=shortname)
+
+
+def backend_server_jobspec(transport_security, shortname):
+    """Create jobspec for running a backend server"""
+    cmdline = [
+        'bin/server',
+        '--port=%d' % _BACKEND_SERVER_PORT,
+    ] + transport_security_to_args(transport_security)
+    return grpc_server_in_docker_jobspec(
+        server_cmdline=cmdline, shortname=shortname)
+
+
+def grpclb_jobspec(transport_security, short_stream, backend_addrs, shortname):
+    """Create jobspec for running a balancer server"""
+    cmdline = [
+        'bin/fake_grpclb',
+        '--backend_addrs=%s' % ','.join(backend_addrs),
+        '--port=%d' % _BALANCER_SERVER_PORT,
+        '--short_stream=%s' % short_stream,
+        '--service_name=%s' % _SERVICE_NAME,
+    ] + transport_security_to_args(transport_security)
+    return grpc_server_in_docker_jobspec(
+        server_cmdline=cmdline, shortname=shortname)
+
+
+def grpc_server_in_docker_jobspec(server_cmdline, shortname):
+    container_name = dockerjob.random_name(shortname)
+    environ = {
+        'GRPC_GO_LOG_VERBOSITY_LEVEL': '3',
+        'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO ',
+    }
+    docker_cmdline = docker_run_cmdline(
+        server_cmdline,
+        cwd='/go',
+        image=docker_images.get(_FAKE_SERVERS_SAFENAME),
+        environ=environ,
+        docker_args=['--name=%s' % container_name])
+    jobset.message(
+        'IDLE',
+        'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
+        do_newline=True)
+    server_job = jobset.JobSpec(
+        cmdline=docker_cmdline, shortname=shortname, timeout_seconds=30 * 60)
+    server_job.container_name = container_name
+    return server_job
+
+
+def dns_server_in_docker_jobspec(grpclb_ips, fallback_ips, shortname,
+                                 cause_no_error_no_data_for_balancer_a_record):
+    container_name = dockerjob.random_name(shortname)
+    run_dns_server_cmdline = [
+        'python',
+        'test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py',
+        '--grpclb_ips=%s' % ','.join(grpclb_ips),
+        '--fallback_ips=%s' % ','.join(fallback_ips),
+    ]
+    if cause_no_error_no_data_for_balancer_a_record:
+        run_dns_server_cmdline.append(
+            '--cause_no_error_no_data_for_balancer_a_record')
+    docker_cmdline = docker_run_cmdline(
+        run_dns_server_cmdline,
+        cwd='/var/local/git/grpc',
+        image=docker_images.get(_FAKE_SERVERS_SAFENAME),
+        docker_args=['--name=%s' % container_name])
+    jobset.message(
+        'IDLE',
+        'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
+        do_newline=True)
+    server_job = jobset.JobSpec(
+        cmdline=docker_cmdline, shortname=shortname, timeout_seconds=30 * 60)
+    server_job.container_name = container_name
+    return server_job
+
+
+def build_interop_image_jobspec(lang_safename, basename_prefix='grpc_interop'):
+    """Creates jobspec for building interop docker image for a language"""
+    tag = '%s_%s:%s' % (basename_prefix, lang_safename, uuid.uuid4())
+    env = {
+        'INTEROP_IMAGE': tag,
+        'BASE_NAME': '%s_%s' % (basename_prefix, lang_safename),
+    }
+    build_job = jobset.JobSpec(
+        cmdline=['tools/run_tests/dockerize/build_interop_image.sh'],
+        environ=env,
+        shortname='build_docker_%s' % lang_safename,
+        timeout_seconds=30 * 60)
+    build_job.tag = tag
+    return build_job
+
+
+argp = argparse.ArgumentParser(description='Run interop tests.')
+argp.add_argument(
+    '-l',
+    '--language',
+    choices=['all'] + sorted(_LANGUAGES),
+    nargs='+',
+    default=['all'],
+    help='Clients to run.')
+argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
+argp.add_argument(
+    '-s',
+    '--scenarios_file',
+    default=None,
+    type=str,
+    help='File containing test scenarios as JSON configs.')
+argp.add_argument(
+    '-n',
+    '--scenario_name',
+    default=None,
+    type=str,
+    help=(
+        'Useful for manual runs: specify the name of '
+        'the scenario to run from scenarios_file. Run all scenarios if unset.'))
+argp.add_argument(
+    '--cxx_image_tag',
+    default=None,
+    type=str,
+    help=('Setting this skips the clients docker image '
+          'build step and runs the client from the named '
+          'image. Only supports running a one client language.'))
+argp.add_argument(
+    '--go_image_tag',
+    default=None,
+    type=str,
+    help=('Setting this skips the clients docker image build '
+          'step and runs the client from the named image. Only '
+          'supports running a one client language.'))
+argp.add_argument(
+    '--java_image_tag',
+    default=None,
+    type=str,
+    help=('Setting this skips the clients docker image build '
+          'step and runs the client from the named image. Only '
+          'supports running a one client language.'))
+argp.add_argument(
+    '--servers_image_tag',
+    default=None,
+    type=str,
+    help=('Setting this skips the fake servers docker image '
+          'build step and runs the servers from the named image.'))
+argp.add_argument(
+    '--no_skips',
+    default=False,
+    type=bool,
+    nargs='?',
+    const=True,
+    help=('Useful for manual runs. Setting this overrides test '
+          '"skips" configured in test scenarios.'))
+argp.add_argument(
+    '--verbose',
+    default=False,
+    type=bool,
+    nargs='?',
+    const=True,
+    help='Increase logging.')
+args = argp.parse_args()
+
+docker_images = {}
+
+build_jobs = []
+if len(args.language) and args.language[0] == 'all':
+    languages = _LANGUAGES.keys()
+else:
+    languages = args.language
+for lang_name in languages:
+    l = _LANGUAGES[lang_name]
+    # First check if a pre-built image was supplied, and avoid
+    # rebuilding the particular docker image if so.
+    if lang_name == 'c++' and args.cxx_image_tag:
+        docker_images[str(l.safename)] = args.cxx_image_tag
+    elif lang_name == 'go' and args.go_image_tag:
+        docker_images[str(l.safename)] = args.go_image_tag
+    elif lang_name == 'java' and args.java_image_tag:
+        docker_images[str(l.safename)] = args.java_image_tag
+    else:
+        # Build the test client in docker and save the fully
+        # built image.
+        job = build_interop_image_jobspec(l.safename)
+        build_jobs.append(job)
+        docker_images[str(l.safename)] = job.tag
+
+# First check if a pre-built image was supplied.
+if args.servers_image_tag:
+    docker_images[_FAKE_SERVERS_SAFENAME] = args.servers_image_tag
+else:
+    # Build the test servers in docker and save the fully
+    # built image.
+    job = build_interop_image_jobspec(
+        _FAKE_SERVERS_SAFENAME, basename_prefix='lb_interop')
+    build_jobs.append(job)
+    docker_images[_FAKE_SERVERS_SAFENAME] = job.tag
+
+if build_jobs:
+    jobset.message('START', 'Building interop docker images.', do_newline=True)
+    print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
+    num_failures, _ = jobset.run(
+        build_jobs, newline_on_success=True, maxjobs=args.jobs)
+    if num_failures == 0:
+        jobset.message(
+            'SUCCESS', 'All docker images built successfully.', do_newline=True)
+    else:
+        jobset.message(
+            'FAILED', 'Failed to build interop docker images.', do_newline=True)
+        sys.exit(1)
+
+
+def wait_until_dns_server_is_up(dns_server_ip):
+    """Probes the DNS server until it's running and safe for tests."""
+    for i in range(0, 30):
+        print('Health check: attempt to connect to DNS server over TCP.')
+        tcp_connect_subprocess = subprocess.Popen([
+            os.path.join(os.getcwd(), 'test/cpp/naming/utils/tcp_connect.py'),
+            '--server_host', dns_server_ip, '--server_port',
+            str(53), '--timeout',
+            str(1)
+        ])
+        tcp_connect_subprocess.communicate()
+        if tcp_connect_subprocess.returncode == 0:
+            print(('Health check: attempt to make an A-record '
+                   'query to DNS server.'))
+            dns_resolver_subprocess = subprocess.Popen(
+                [
+                    os.path.join(os.getcwd(),
+                                 'test/cpp/naming/utils/dns_resolver.py'),
+                    '--qname', ('health-check-local-dns-server-is-alive.'
+                                'resolver-tests.grpctestingexp'),
+                    '--server_host', dns_server_ip, '--server_port',
+                    str(53)
+                ],
+                stdout=subprocess.PIPE)
+            dns_resolver_stdout, _ = dns_resolver_subprocess.communicate()
+            if dns_resolver_subprocess.returncode == 0:
+                if '123.123.123.123' in dns_resolver_stdout:
+                    print(('DNS server is up! '
+                           'Successfully reached it over UDP and TCP.'))
+                    return
+        time.sleep(0.1)
+    raise Exception(('Failed to reach DNS server over TCP and/or UDP. '
+                     'Exitting without running tests.'))
+
+
+def shortname(shortname_prefix, shortname, index):
+    return '%s_%s_%d' % (shortname_prefix, shortname, index)
+
+
+def run_one_scenario(scenario_config):
+    jobset.message('START', 'Run scenario: %s' % scenario_config['name'])
+    server_jobs = {}
+    server_addresses = {}
+    suppress_server_logs = True
+    try:
+        backend_addrs = []
+        fallback_ips = []
+        grpclb_ips = []
+        shortname_prefix = scenario_config['name']
+        # Start backends
+        for i in xrange(len(scenario_config['backend_configs'])):
+            backend_config = scenario_config['backend_configs'][i]
+            backend_shortname = shortname(shortname_prefix, 'backend_server', i)
+            backend_spec = backend_server_jobspec(
+                backend_config['transport_sec'], backend_shortname)
+            backend_job = dockerjob.DockerJob(backend_spec)
+            server_jobs[backend_shortname] = backend_job
+            backend_addrs.append('%s:%d' % (backend_job.ip_address(),
+                                            _BACKEND_SERVER_PORT))
+        # Start fallbacks
+        for i in xrange(len(scenario_config['fallback_configs'])):
+            fallback_config = scenario_config['fallback_configs'][i]
+            fallback_shortname = shortname(shortname_prefix, 'fallback_server',
+                                           i)
+            fallback_spec = fallback_server_jobspec(
+                fallback_config['transport_sec'], fallback_shortname)
+            fallback_job = dockerjob.DockerJob(fallback_spec)
+            server_jobs[fallback_shortname] = fallback_job
+            fallback_ips.append(fallback_job.ip_address())
+        # Start balancers
+        for i in xrange(len(scenario_config['balancer_configs'])):
+            balancer_config = scenario_config['balancer_configs'][i]
+            grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i)
+            grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'],
+                                         balancer_config['short_stream'],
+                                         backend_addrs, grpclb_shortname)
+            grpclb_job = dockerjob.DockerJob(grpclb_spec)
+            server_jobs[grpclb_shortname] = grpclb_job
+            grpclb_ips.append(grpclb_job.ip_address())
+        # Start DNS server
+        dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0)
+        dns_server_spec = dns_server_in_docker_jobspec(
+            grpclb_ips, fallback_ips, dns_server_shortname,
+            scenario_config['cause_no_error_no_data_for_balancer_a_record'])
+        dns_server_job = dockerjob.DockerJob(dns_server_spec)
+        server_jobs[dns_server_shortname] = dns_server_job
+        # Get the IP address of the docker container running the DNS server.
+        # The DNS server is running on port 53 of that IP address. Note we will
+        # point the DNS resolvers of grpc clients under test to our controlled
+        # DNS server by effectively modifying the /etc/resolve.conf "nameserver"
+        # lists of their docker containers.
+        dns_server_ip = dns_server_job.ip_address()
+        wait_until_dns_server_is_up(dns_server_ip)
+        # Run clients
+        jobs = []
+        for lang_name in languages:
+            # Skip languages that are known to not currently
+            # work for this test.
+            if not args.no_skips and lang_name in scenario_config.get(
+                    'skip_langs', []):
+                jobset.message('IDLE',
+                               'Skipping scenario: %s for language: %s\n' %
+                               (scenario_config['name'], lang_name))
+                continue
+            lang = _LANGUAGES[lang_name]
+            test_job = lb_client_interop_jobspec(
+                lang,
+                dns_server_ip,
+                docker_image=docker_images.get(lang.safename),
+                transport_security=scenario_config['transport_sec'])
+            jobs.append(test_job)
+        jobset.message('IDLE', 'Jobs to run: \n%s\n' % '\n'.join(
+            str(job) for job in jobs))
+        num_failures, resultset = jobset.run(
+            jobs, newline_on_success=True, maxjobs=args.jobs)
+        report_utils.render_junit_xml_report(resultset, 'sponge_log.xml')
+        if num_failures:
+            suppress_server_logs = False
+            jobset.message(
+                'FAILED',
+                'Scenario: %s. Some tests failed' % scenario_config['name'],
+                do_newline=True)
+        else:
+            jobset.message(
+                'SUCCESS',
+                'Scenario: %s. All tests passed' % scenario_config['name'],
+                do_newline=True)
+        return num_failures
+    finally:
+        # Check if servers are still running.
+        for server, job in server_jobs.items():
+            if not job.is_running():
+                print('Server "%s" has exited prematurely.' % server)
+        suppress_failure = suppress_server_logs and not args.verbose
+        dockerjob.finish_jobs(
+            [j for j in six.itervalues(server_jobs)],
+            suppress_failure=suppress_failure)
+
+
+num_failures = 0
+with open(args.scenarios_file, 'r') as scenarios_input:
+    all_scenarios = json.loads(scenarios_input.read())
+    for scenario in all_scenarios:
+        if args.scenario_name:
+            if args.scenario_name != scenario['name']:
+                jobset.message('IDLE',
+                               'Skipping scenario: %s' % scenario['name'])
+                continue
+        num_failures += run_one_scenario(scenario)
+if num_failures == 0:
+    sys.exit(0)
+else:
+    sys.exit(1)