yapf run_tests

tools/distrib/yapf_code.sh

@@ -25,10 +25,7 @@ DIRS=(
     'tools/distrib'
     'tools/interop_matrix'
     'tools/profiling'
-    'tools/run_tests/python_utils'
-    'tools/run_tests/sanity'
-    'tools/run_tests/performance'
-    'tools/run_tests/artifacts'
+    'tools/run_tests'
 )
 EXCLUSIONS=(
     'grpcio/grpc_*.py'

tools/run_tests/run_build_statistics.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Tool to get build statistics from Jenkins and upload to BigQuery."""
 
 from __future__ import print_function
@@ -27,39 +26,38 @@ import re
 import sys
 import urllib
 
-
-gcp_utils_dir = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../gcp/utils'))
+gcp_utils_dir = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '../gcp/utils'))
 sys.path.append(gcp_utils_dir)
 import big_query_utils
 
-
 _PROJECT_ID = 'grpc-testing'
 _HAS_MATRIX = True
-_BUILDS = {'gRPC_interop_master': not _HAS_MATRIX,
-           'gRPC_master_linux': not _HAS_MATRIX,
-           'gRPC_master_macos': not _HAS_MATRIX,
-           'gRPC_master_windows': not _HAS_MATRIX,
-           'gRPC_performance_master': not _HAS_MATRIX,
-           'gRPC_portability_master_linux': not _HAS_MATRIX,
-           'gRPC_portability_master_windows': not _HAS_MATRIX,
-           'gRPC_master_asanitizer_c': not _HAS_MATRIX,
-           'gRPC_master_asanitizer_cpp': not _HAS_MATRIX,
-           'gRPC_master_msan_c': not _HAS_MATRIX,
-           'gRPC_master_tsanitizer_c': not _HAS_MATRIX,
-           'gRPC_master_tsan_cpp': not _HAS_MATRIX,
-           'gRPC_interop_pull_requests': not _HAS_MATRIX,
-           'gRPC_performance_pull_requests': not _HAS_MATRIX,
-           'gRPC_portability_pull_requests_linux': not _HAS_MATRIX,
-           'gRPC_portability_pr_win': not _HAS_MATRIX,
-           'gRPC_pull_requests_linux': not _HAS_MATRIX,
-           'gRPC_pull_requests_macos': not _HAS_MATRIX,
-           'gRPC_pr_win': not _HAS_MATRIX,
-           'gRPC_pull_requests_asan_c': not _HAS_MATRIX,
-           'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX,
-           'gRPC_pull_requests_msan_c': not _HAS_MATRIX,
-           'gRPC_pull_requests_tsan_c': not _HAS_MATRIX,
-           'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX,
+_BUILDS = {
+    'gRPC_interop_master': not _HAS_MATRIX,
+    'gRPC_master_linux': not _HAS_MATRIX,
+    'gRPC_master_macos': not _HAS_MATRIX,
+    'gRPC_master_windows': not _HAS_MATRIX,
+    'gRPC_performance_master': not _HAS_MATRIX,
+    'gRPC_portability_master_linux': not _HAS_MATRIX,
+    'gRPC_portability_master_windows': not _HAS_MATRIX,
+    'gRPC_master_asanitizer_c': not _HAS_MATRIX,
+    'gRPC_master_asanitizer_cpp': not _HAS_MATRIX,
+    'gRPC_master_msan_c': not _HAS_MATRIX,
+    'gRPC_master_tsanitizer_c': not _HAS_MATRIX,
+    'gRPC_master_tsan_cpp': not _HAS_MATRIX,
+    'gRPC_interop_pull_requests': not _HAS_MATRIX,
+    'gRPC_performance_pull_requests': not _HAS_MATRIX,
+    'gRPC_portability_pull_requests_linux': not _HAS_MATRIX,
+    'gRPC_portability_pr_win': not _HAS_MATRIX,
+    'gRPC_pull_requests_linux': not _HAS_MATRIX,
+    'gRPC_pull_requests_macos': not _HAS_MATRIX,
+    'gRPC_pr_win': not _HAS_MATRIX,
+    'gRPC_pull_requests_asan_c': not _HAS_MATRIX,
+    'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX,
+    'gRPC_pull_requests_msan_c': not _HAS_MATRIX,
+    'gRPC_pull_requests_tsan_c': not _HAS_MATRIX,
+    'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX,
 }
 _URL_BASE = 'https://grpc-testing.appspot.com/job'
 
@@ -99,147 +97,155 @@ _DATASET_ID = 'build_statistics'
 
 
 def _scrape_for_known_errors(html):
-  error_list = []
-  for known_error in _KNOWN_ERRORS:
-    errors = re.findall(known_error, html)
-    this_error_count = len(errors)
-    if this_error_count > 0: 
-      error_list.append({'description': known_error,
-                         'count': this_error_count})
-      print('====> %d failures due to %s' % (this_error_count, known_error))
-  return error_list
+    error_list = []
+    for known_error in _KNOWN_ERRORS:
+        errors = re.findall(known_error, html)
+        this_error_count = len(errors)
+        if this_error_count > 0:
+            error_list.append({
+                'description': known_error,
+                'count': this_error_count
+            })
+            print('====> %d failures due to %s' %
+                  (this_error_count, known_error))
+    return error_list
 
 
 def _no_report_files_found(html):
-  return _NO_REPORT_FILES_FOUND_ERROR in html
+    return _NO_REPORT_FILES_FOUND_ERROR in html
 
 
 def _get_last_processed_buildnumber(build_name):
-  query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
-      _PROJECT_ID, _DATASET_ID, build_name)
-  query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
-  page = bq.jobs().getQueryResults(
-      pageToken=None,
-      **query_job['jobReference']).execute(num_retries=3)
-  if page['rows'][0]['f'][0]['v']:
-    return int(page['rows'][0]['f'][0]['v'])
-  return 0
+    query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
+        _PROJECT_ID, _DATASET_ID, build_name)
+    query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
+    page = bq.jobs().getQueryResults(
+        pageToken=None, **query_job['jobReference']).execute(num_retries=3)
+    if page['rows'][0]['f'][0]['v']:
+        return int(page['rows'][0]['f'][0]['v'])
+    return 0
 
 
 def _process_matrix(build, url_base):
-  matrix_list = []
-  for matrix in build.get_matrix_runs():
-    matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*', 
-                          matrix.name).groups()[0]
-    matrix_tuple = matrix_str.split(',')
-    json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % (
-        url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
-    console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % (
-        url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
-    matrix_dict = {'name': matrix_str,
-                   'duration': matrix.get_duration().total_seconds()}
-    matrix_dict.update(_process_build(json_url, console_url))
-    matrix_list.append(matrix_dict)
-
-  return matrix_list 
+    matrix_list = []
+    for matrix in build.get_matrix_runs():
+        matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*',
+                              matrix.name).groups()[0]
+        matrix_tuple = matrix_str.split(',')
+        json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % (
+            url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
+        console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % (
+            url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
+        matrix_dict = {
+            'name': matrix_str,
+            'duration': matrix.get_duration().total_seconds()
+        }
+        matrix_dict.update(_process_build(json_url, console_url))
+        matrix_list.append(matrix_dict)
+
+    return matrix_list
 
 
 def _process_build(json_url, console_url):
-  build_result = {}
-  error_list = []
-  try:
-    html = urllib.urlopen(json_url).read()
-    test_result = json.loads(html)
-    print('====> Parsing result from %s' % json_url)
-    failure_count = test_result['failCount']
-    build_result['pass_count'] = test_result['passCount']
-    build_result['failure_count'] = failure_count
-    # This means Jenkins failure occurred.
-    build_result['no_report_files_found'] = _no_report_files_found(html)
-    # Only check errors if Jenkins failure occurred.
-    if build_result['no_report_files_found']:
-      error_list = _scrape_for_known_errors(html)
-  except Exception as e:
-    print('====> Got exception for %s: %s.' % (json_url, str(e)))   
-    print('====> Parsing errors from %s.' % console_url)
-    html = urllib.urlopen(console_url).read()
-    build_result['pass_count'] = 0  
-    build_result['failure_count'] = 1
-    # In this case, the string doesn't exist in the result html but the fact 
-    # that we fail to parse the result html indicates Jenkins failure and hence 
-    # no report files were generated.
-    build_result['no_report_files_found'] = True
-    error_list = _scrape_for_known_errors(html)
-
-  if error_list:
-    build_result['error'] = error_list
-  elif build_result['no_report_files_found']:
-    build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}]
-  else:
-    build_result['error'] = [{'description': '', 'count': 0}]
-
-  return build_result 
+    build_result = {}
+    error_list = []
+    try:
+        html = urllib.urlopen(json_url).read()
+        test_result = json.loads(html)
+        print('====> Parsing result from %s' % json_url)
+        failure_count = test_result['failCount']
+        build_result['pass_count'] = test_result['passCount']
+        build_result['failure_count'] = failure_count
+        # This means Jenkins failure occurred.
+        build_result['no_report_files_found'] = _no_report_files_found(html)
+        # Only check errors if Jenkins failure occurred.
+        if build_result['no_report_files_found']:
+            error_list = _scrape_for_known_errors(html)
+    except Exception as e:
+        print('====> Got exception for %s: %s.' % (json_url, str(e)))
+        print('====> Parsing errors from %s.' % console_url)
+        html = urllib.urlopen(console_url).read()
+        build_result['pass_count'] = 0
+        build_result['failure_count'] = 1
+        # In this case, the string doesn't exist in the result html but the fact 
+        # that we fail to parse the result html indicates Jenkins failure and hence 
+        # no report files were generated.
+        build_result['no_report_files_found'] = True
+        error_list = _scrape_for_known_errors(html)
+
+    if error_list:
+        build_result['error'] = error_list
+    elif build_result['no_report_files_found']:
+        build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}]
+    else:
+        build_result['error'] = [{'description': '', 'count': 0}]
+
+    return build_result
 
 
 # parse command line
 argp = argparse.ArgumentParser(description='Get build statistics.')
 argp.add_argument('-u', '--username', default='jenkins')
-argp.add_argument('-b', '--builds', 
-                  choices=['all'] + sorted(_BUILDS.keys()),
-                  nargs='+',
-                  default=['all'])
+argp.add_argument(
+    '-b',
+    '--builds',
+    choices=['all'] + sorted(_BUILDS.keys()),
+    nargs='+',
+    default=['all'])
 args = argp.parse_args()
 
 J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken')
 bq = big_query_utils.create_big_query()
 
 for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
-  print('====> Build: %s' % build_name)
-  # Since get_last_completed_build() always fails due to malformatted string
-  # error, we use get_build_metadata() instead.
-  job = None
-  try:
-    job = J[build_name]
-  except Exception as e:
-    print('====> Failed to get build %s: %s.' % (build_name, str(e)))
-    continue
-  last_processed_build_number = _get_last_processed_buildnumber(build_name)
-  last_complete_build_number = job.get_last_completed_buildnumber()
-  # To avoid processing all builds for a project never looked at. In this case,
-  # only examine 10 latest builds.
-  starting_build_number = max(last_processed_build_number+1, 
-                              last_complete_build_number-9)
-  for build_number in xrange(starting_build_number, 
-                             last_complete_build_number+1):
-    print('====> Processing %s build %d.' % (build_name, build_number))
-    build = None
+    print('====> Build: %s' % build_name)
+    # Since get_last_completed_build() always fails due to malformatted string
+    # error, we use get_build_metadata() instead.
+    job = None
     try:
-      build = job.get_build_metadata(build_number)
-      print('====> Build status: %s.' % build.get_status())
-      if build.get_status() == 'ABORTED':
+        job = J[build_name]
+    except Exception as e:
+        print('====> Failed to get build %s: %s.' % (build_name, str(e)))
         continue
-      # If any build is still running, stop processing this job. Next time, we
-      # start from where it was left so that all builds are processed 
-      # sequentially.
-      if build.is_running():
-        print('====> Build %d is still running.' % build_number)
-        break
-    except KeyError:
-      print('====> Build %s is missing. Skip.' % build_number)
-      continue
-    build_result = {'build_number': build_number, 
-                    'timestamp': str(build.get_timestamp())}
-    url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number)
-    if _BUILDS[build_name]:  # The build has matrix, such as gRPC_master.
-      build_result['matrix'] = _process_matrix(build, url_base)
-    else:
-      json_url = '%s/testReport/api/json' % url_base
-      console_url = '%s/consoleFull' % url_base
-      build_result['duration'] = build.get_duration().total_seconds()
-      build_stat = _process_build(json_url, console_url)
-      build_result.update(build_stat)
-    rows = [big_query_utils.make_row(build_number, build_result)]
-    if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, 
-                                       rows):
-      print('====> Error uploading result to bigquery.')
-      sys.exit(1)
+    last_processed_build_number = _get_last_processed_buildnumber(build_name)
+    last_complete_build_number = job.get_last_completed_buildnumber()
+    # To avoid processing all builds for a project never looked at. In this case,
+    # only examine 10 latest builds.
+    starting_build_number = max(last_processed_build_number + 1,
+                                last_complete_build_number - 9)
+    for build_number in xrange(starting_build_number,
+                               last_complete_build_number + 1):
+        print('====> Processing %s build %d.' % (build_name, build_number))
+        build = None
+        try:
+            build = job.get_build_metadata(build_number)
+            print('====> Build status: %s.' % build.get_status())
+            if build.get_status() == 'ABORTED':
+                continue
+            # If any build is still running, stop processing this job. Next time, we
+            # start from where it was left so that all builds are processed 
+            # sequentially.
+            if build.is_running():
+                print('====> Build %d is still running.' % build_number)
+                break
+        except KeyError:
+            print('====> Build %s is missing. Skip.' % build_number)
+            continue
+        build_result = {
+            'build_number': build_number,
+            'timestamp': str(build.get_timestamp())
+        }
+        url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number)
+        if _BUILDS[build_name]:  # The build has matrix, such as gRPC_master.
+            build_result['matrix'] = _process_matrix(build, url_base)
+        else:
+            json_url = '%s/testReport/api/json' % url_base
+            console_url = '%s/consoleFull' % url_base
+            build_result['duration'] = build.get_duration().total_seconds()
+            build_stat = _process_build(json_url, console_url)
+            build_result.update(build_stat)
+        rows = [big_query_utils.make_row(build_number, build_result)]
+        if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID,
+                                           build_name, rows):
+            print('====> Error uploading result to bigquery.')
+            sys.exit(1)

tools/run_tests/run_microbenchmark.py

@@ -23,26 +23,31 @@ import argparse
 import python_utils.jobset as jobset
 import python_utils.start_port_server as start_port_server
 
-sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', 'bm_diff'))
+sys.path.append(
+    os.path.join(
+        os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks',
+        'bm_diff'))
 import bm_constants
 
 flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
 
 os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 if not os.path.exists('reports'):
-  os.makedirs('reports')
+    os.makedirs('reports')
 
 start_port_server.start_port_server()
 
+
 def fnize(s):
-  out = ''
-  for c in s:
-    if c in '<>, /':
-      if len(out) and out[-1] == '_': continue
-      out += '_'
-    else:
-      out += c
-  return out
+    out = ''
+    for c in s:
+        if c in '<>, /':
+            if len(out) and out[-1] == '_': continue
+            out += '_'
+        else:
+            out += c
+    return out
+
 
 # index html
 index_html = """
@@ -53,169 +58,202 @@ index_html = """
 <body>
 """
 
+
 def heading(name):
-  global index_html
-  index_html += "<h1>%s</h1>\n" % name
+    global index_html
+    index_html += "<h1>%s</h1>\n" % name
+
 
 def link(txt, tgt):
-  global index_html
-  index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
-      cgi.escape(tgt, quote=True), cgi.escape(txt))
+    global index_html
+    index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
+        cgi.escape(tgt, quote=True), cgi.escape(txt))
+
 
 def text(txt):
-  global index_html
-  index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
+    global index_html
+    index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
+
 
 def collect_latency(bm_name, args):
-  """generate latency profiles"""
-  benchmarks = []
-  profile_analysis = []
-  cleanup = []
-
-  heading('Latency Profiles: %s' % bm_name)
-  subprocess.check_call(
-      ['make', bm_name,
-       'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
-  for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
-                                       '--benchmark_list_tests']).splitlines():
-    link(line, '%s.txt' % fnize(line))
-    benchmarks.append(
-        jobset.JobSpec(['bins/basicprof/%s' % bm_name,
-                        '--benchmark_filter=^%s$' % line,
-                        '--benchmark_min_time=0.05'],
-                       environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
-                       shortname='profile-%s' % fnize(line)))
-    profile_analysis.append(
-        jobset.JobSpec([sys.executable,
-                        'tools/profiling/latency_profile/profile_analyzer.py',
-                        '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
-                        '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=20*60,
-                        shortname='analyze-%s' % fnize(line)))
-    cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
-    # periodically flush out the list of jobs: profile_analysis jobs at least
-    # consume upwards of five gigabytes of ram in some cases, and so analysing
-    # hundreds of them at once is impractical -- but we want at least some
-    # concurrency or the work takes too long
-    if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
-      # run up to half the cpu count: each benchmark can use up to two cores
-      # (one for the microbenchmark, one for the data flush)
-      jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
-      jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
-      jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
-      benchmarks = []
-      profile_analysis = []
-      cleanup = []
-  # run the remaining benchmarks that weren't flushed
-  if len(benchmarks):
-    jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
-    jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
-    jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+    """generate latency profiles"""
+    benchmarks = []
+    profile_analysis = []
+    cleanup = []
+
+    heading('Latency Profiles: %s' % bm_name)
+    subprocess.check_call([
+        'make', bm_name, 'CONFIG=basicprof', '-j',
+        '%d' % multiprocessing.cpu_count()
+    ])
+    for line in subprocess.check_output(
+        ['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines():
+        link(line, '%s.txt' % fnize(line))
+        benchmarks.append(
+            jobset.JobSpec(
+                [
+                    'bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' %
+                    line, '--benchmark_min_time=0.05'
+                ],
+                environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
+                shortname='profile-%s' % fnize(line)))
+        profile_analysis.append(
+            jobset.JobSpec(
+                [
+                    sys.executable,
+                    'tools/profiling/latency_profile/profile_analyzer.py',
+                    '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
+                    '--out', 'reports/%s.txt' % fnize(line)
+                ],
+                timeout_seconds=20 * 60,
+                shortname='analyze-%s' % fnize(line)))
+        cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
+        # periodically flush out the list of jobs: profile_analysis jobs at least
+        # consume upwards of five gigabytes of ram in some cases, and so analysing
+        # hundreds of them at once is impractical -- but we want at least some
+        # concurrency or the work takes too long
+        if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
+            # run up to half the cpu count: each benchmark can use up to two cores
+            # (one for the microbenchmark, one for the data flush)
+            jobset.run(
+                benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
+            jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+            jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+            benchmarks = []
+            profile_analysis = []
+            cleanup = []
+    # run the remaining benchmarks that weren't flushed
+    if len(benchmarks):
+        jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
+        jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+        jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+
 
 def collect_perf(bm_name, args):
-  """generate flamegraphs"""
-  heading('Flamegraphs: %s' % bm_name)
-  subprocess.check_call(
-      ['make', bm_name,
-       'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
-  benchmarks = []
-  profile_analysis = []
-  cleanup = []
-  for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
-                                       '--benchmark_list_tests']).splitlines():
-    link(line, '%s.svg' % fnize(line))
-    benchmarks.append(
-        jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),
-                        '-g', '-F', '997',
-                        'bins/mutrace/%s' % bm_name,
-                        '--benchmark_filter=^%s$' % line,
-                        '--benchmark_min_time=10'],
-                        shortname='perf-%s' % fnize(line)))
-    profile_analysis.append(
-        jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],
-                       environ = {
-                           'PERF_BASE_NAME': fnize(line),
-                           'OUTPUT_DIR': 'reports',
-                           'OUTPUT_FILENAME': fnize(line),
-                       },
-                       shortname='flame-%s' % fnize(line)))
-    cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
-    cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
-    # periodically flush out the list of jobs: temporary space required for this
-    # processing is large
-    if len(benchmarks) >= 20:
-      # run up to half the cpu count: each benchmark can use up to two cores
-      # (one for the microbenchmark, one for the data flush)
-      jobset.run(benchmarks, maxjobs=1)
-      jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
-      jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
-      benchmarks = []
-      profile_analysis = []
-      cleanup = []
-  # run the remaining benchmarks that weren't flushed
-  if len(benchmarks):
-    jobset.run(benchmarks, maxjobs=1)
-    jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
-    jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+    """generate flamegraphs"""
+    heading('Flamegraphs: %s' % bm_name)
+    subprocess.check_call([
+        'make', bm_name, 'CONFIG=mutrace', '-j',
+        '%d' % multiprocessing.cpu_count()
+    ])
+    benchmarks = []
+    profile_analysis = []
+    cleanup = []
+    for line in subprocess.check_output(
+        ['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines():
+        link(line, '%s.svg' % fnize(line))
+        benchmarks.append(
+            jobset.JobSpec(
+                [
+                    'perf', 'record', '-o', '%s-perf.data' % fnize(
+                        line), '-g', '-F', '997', 'bins/mutrace/%s' % bm_name,
+                    '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'
+                ],
+                shortname='perf-%s' % fnize(line)))
+        profile_analysis.append(
+            jobset.JobSpec(
+                [
+                    'tools/run_tests/performance/process_local_perf_flamegraphs.sh'
+                ],
+                environ={
+                    'PERF_BASE_NAME': fnize(line),
+                    'OUTPUT_DIR': 'reports',
+                    'OUTPUT_FILENAME': fnize(line),
+                },
+                shortname='flame-%s' % fnize(line)))
+        cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
+        cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
+        # periodically flush out the list of jobs: temporary space required for this
+        # processing is large
+        if len(benchmarks) >= 20:
+            # run up to half the cpu count: each benchmark can use up to two cores
+            # (one for the microbenchmark, one for the data flush)
+            jobset.run(benchmarks, maxjobs=1)
+            jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+            jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+            benchmarks = []
+            profile_analysis = []
+            cleanup = []
+    # run the remaining benchmarks that weren't flushed
+    if len(benchmarks):
+        jobset.run(benchmarks, maxjobs=1)
+        jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+        jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+
 
 def run_summary(bm_name, cfg, base_json_name):
-  subprocess.check_call(
-      ['make', bm_name,
-       'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])
-  cmd = ['bins/%s/%s' % (cfg, bm_name),
-         '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
-         '--benchmark_out_format=json']
-  if args.summary_time is not None:
-    cmd += ['--benchmark_min_time=%d' % args.summary_time]
-  return subprocess.check_output(cmd)
+    subprocess.check_call([
+        'make', bm_name, 'CONFIG=%s' % cfg, '-j',
+        '%d' % multiprocessing.cpu_count()
+    ])
+    cmd = [
+        'bins/%s/%s' % (cfg, bm_name), '--benchmark_out=%s.%s.json' %
+        (base_json_name, cfg), '--benchmark_out_format=json'
+    ]
+    if args.summary_time is not None:
+        cmd += ['--benchmark_min_time=%d' % args.summary_time]
+    return subprocess.check_output(cmd)
+
 
 def collect_summary(bm_name, args):
-  heading('Summary: %s [no counters]' % bm_name)
-  text(run_summary(bm_name, 'opt', bm_name))
-  heading('Summary: %s [with counters]' % bm_name)
-  text(run_summary(bm_name, 'counters', bm_name))
-  if args.bigquery_upload:
-    with open('%s.csv' % bm_name, 'w') as f:
-      f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',
-                                       '%s.counters.json' % bm_name,
-                                       '%s.opt.json' % bm_name]))
-    subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])
+    heading('Summary: %s [no counters]' % bm_name)
+    text(run_summary(bm_name, 'opt', bm_name))
+    heading('Summary: %s [with counters]' % bm_name)
+    text(run_summary(bm_name, 'counters', bm_name))
+    if args.bigquery_upload:
+        with open('%s.csv' % bm_name, 'w') as f:
+            f.write(
+                subprocess.check_output([
+                    'tools/profiling/microbenchmarks/bm2bq.py',
+                    '%s.counters.json' % bm_name, '%s.opt.json' % bm_name
+                ]))
+        subprocess.check_call([
+            'bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name
+        ])
+
 
 collectors = {
-  'latency': collect_latency,
-  'perf': collect_perf,
-  'summary': collect_summary,
+    'latency': collect_latency,
+    'perf': collect_perf,
+    'summary': collect_summary,
 }
 
 argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
-argp.add_argument('-c', '--collect',
-                  choices=sorted(collectors.keys()),
-                  nargs='*',
-                  default=sorted(collectors.keys()),
-                  help='Which collectors should be run against each benchmark')
-argp.add_argument('-b', '--benchmarks',
-                  choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-                  default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
-                  nargs='+',
-                  type=str,
-                  help='Which microbenchmarks should be run')
-argp.add_argument('--bigquery_upload',
-                  default=False,
-                  action='store_const',
-                  const=True,
-                  help='Upload results from summary collection to bigquery')
-argp.add_argument('--summary_time',
-                  default=None,
-                  type=int,
-                  help='Minimum time to run benchmarks for the summary collection')
+argp.add_argument(
+    '-c',
+    '--collect',
+    choices=sorted(collectors.keys()),
+    nargs='*',
+    default=sorted(collectors.keys()),
+    help='Which collectors should be run against each benchmark')
+argp.add_argument(
+    '-b',
+    '--benchmarks',
+    choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+    default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+    nargs='+',
+    type=str,
+    help='Which microbenchmarks should be run')
+argp.add_argument(
+    '--bigquery_upload',
+    default=False,
+    action='store_const',
+    const=True,
+    help='Upload results from summary collection to bigquery')
+argp.add_argument(
+    '--summary_time',
+    default=None,
+    type=int,
+    help='Minimum time to run benchmarks for the summary collection')
 args = argp.parse_args()
 
 try:
-  for collect in args.collect:
-    for bm_name in args.benchmarks:
-      collectors[collect](bm_name, args)
+    for collect in args.collect:
+        for bm_name in args.benchmarks:
+            collectors[collect](bm_name, args)
 finally:
-  if not os.path.exists('reports'):
-    os.makedirs('reports')
-  index_html += "</body>\n</html>\n"
-  with open('reports/index.html', 'w') as f:
-    f.write(index_html)
+    if not os.path.exists('reports'):
+        os.makedirs('reports')
+    index_html += "</body>\n</html>\n"
+    with open('reports/index.html', 'w') as f:
+        f.write(index_html)

tools/run_tests/run_tests_matrix.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Run test matrix."""
 
 from __future__ import print_function
@@ -29,14 +28,14 @@ from python_utils.filter_pull_request_tests import filter_tests
 _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 os.chdir(_ROOT)
 
-_DEFAULT_RUNTESTS_TIMEOUT = 1*60*60
+_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60
 
 # Set the timeout high to allow enough time for sanitizers and pre-building
 # clang docker.
-_CPP_RUNTESTS_TIMEOUT = 4*60*60
+_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60
 
 # C++ TSAN takes longer than other sanitizers
-_CPP_TSAN_RUNTESTS_TIMEOUT = 8*60*60
+_CPP_TSAN_RUNTESTS_TIMEOUT = 8 * 60 * 60
 
 # Number of jobs assigned to each run_tests.py instance
 _DEFAULT_INNER_JOBS = 2
@@ -46,448 +45,517 @@ _REPORT_SUFFIX = 'sponge_log.xml'
 
 
 def _report_filename(name):
-  """Generates report file name"""
-  return 'report_%s_%s' % (name, _REPORT_SUFFIX)
+    """Generates report file name"""
+    return 'report_%s_%s' % (name, _REPORT_SUFFIX)
 
 
 def _report_filename_internal_ci(name):
-  """Generates report file name that leads to better presentation by internal CI"""
-  return '%s/%s' % (name, _REPORT_SUFFIX)
+    """Generates report file name that leads to better presentation by internal CI"""
+    return '%s/%s' % (name, _REPORT_SUFFIX)
 
 
-def _docker_jobspec(name, runtests_args=[], runtests_envs={},
+def _docker_jobspec(name,
+                    runtests_args=[],
+                    runtests_envs={},
                     inner_jobs=_DEFAULT_INNER_JOBS,
                     timeout_seconds=None):
-  """Run a single instance of run_tests.py in a docker container"""
-  if not timeout_seconds:
-    timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
-  test_job = jobset.JobSpec(
-          cmdline=['python', 'tools/run_tests/run_tests.py',
-                   '--use_docker',
-                   '-t',
-                   '-j', str(inner_jobs),
-                   '-x', _report_filename(name),
-                   '--report_suite_name', '%s' % name] + runtests_args,
-          environ=runtests_envs,
-          shortname='run_tests_%s' % name,
-          timeout_seconds=timeout_seconds)
-  return test_job
-
-
-def _workspace_jobspec(name, runtests_args=[], workspace_name=None,
-                       runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS,
+    """Run a single instance of run_tests.py in a docker container"""
+    if not timeout_seconds:
+        timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
+    test_job = jobset.JobSpec(
+        cmdline=[
+            'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t',
+            '-j', str(inner_jobs), '-x', _report_filename(name),
+            '--report_suite_name', '%s' % name
+        ] + runtests_args,
+        environ=runtests_envs,
+        shortname='run_tests_%s' % name,
+        timeout_seconds=timeout_seconds)
+    return test_job
+
+
+def _workspace_jobspec(name,
+                       runtests_args=[],
+                       workspace_name=None,
+                       runtests_envs={},
+                       inner_jobs=_DEFAULT_INNER_JOBS,
                        timeout_seconds=None):
-  """Run a single instance of run_tests.py in a separate workspace"""
-  if not workspace_name:
-    workspace_name = 'workspace_%s' % name
-  if not timeout_seconds:
-    timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
-  env = {'WORKSPACE_NAME': workspace_name}
-  env.update(runtests_envs)
-  test_job = jobset.JobSpec(
-          cmdline=['bash',
-                   'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
-                   '-t',
-                   '-j', str(inner_jobs),
-                   '-x', '../%s' % _report_filename(name),
-                   '--report_suite_name', '%s' % name] + runtests_args,
-          environ=env,
-          shortname='run_tests_%s' % name,
-          timeout_seconds=timeout_seconds)
-  return test_job
-
-
-def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native',
-                  arch=None, compiler=None,
-                  labels=[], extra_args=[], extra_envs={},
-                  inner_jobs=_DEFAULT_INNER_JOBS,
-                  timeout_seconds=None):
-  result = []
-  for language in languages:
-    for platform in platforms:
-      for config in configs:
-        name = '%s_%s_%s_%s' % (language, platform, config, iomgr_platform)
-        runtests_args = ['-l', language,
-                         '-c', config,
-                         '--iomgr_platform', iomgr_platform]
-        if arch or compiler:
-          name += '_%s_%s' % (arch, compiler)
-          runtests_args += ['--arch', arch,
-                            '--compiler', compiler]
-        if '--build_only' in extra_args:
-          name += '_buildonly'
-        for extra_env in extra_envs:
-          name += '_%s_%s' % (extra_env, extra_envs[extra_env])
-
-        runtests_args += extra_args
-        if platform == 'linux':
-          job = _docker_jobspec(name=name, runtests_args=runtests_args,
-                                runtests_envs=extra_envs, inner_jobs=inner_jobs,
-                                timeout_seconds=timeout_seconds)
-        else:
-          job = _workspace_jobspec(name=name, runtests_args=runtests_args,
-                                   runtests_envs=extra_envs, inner_jobs=inner_jobs,
-                                   timeout_seconds=timeout_seconds)
-
-        job.labels = [platform, config, language, iomgr_platform] + labels
-        result.append(job)
-  return result
+    """Run a single instance of run_tests.py in a separate workspace"""
+    if not workspace_name:
+        workspace_name = 'workspace_%s' % name
+    if not timeout_seconds:
+        timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
+    env = {'WORKSPACE_NAME': workspace_name}
+    env.update(runtests_envs)
+    test_job = jobset.JobSpec(
+        cmdline=[
+            'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
+            '-t', '-j', str(inner_jobs), '-x', '../%s' % _report_filename(name),
+            '--report_suite_name', '%s' % name
+        ] + runtests_args,
+        environ=env,
+        shortname='run_tests_%s' % name,
+        timeout_seconds=timeout_seconds)
+    return test_job
+
+
+def _generate_jobs(languages,
+                   configs,
+                   platforms,
+                   iomgr_platform='native',
+                   arch=None,
+                   compiler=None,
+                   labels=[],
+                   extra_args=[],
+                   extra_envs={},
+                   inner_jobs=_DEFAULT_INNER_JOBS,
+                   timeout_seconds=None):
+    result = []
+    for language in languages:
+        for platform in platforms:
+            for config in configs:
+                name = '%s_%s_%s_%s' % (language, platform, config,
+                                        iomgr_platform)
+                runtests_args = [
+                    '-l', language, '-c', config, '--iomgr_platform',
+                    iomgr_platform
+                ]
+                if arch or compiler:
+                    name += '_%s_%s' % (arch, compiler)
+                    runtests_args += ['--arch', arch, '--compiler', compiler]
+                if '--build_only' in extra_args:
+                    name += '_buildonly'
+                for extra_env in extra_envs:
+                    name += '_%s_%s' % (extra_env, extra_envs[extra_env])
+
+                runtests_args += extra_args
+                if platform == 'linux':
+                    job = _docker_jobspec(
+                        name=name,
+                        runtests_args=runtests_args,
+                        runtests_envs=extra_envs,
+                        inner_jobs=inner_jobs,
+                        timeout_seconds=timeout_seconds)
+                else:
+                    job = _workspace_jobspec(
+                        name=name,
+                        runtests_args=runtests_args,
+                        runtests_envs=extra_envs,
+                        inner_jobs=inner_jobs,
+                        timeout_seconds=timeout_seconds)
+
+                job.labels = [platform, config, language, iomgr_platform
+                             ] + labels
+                result.append(job)
+    return result
 
 
 def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
-  test_jobs = []
-  # supported on linux only
-  test_jobs += _generate_jobs(languages=['sanity', 'php7'],
-                             configs=['dbg', 'opt'],
-                             platforms=['linux'],
-                             labels=['basictests', 'multilang'],
-                             extra_args=extra_args,
-                             inner_jobs=inner_jobs)
-
-  # supported on all platforms.
-  test_jobs += _generate_jobs(languages=['c'],
-                             configs=['dbg', 'opt'],
-                             platforms=['linux', 'macos', 'windows'],
-                             labels=['basictests', 'corelang'],
-                             extra_args=extra_args,
-                             inner_jobs=inner_jobs,
-                             timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-
-  test_jobs += _generate_jobs(languages=['csharp', 'python'],
-                             configs=['dbg', 'opt'],
-                             platforms=['linux', 'macos', 'windows'],
-                             labels=['basictests', 'multilang'],
-                             extra_args=extra_args,
-                             inner_jobs=inner_jobs)
-
-  # supported on linux and mac.
-  test_jobs += _generate_jobs(languages=['c++'],
-                              configs=['dbg', 'opt'],
-                              platforms=['linux', 'macos'],
-                              labels=['basictests', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs,
-                              timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-
-  test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php'],
-                              configs=['dbg', 'opt'],
-                              platforms=['linux', 'macos'],
-                              labels=['basictests', 'multilang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs)
-
-  # supported on mac only.
-  test_jobs += _generate_jobs(languages=['objc'],
-                              configs=['dbg', 'opt'],
-                              platforms=['macos'],
-                              labels=['basictests', 'multilang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs)
-
-  # sanitizers
-  test_jobs += _generate_jobs(languages=['c'],
-                              configs=['msan', 'asan', 'tsan', 'ubsan'],
-                              platforms=['linux'],
-                              labels=['sanitizers', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs,
-                              timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-  test_jobs += _generate_jobs(languages=['c++'],
-                              configs=['asan'],
-                              platforms=['linux'],
-                              labels=['sanitizers', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs,
-                              timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-  test_jobs += _generate_jobs(languages=['c++'],
-                              configs=['tsan'],
-                              platforms=['linux'],
-                              labels=['sanitizers', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs,
-                              timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT)
-
-  return test_jobs
-
-
-def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
-  test_jobs = []
-  # portability C x86
-  test_jobs += _generate_jobs(languages=['c'],
-                              configs=['dbg'],
-                              platforms=['linux'],
-                              arch='x86',
-                              compiler='default',
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs)
-
-  # portability C and C++ on x64
-  for compiler in ['gcc4.8', 'gcc5.3', 'gcc_musl',
-                   'clang3.5', 'clang3.6', 'clang3.7']:
-    test_jobs += _generate_jobs(languages=['c', 'c++'],
-                                configs=['dbg'],
-                                platforms=['linux'],
-                                arch='x64',
-                                compiler=compiler,
-                                labels=['portability', 'corelang'],
-                                extra_args=extra_args,
-                                inner_jobs=inner_jobs,
-                                timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-
-  # portability C on Windows 64-bit (x86 is the default)
-  test_jobs += _generate_jobs(languages=['c'],
-                              configs=['dbg'],
-                              platforms=['windows'],
-                              arch='x64',
-                              compiler='default',
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs)
-
-  # portability C++ on Windows
-  # TODO(jtattermusch): some of the tests are failing, so we force --build_only
-  test_jobs += _generate_jobs(languages=['c++'],
-                              configs=['dbg'],
-                              platforms=['windows'],
-                              arch='default',
-                              compiler='default',
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args + ['--build_only'],
-                              inner_jobs=inner_jobs)
-
-  # portability C and C++ on Windows using VS2017 (build only)
-  # TODO(jtattermusch): some of the tests are failing, so we force --build_only
-  test_jobs += _generate_jobs(languages=['c', 'c++'],
-                              configs=['dbg'],
-                              platforms=['windows'],
-                              arch='x64',
-                              compiler='cmake_vs2017',
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args + ['--build_only'],
-                              inner_jobs=inner_jobs)
-
-  # C and C++ with the c-ares DNS resolver on Linux
-  test_jobs += _generate_jobs(languages=['c', 'c++'],
-                              configs=['dbg'], platforms=['linux'],
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args,
-                              extra_envs={'GRPC_DNS_RESOLVER': 'ares'},
-                              timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-
-  # TODO(zyc): Turn on this test after adding c-ares support on windows.
-  # C with the c-ares DNS resolver on Windows
-  # test_jobs += _generate_jobs(languages=['c'],
-  #                             configs=['dbg'], platforms=['windows'],
-  #                             labels=['portability', 'corelang'],
-  #                             extra_args=extra_args,
-  #                             extra_envs={'GRPC_DNS_RESOLVER': 'ares'})
-
-  # C and C++ build with cmake on Linux
-  # TODO(jtattermusch): some of the tests are failing, so we force --build_only
-  # to make sure it's buildable at least.
-  test_jobs += _generate_jobs(languages=['c', 'c++'],
-                              configs=['dbg'],
-                              platforms=['linux'],
-                              arch='default',
-                              compiler='cmake',
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args + ['--build_only'],
-                              inner_jobs=inner_jobs)
-
-  test_jobs += _generate_jobs(languages=['python'],
-                              configs=['dbg'],
-                              platforms=['linux'],
-                              arch='default',
-                              compiler='python_alpine',
-                              labels=['portability', 'multilang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs)
-
-  test_jobs += _generate_jobs(languages=['csharp'],
-                              configs=['dbg'],
-                              platforms=['linux'],
-                              arch='default',
-                              compiler='coreclr',
-                              labels=['portability', 'multilang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs)
-
-  test_jobs += _generate_jobs(languages=['c'],
-                              configs=['dbg'],
-                              platforms=['linux'],
-                              iomgr_platform='uv',
-                              labels=['portability', 'corelang'],
-                              extra_args=extra_args,
-                              inner_jobs=inner_jobs,
-                              timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
-
-  return test_jobs
+    test_jobs = []
+    # supported on linux only
+    test_jobs += _generate_jobs(
+        languages=['sanity', 'php7'],
+        configs=['dbg', 'opt'],
+        platforms=['linux'],
+        labels=['basictests', 'multilang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    # supported on all platforms.
+    test_jobs += _generate_jobs(
+        languages=['c'],
+        configs=['dbg', 'opt'],
+        platforms=['linux', 'macos', 'windows'],
+        labels=['basictests', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs,
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+
+    test_jobs += _generate_jobs(
+        languages=['csharp', 'python'],
+        configs=['dbg', 'opt'],
+        platforms=['linux', 'macos', 'windows'],
+        labels=['basictests', 'multilang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    # supported on linux and mac.
+    test_jobs += _generate_jobs(
+        languages=['c++'],
+        configs=['dbg', 'opt'],
+        platforms=['linux', 'macos'],
+        labels=['basictests', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs,
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+
+    test_jobs += _generate_jobs(
+        languages=['grpc-node', 'ruby', 'php'],
+        configs=['dbg', 'opt'],
+        platforms=['linux', 'macos'],
+        labels=['basictests', 'multilang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    # supported on mac only.
+    test_jobs += _generate_jobs(
+        languages=['objc'],
+        configs=['dbg', 'opt'],
+        platforms=['macos'],
+        labels=['basictests', 'multilang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    # sanitizers
+    test_jobs += _generate_jobs(
+        languages=['c'],
+        configs=['msan', 'asan', 'tsan', 'ubsan'],
+        platforms=['linux'],
+        labels=['sanitizers', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs,
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+    test_jobs += _generate_jobs(
+        languages=['c++'],
+        configs=['asan'],
+        platforms=['linux'],
+        labels=['sanitizers', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs,
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+    test_jobs += _generate_jobs(
+        languages=['c++'],
+        configs=['tsan'],
+        platforms=['linux'],
+        labels=['sanitizers', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs,
+        timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT)
+
+    return test_jobs
+
+
+def _create_portability_test_jobs(extra_args=[],
+                                  inner_jobs=_DEFAULT_INNER_JOBS):
+    test_jobs = []
+    # portability C x86
+    test_jobs += _generate_jobs(
+        languages=['c'],
+        configs=['dbg'],
+        platforms=['linux'],
+        arch='x86',
+        compiler='default',
+        labels=['portability', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    # portability C and C++ on x64
+    for compiler in [
+            'gcc4.8', 'gcc5.3', 'gcc_musl', 'clang3.5', 'clang3.6', 'clang3.7'
+    ]:
+        test_jobs += _generate_jobs(
+            languages=['c', 'c++'],
+            configs=['dbg'],
+            platforms=['linux'],
+            arch='x64',
+            compiler=compiler,
+            labels=['portability', 'corelang'],
+            extra_args=extra_args,
+            inner_jobs=inner_jobs,
+            timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+
+    # portability C on Windows 64-bit (x86 is the default)
+    test_jobs += _generate_jobs(
+        languages=['c'],
+        configs=['dbg'],
+        platforms=['windows'],
+        arch='x64',
+        compiler='default',
+        labels=['portability', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    # portability C++ on Windows
+    # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+    test_jobs += _generate_jobs(
+        languages=['c++'],
+        configs=['dbg'],
+        platforms=['windows'],
+        arch='default',
+        compiler='default',
+        labels=['portability', 'corelang'],
+        extra_args=extra_args + ['--build_only'],
+        inner_jobs=inner_jobs)
+
+    # portability C and C++ on Windows using VS2017 (build only)
+    # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+    test_jobs += _generate_jobs(
+        languages=['c', 'c++'],
+        configs=['dbg'],
+        platforms=['windows'],
+        arch='x64',
+        compiler='cmake_vs2017',
+        labels=['portability', 'corelang'],
+        extra_args=extra_args + ['--build_only'],
+        inner_jobs=inner_jobs)
+
+    # C and C++ with the c-ares DNS resolver on Linux
+    test_jobs += _generate_jobs(
+        languages=['c', 'c++'],
+        configs=['dbg'],
+        platforms=['linux'],
+        labels=['portability', 'corelang'],
+        extra_args=extra_args,
+        extra_envs={'GRPC_DNS_RESOLVER': 'ares'},
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+
+    # TODO(zyc): Turn on this test after adding c-ares support on windows.
+    # C with the c-ares DNS resolver on Windows
+    # test_jobs += _generate_jobs(languages=['c'],
+    #                             configs=['dbg'], platforms=['windows'],
+    #                             labels=['portability', 'corelang'],
+    #                             extra_args=extra_args,
+    #                             extra_envs={'GRPC_DNS_RESOLVER': 'ares'})
+
+    # C and C++ build with cmake on Linux
+    # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+    # to make sure it's buildable at least.
+    test_jobs += _generate_jobs(
+        languages=['c', 'c++'],
+        configs=['dbg'],
+        platforms=['linux'],
+        arch='default',
+        compiler='cmake',
+        labels=['portability', 'corelang'],
+        extra_args=extra_args + ['--build_only'],
+        inner_jobs=inner_jobs)
+
+    test_jobs += _generate_jobs(
+        languages=['python'],
+        configs=['dbg'],
+        platforms=['linux'],
+        arch='default',
+        compiler='python_alpine',
+        labels=['portability', 'multilang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    test_jobs += _generate_jobs(
+        languages=['csharp'],
+        configs=['dbg'],
+        platforms=['linux'],
+        arch='default',
+        compiler='coreclr',
+        labels=['portability', 'multilang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs)
+
+    test_jobs += _generate_jobs(
+        languages=['c'],
+        configs=['dbg'],
+        platforms=['linux'],
+        iomgr_platform='uv',
+        labels=['portability', 'corelang'],
+        extra_args=extra_args,
+        inner_jobs=inner_jobs,
+        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+
+    return test_jobs
 
 
 def _allowed_labels():
-  """Returns a list of existing job labels."""
-  all_labels = set()
-  for job in _create_test_jobs() + _create_portability_test_jobs():
-    for label in job.labels:
-      all_labels.add(label)
-  return sorted(all_labels)
+    """Returns a list of existing job labels."""
+    all_labels = set()
+    for job in _create_test_jobs() + _create_portability_test_jobs():
+        for label in job.labels:
+            all_labels.add(label)
+    return sorted(all_labels)
 
 
 def _runs_per_test_type(arg_str):
-  """Auxiliary function to parse the "runs_per_test" flag."""
-  try:
-    n = int(arg_str)
-    if n <= 0: raise ValueError
-    return n
-  except:
-    msg = '\'{}\' is not a positive integer'.format(arg_str)
-    raise argparse.ArgumentTypeError(msg)
+    """Auxiliary function to parse the "runs_per_test" flag."""
+    try:
+        n = int(arg_str)
+        if n <= 0: raise ValueError
+        return n
+    except:
+        msg = '\'{}\' is not a positive integer'.format(arg_str)
+        raise argparse.ArgumentTypeError(msg)
 
 
 if __name__ == "__main__":
-  argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
-  argp.add_argument('-j', '--jobs',
-                    default=multiprocessing.cpu_count()/_DEFAULT_INNER_JOBS,
-                    type=int,
-                    help='Number of concurrent run_tests.py instances.')
-  argp.add_argument('-f', '--filter',
-                    choices=_allowed_labels(),
-                    nargs='+',
-                    default=[],
-                    help='Filter targets to run by label with AND semantics.')
-  argp.add_argument('--exclude',
-                    choices=_allowed_labels(),
-                    nargs='+',
-                    default=[],
-                    help='Exclude targets with any of given labels.')
-  argp.add_argument('--build_only',
-                    default=False,
-                    action='store_const',
-                    const=True,
-                    help='Pass --build_only flag to run_tests.py instances.')
-  argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
-                    help='Pass --force_default_poller to run_tests.py instances.')
-  argp.add_argument('--dry_run',
-                    default=False,
-                    action='store_const',
-                    const=True,
-                    help='Only print what would be run.')
-  argp.add_argument('--filter_pr_tests',
-                    default=False,
-                    action='store_const',
-                    const=True,
-                    help='Filters out tests irrelevant to pull request changes.')
-  argp.add_argument('--base_branch',
-                    default='origin/master',
-                    type=str,
-                    help='Branch that pull request is requesting to merge into')
-  argp.add_argument('--inner_jobs',
-                    default=_DEFAULT_INNER_JOBS,
-                    type=int,
-                    help='Number of jobs in each run_tests.py instance')
-  argp.add_argument('-n', '--runs_per_test', default=1, type=_runs_per_test_type,
-                    help='How many times to run each tests. >1 runs implies ' +
-                    'omitting passing test from the output & reports.')
-  argp.add_argument('--max_time', default=-1, type=int,
-                    help='Maximum amount of time to run tests for' +
-                         '(other tests will be skipped)')
-  argp.add_argument('--internal_ci',
-                    default=False,
-                    action='store_const',
-                    const=True,
-                    help='Put reports into subdirectories to improve presentation of '
-                    'results by Internal CI.')
-  argp.add_argument('--bq_result_table',
-                    default='',
-                    type=str,
-                    nargs='?',
-                    help='Upload test results to a specified BQ table.')
-  args = argp.parse_args()
-
-  if args.internal_ci:
-    _report_filename = _report_filename_internal_ci  # override the function
-
-  extra_args = []
-  if args.build_only:
-    extra_args.append('--build_only')
-  if args.force_default_poller:
-    extra_args.append('--force_default_poller')
-  if args.runs_per_test > 1:
-    extra_args.append('-n')
-    extra_args.append('%s' % args.runs_per_test)
-    extra_args.append('--quiet_success')
-  if args.max_time > 0:
-    extra_args.extend(('--max_time', '%d' % args.max_time))
-  if args.bq_result_table:
-    extra_args.append('--bq_result_table')
-    extra_args.append('%s' % args.bq_result_table)
-    extra_args.append('--measure_cpu_costs')
-    extra_args.append('--disable_auto_set_flakes')
-
-  all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
-             _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
-
-  jobs = []
-  for job in all_jobs:
-    if not args.filter or all(filter in job.labels for filter in args.filter):
-      if not any(exclude_label in job.labels for exclude_label in args.exclude):
-        jobs.append(job)
-
-  if not jobs:
-    jobset.message('FAILED', 'No test suites match given criteria.',
-                   do_newline=True)
-    sys.exit(1)
-
-  print('IMPORTANT: The changes you are testing need to be locally committed')
-  print('because only the committed changes in the current branch will be')
-  print('copied to the docker environment or into subworkspaces.')
-
-  skipped_jobs = []
-
-  if args.filter_pr_tests:
-    print('Looking for irrelevant tests to skip...')
-    relevant_jobs = filter_tests(jobs, args.base_branch)
-    if len(relevant_jobs) == len(jobs):
-      print('No tests will be skipped.')
-    else:
-      print('These tests will be skipped:')
-      skipped_jobs = list(set(jobs) - set(relevant_jobs))
-      # Sort by shortnames to make printing of skipped tests consistent
-      skipped_jobs.sort(key=lambda job: job.shortname)
-      for job in list(skipped_jobs):
-        print('  %s' % job.shortname)
-    jobs = relevant_jobs
-
-  print('Will run these tests:')
-  for job in jobs:
+    argp = argparse.ArgumentParser(
+        description='Run a matrix of run_tests.py tests.')
+    argp.add_argument(
+        '-j',
+        '--jobs',
+        default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS,
+        type=int,
+        help='Number of concurrent run_tests.py instances.')
+    argp.add_argument(
+        '-f',
+        '--filter',
+        choices=_allowed_labels(),
+        nargs='+',
+        default=[],
+        help='Filter targets to run by label with AND semantics.')
+    argp.add_argument(
+        '--exclude',
+        choices=_allowed_labels(),
+        nargs='+',
+        default=[],
+        help='Exclude targets with any of given labels.')
+    argp.add_argument(
+        '--build_only',
+        default=False,
+        action='store_const',
+        const=True,
+        help='Pass --build_only flag to run_tests.py instances.')
+    argp.add_argument(
+        '--force_default_poller',
+        default=False,
+        action='store_const',
+        const=True,
+        help='Pass --force_default_poller to run_tests.py instances.')
+    argp.add_argument(
+        '--dry_run',
+        default=False,
+        action='store_const',
+        const=True,
+        help='Only print what would be run.')
+    argp.add_argument(
+        '--filter_pr_tests',
+        default=False,
+        action='store_const',
+        const=True,
+        help='Filters out tests irrelevant to pull request changes.')
+    argp.add_argument(
+        '--base_branch',
+        default='origin/master',
+        type=str,
+        help='Branch that pull request is requesting to merge into')
+    argp.add_argument(
+        '--inner_jobs',
+        default=_DEFAULT_INNER_JOBS,
+        type=int,
+        help='Number of jobs in each run_tests.py instance')
+    argp.add_argument(
+        '-n',
+        '--runs_per_test',
+        default=1,
+        type=_runs_per_test_type,
+        help='How many times to run each tests. >1 runs implies ' +
+        'omitting passing test from the output & reports.')
+    argp.add_argument(
+        '--max_time',
+        default=-1,
+        type=int,
+        help='Maximum amount of time to run tests for' +
+        '(other tests will be skipped)')
+    argp.add_argument(
+        '--internal_ci',
+        default=False,
+        action='store_const',
+        const=True,
+        help='Put reports into subdirectories to improve presentation of '
+        'results by Internal CI.')
+    argp.add_argument(
+        '--bq_result_table',
+        default='',
+        type=str,
+        nargs='?',
+        help='Upload test results to a specified BQ table.')
+    args = argp.parse_args()
+
+    if args.internal_ci:
+        _report_filename = _report_filename_internal_ci  # override the function
+
+    extra_args = []
+    if args.build_only:
+        extra_args.append('--build_only')
+    if args.force_default_poller:
+        extra_args.append('--force_default_poller')
+    if args.runs_per_test > 1:
+        extra_args.append('-n')
+        extra_args.append('%s' % args.runs_per_test)
+        extra_args.append('--quiet_success')
+    if args.max_time > 0:
+        extra_args.extend(('--max_time', '%d' % args.max_time))
+    if args.bq_result_table:
+        extra_args.append('--bq_result_table')
+        extra_args.append('%s' % args.bq_result_table)
+        extra_args.append('--measure_cpu_costs')
+        extra_args.append('--disable_auto_set_flakes')
+
+    all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
+               _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
+
+    jobs = []
+    for job in all_jobs:
+        if not args.filter or all(filter in job.labels
+                                  for filter in args.filter):
+            if not any(exclude_label in job.labels
+                       for exclude_label in args.exclude):
+                jobs.append(job)
+
+    if not jobs:
+        jobset.message(
+            'FAILED', 'No test suites match given criteria.', do_newline=True)
+        sys.exit(1)
+
+    print('IMPORTANT: The changes you are testing need to be locally committed')
+    print('because only the committed changes in the current branch will be')
+    print('copied to the docker environment or into subworkspaces.')
+
+    skipped_jobs = []
+
+    if args.filter_pr_tests:
+        print('Looking for irrelevant tests to skip...')
+        relevant_jobs = filter_tests(jobs, args.base_branch)
+        if len(relevant_jobs) == len(jobs):
+            print('No tests will be skipped.')
+        else:
+            print('These tests will be skipped:')
+            skipped_jobs = list(set(jobs) - set(relevant_jobs))
+            # Sort by shortnames to make printing of skipped tests consistent
+            skipped_jobs.sort(key=lambda job: job.shortname)
+            for job in list(skipped_jobs):
+                print('  %s' % job.shortname)
+        jobs = relevant_jobs
+
+    print('Will run these tests:')
+    for job in jobs:
+        if args.dry_run:
+            print('  %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
+        else:
+            print('  %s' % job.shortname)
+    print
+
     if args.dry_run:
-      print('  %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
+        print('--dry_run was used, exiting')
+        sys.exit(1)
+
+    jobset.message('START', 'Running test matrix.', do_newline=True)
+    num_failures, resultset = jobset.run(
+        jobs, newline_on_success=True, travis=True, maxjobs=args.jobs)
+    # Merge skipped tests into results to show skipped tests on report.xml
+    if skipped_jobs:
+        ignored_num_skipped_failures, skipped_results = jobset.run(
+            skipped_jobs, skip_jobs=True)
+        resultset.update(skipped_results)
+    report_utils.render_junit_xml_report(
+        resultset,
+        _report_filename('aggregate_tests'),
+        suite_name='aggregate_tests')
+
+    if num_failures == 0:
+        jobset.message(
+            'SUCCESS',
+            'All run_tests.py instance finished successfully.',
+            do_newline=True)
     else:
-      print('  %s' % job.shortname)
-  print
-
-  if args.dry_run:
-    print('--dry_run was used, exiting')
-    sys.exit(1)
-
-  jobset.message('START', 'Running test matrix.', do_newline=True)
-  num_failures, resultset = jobset.run(jobs,
-                                       newline_on_success=True,
-                                       travis=True,
-                                       maxjobs=args.jobs)
-  # Merge skipped tests into results to show skipped tests on report.xml
-  if skipped_jobs:
-    ignored_num_skipped_failures, skipped_results = jobset.run(
-        skipped_jobs, skip_jobs=True)
-    resultset.update(skipped_results)
-  report_utils.render_junit_xml_report(resultset, _report_filename('aggregate_tests'),
-                                       suite_name='aggregate_tests')
-
-  if num_failures == 0:
-    jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
-                   do_newline=True)
-  else:
-    jobset.message('FAILED', 'Some run_tests.py instance have failed.',
-                   do_newline=True)
-    sys.exit(1)
+        jobset.message(
+            'FAILED',
+            'Some run_tests.py instance have failed.',
+            do_newline=True)
+        sys.exit(1)

tools/run_tests/start_port_server.py

@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """
 Wrapper around port server starting code.
 

tools/run_tests/task_runner.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Runs selected gRPC test/build tasks."""
 
 from __future__ import print_function
@@ -32,52 +31,54 @@ _TARGETS += artifact_targets.targets()
 _TARGETS += distribtest_targets.targets()
 _TARGETS += package_targets.targets()
 
+
 def _create_build_map():
-  """Maps task names and labels to list of tasks to be built."""
-  target_build_map = dict([(target.name, [target])
-                           for target in _TARGETS])
-  if len(_TARGETS) > len(target_build_map.keys()):
-    raise Exception('Target names need to be unique')
-
-  label_build_map = {}
-  label_build_map['all'] = [t for t in _TARGETS]  # to build all targets
-  for target in _TARGETS:
-    for label in target.labels:
-      if label in label_build_map:
-        label_build_map[label].append(target)
-      else:
-        label_build_map[label] = [target]
-
-  if set(target_build_map.keys()).intersection(label_build_map.keys()):
-    raise Exception('Target names need to be distinct from label names')
-  return dict( target_build_map.items() + label_build_map.items())
+    """Maps task names and labels to list of tasks to be built."""
+    target_build_map = dict([(target.name, [target]) for target in _TARGETS])
+    if len(_TARGETS) > len(target_build_map.keys()):
+        raise Exception('Target names need to be unique')
+
+    label_build_map = {}
+    label_build_map['all'] = [t for t in _TARGETS]  # to build all targets
+    for target in _TARGETS:
+        for label in target.labels:
+            if label in label_build_map:
+                label_build_map[label].append(target)
+            else:
+                label_build_map[label] = [target]
+
+    if set(target_build_map.keys()).intersection(label_build_map.keys()):
+        raise Exception('Target names need to be distinct from label names')
+    return dict(target_build_map.items() + label_build_map.items())
 
 
 _BUILD_MAP = _create_build_map()
 
 argp = argparse.ArgumentParser(description='Runs build/test targets.')
-argp.add_argument('-b', '--build',
-                  choices=sorted(_BUILD_MAP.keys()),
-                  nargs='+',
-                  default=['all'],
-                  help='Target name or target label to build.')
-argp.add_argument('-f', '--filter',
-                  choices=sorted(_BUILD_MAP.keys()),
-                  nargs='+',
-                  default=[],
-                  help='Filter targets to build with AND semantics.')
+argp.add_argument(
+    '-b',
+    '--build',
+    choices=sorted(_BUILD_MAP.keys()),
+    nargs='+',
+    default=['all'],
+    help='Target name or target label to build.')
+argp.add_argument(
+    '-f',
+    '--filter',
+    choices=sorted(_BUILD_MAP.keys()),
+    nargs='+',
+    default=[],
+    help='Filter targets to build with AND semantics.')
 argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
-argp.add_argument('-t', '--travis',
-                  default=False,
-                  action='store_const',
-                  const=True)
+argp.add_argument(
+    '-t', '--travis', default=False, action='store_const', const=True)
 
 args = argp.parse_args()
 
 # Figure out which targets to build
 targets = []
 for label in args.build:
-  targets += _BUILD_MAP[label]
+    targets += _BUILD_MAP[label]
 
 # Among targets selected by -b, filter out those that don't match the filter
 targets = [t for t in targets if all(f in t.labels for f in args.filter)]
@@ -86,30 +87,29 @@ targets = sorted(set(targets))
 # Execute pre-build phase
 prebuild_jobs = []
 for target in targets:
-  prebuild_jobs += target.pre_build_jobspecs()
+    prebuild_jobs += target.pre_build_jobspecs()
 if prebuild_jobs:
-  num_failures, _ = jobset.run(
-    prebuild_jobs, newline_on_success=True, maxjobs=args.jobs)
-  if num_failures != 0:
-    jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True)
-    sys.exit(1)
+    num_failures, _ = jobset.run(
+        prebuild_jobs, newline_on_success=True, maxjobs=args.jobs)
+    if num_failures != 0:
+        jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True)
+        sys.exit(1)
 
 build_jobs = []
 for target in targets:
-  build_jobs.append(target.build_jobspec())
+    build_jobs.append(target.build_jobspec())
 if not build_jobs:
-  print('Nothing to build.')
-  sys.exit(1)
+    print('Nothing to build.')
+    sys.exit(1)
 
 jobset.message('START', 'Building targets.', do_newline=True)
 num_failures, resultset = jobset.run(
     build_jobs, newline_on_success=True, maxjobs=args.jobs)
-report_utils.render_junit_xml_report(resultset, 'report_taskrunner_sponge_log.xml',
-                                     suite_name='tasks')
+report_utils.render_junit_xml_report(
+    resultset, 'report_taskrunner_sponge_log.xml', suite_name='tasks')
 if num_failures == 0:
-  jobset.message('SUCCESS', 'All targets built successfully.',
-                 do_newline=True)
+    jobset.message(
+        'SUCCESS', 'All targets built successfully.', do_newline=True)
 else:
-  jobset.message('FAILED', 'Failed to build targets.',
-                 do_newline=True)
-  sys.exit(1)
+    jobset.message('FAILED', 'Failed to build targets.', do_newline=True)
+    sys.exit(1)