Merge pull request #865 from nicolasnoble/flaky

Adding flaky attribute for tests.

build.json

@@ -866,7 +866,8 @@
         "grpc",
         "gpr_test_util",
         "gpr"
-      ]
+      ],
+      "flaky": true
     },
     {
       "name": "fling_test",
@@ -880,7 +881,8 @@
         "grpc",
         "gpr_test_util",
         "gpr"
-      ]
+      ],
+      "flaky": true
     },
     {
       "name": "gen_hpack_tables",

templates/tools/run_tests/tests.json.template

@@ -2,7 +2,9 @@
 import json
 %>
 
-${json.dumps([{"name": tgt.name, "language": tgt.language}
+${json.dumps([{"name": tgt.name,
+               "language": tgt.language,
+               "flaky": tgt.get("flaky", False)}
               for tgt in targets
               if tgt.get('run', True) and tgt.build == 'test'],
              sort_keys=True, indent=2)}

tools/run_tests/run_tests.py

@@ -83,14 +83,14 @@ class CLanguage(object):
     self.make_target = make_target
     with open('tools/run_tests/tests.json') as f:
       js = json.load(f)
-      self.binaries = [tgt['name']
+      self.binaries = [tgt for tgt in js if tgt['language'] == test_lang]
-                       for tgt in js
-                       if tgt['language'] == test_lang]
 
-  def test_specs(self, config):
+  def test_specs(self, config, travis):
     out = []
-    for name in self.binaries:
+    for target in self.binaries:
-      binary = 'bins/%s/%s' % (config.build_config, name)
+      if travis and target['flaky']:
+        continue
+      binary = 'bins/%s/%s' % (config.build_config, target['name'])
       out.append(config.job_spec(binary, [binary]))
     return out
 
@@ -103,7 +103,7 @@ class CLanguage(object):
 
 class NodeLanguage(object):
 
-  def test_specs(self, config):
+  def test_specs(self, config, travis):
     return [config.job_spec('tools/run_tests/run_node.sh', None)]
 
   def make_targets(self):
@@ -115,7 +115,7 @@ class NodeLanguage(object):
 
 class PhpLanguage(object):
 
-  def test_specs(self, config):
+  def test_specs(self, config, travis):
     return [config.job_spec('src/php/bin/run_tests.sh', None)]
 
   def make_targets(self):
@@ -127,7 +127,7 @@ class PhpLanguage(object):
 
 class PythonLanguage(object):
 
-  def test_specs(self, config):
+  def test_specs(self, config, travis):
     return [config.job_spec('tools/run_tests/run_python.sh', None)]
 
   def make_targets(self):
@@ -211,7 +211,7 @@ one_run = set(
     spec
     for config in run_configs
     for language in args.language
-    for spec in _LANGUAGES[language].test_specs(config)
+    for spec in _LANGUAGES[language].test_specs(config, args.travis)
     if re.search(args.regex, spec.shortname))
 
 runs_per_test = args.runs_per_test