Merge pull request #2225 from nicolasnoble/make-runtest-output-junit-reports

Make runtest output junit reports

tools/jenkins/docker_run_jenkins.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This script is invoked by run_jekins.sh when piggy-backing into docker.
+set -e
+
+export CONFIG=$config
+export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5
+export CPPFLAGS=-I/tmp/prebuilt/include
+
+mkdir -p /var/local/git
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+cd /var/local/git/grpc
+nvm use 0.12
+rvm use ruby-2.1
+tools/run_tests/prepare_travis.sh
+tools/run_tests/run_tests.py -t -c $config -l $language -x report.xml

tools/jenkins/grpc_jenkins_slave/Dockerfile

@@ -136,5 +136,7 @@ RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add -
 RUN apt-get update && apt-get install -y \
     git php5 php5-dev phpunit unzip
 
+RUN mkdir /var/local/jenkins
+
 # Define the default command.
 CMD ["bash"]

tools/jenkins/run_jenkins.sh

@@ -41,6 +41,10 @@ if [ "$platform" == "linux" ]
 then
   echo "building $language on Linux"
 
+  cd `dirname $0`/../..
+  git_root=`pwd`
+  cd -
+
   # Use image name based on Dockerfile checksum
   DOCKER_IMAGE_NAME=grpc_jenkins_slave_`sha1sum tools/jenkins/grpc_jenkins_slave/Dockerfile | cut -f1 -d\ `
 
@@ -57,17 +61,18 @@ then
   rm -f docker.cid
 
   # Run tests inside docker
-  docker run --cidfile=docker.cid $DOCKER_IMAGE_NAME bash -c -l "git clone --recursive $GIT_URL /var/local/git/grpc \
-    && cd /var/local/git/grpc \
-    $FETCH_PULL_REQUEST_CMD \
-    && git checkout -f $GIT_COMMIT \
-    && git submodule update \
-    && nvm use 0.12 \
-    && rvm use ruby-2.1 \
-    && CONFIG=$config tools/run_tests/prepare_travis.sh \
-    && CPPFLAGS=-I/tmp/prebuilt/include tools/run_tests/run_tests.py -t -c $config -l $language" || DOCKER_FAILED="true"
+  docker run \
+    -e "config=$config" \
+    -e "language=$language" \
+    -i \
+    -v "$git_root:/var/local/jenkins/grpc" \
+    --cidfile=docker.cid \
+    $DOCKER_IMAGE_NAME \
+    bash -l /var/local/jenkins/grpc/tools/jenkins/docker_run_jenkins.sh || DOCKER_FAILED="true"
 
   DOCKER_CID=`cat docker.cid`
+  docker kill $DOCKER_CID
+  docker cp $DOCKER_CID:/var/local/git/grpc/report.xml $git_root
   if [ "$DOCKER_FAILED" == "" ]
   then
     echo "Docker finished successfully, deleting the container $DOCKER_CID"
@@ -89,7 +94,7 @@ then
   /cygdrive/c/nuget/nuget.exe restore vsprojects/grpc.sln
   /cygdrive/c/nuget/nuget.exe restore src/csharp/Grpc.sln
 
-  python tools/run_tests/run_tests.py -t -l $language
+  python tools/run_tests/run_tests.py -t -l $language -x report.xml
 else
   echo "Unknown platform $platform"
   exit 1

tools/run_tests/jobset.py

@@ -34,10 +34,12 @@ import multiprocessing
 import os
 import platform
 import signal
+import string
 import subprocess
 import sys
 import tempfile
 import time
+import xml.etree.cElementTree as ET
 
 
 _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
@@ -159,7 +161,7 @@ class JobSpec(object):
 class Job(object):
   """Manages one job."""
 
-  def __init__(self, spec, bin_hash, newline_on_success, travis):
+  def __init__(self, spec, bin_hash, newline_on_success, travis, xml_report):
     self._spec = spec
     self._bin_hash = bin_hash
     self._tempfile = tempfile.TemporaryFile()
@@ -176,19 +178,27 @@ class Job(object):
     self._state = _RUNNING
     self._newline_on_success = newline_on_success
     self._travis = travis
+    self._xml_test = ET.SubElement(xml_report, 'testcase',
+                                   name=self._spec.shortname) if xml_report is not None else None
     message('START', spec.shortname, do_newline=self._travis)
 
   def state(self, update_cache):
     """Poll current state of the job. Prints messages at completion."""
     if self._state == _RUNNING and self._process.poll() is not None:
       elapsed = time.time() - self._start
-      if self._process.returncode != 0:
-        self._state = _FAILURE
       self._tempfile.seek(0)
       stdout = self._tempfile.read()
+      filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))
+      if self._xml_test is not None:
+        self._xml_test.set('time', str(elapsed))
+        ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
+      if self._process.returncode != 0:
+        self._state = _FAILURE
         message('FAILED', '%s [ret=%d, pid=%d]' % (
             self._spec.shortname, self._process.returncode, self._process.pid),
             stdout, do_newline=True)
+        if self._xml_test is not None:
+          ET.SubElement(self._xml_test, 'failure', message='Failure').text
       else:
         self._state = _SUCCESS
         message('PASSED', '%s [time=%.1fsec]' % (self._spec.shortname, elapsed),
@@ -200,6 +210,9 @@ class Job(object):
       stdout = self._tempfile.read()
       message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
       self.kill()
+      if self._xml_test is not None:
+        ET.SubElement(self._xml_test, 'system-out').text = stdout
+        ET.SubElement(self._xml_test, 'error', message='Timeout')
     return self._state
 
   def kill(self):
@@ -212,7 +225,7 @@ class Jobset(object):
   """Manages one run of jobs."""
 
   def __init__(self, check_cancelled, maxjobs, newline_on_success, travis,
-               stop_on_failure, cache):
+               stop_on_failure, cache, xml_report):
     self._running = set()
     self._check_cancelled = check_cancelled
     self._cancelled = False
@@ -224,6 +237,7 @@ class Jobset(object):
     self._cache = cache
     self._stop_on_failure = stop_on_failure
     self._hashes = {}
+    self._xml_report = xml_report
 
   def start(self, spec):
     """Start a job. Return True on success, False on failure."""
@@ -250,7 +264,8 @@ class Jobset(object):
         self._running.add(Job(spec,
                               bin_hash,
                               self._newline_on_success,
-                              self._travis))
+                              self._travis,
+                              self._xml_report))
       except:
         message('FAILED', spec.shortname)
         self._cancelled = True
@@ -324,11 +339,13 @@ def run(cmdlines,
         travis=False,
         infinite_runs=False,
         stop_on_failure=False,
-        cache=None):
+        cache=None,
+        xml_report=None):
   js = Jobset(check_cancelled,
               maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
               newline_on_success, travis, stop_on_failure,
-              cache if cache is not None else NoCache())
+              cache if cache is not None else NoCache(),
+              xml_report)
   for cmdline in cmdlines:
     if not js.start(cmdline):
       break

tools/run_tests/run_tests.py

@@ -42,6 +42,7 @@ import re
 import subprocess
 import sys
 import time
+import xml.etree.cElementTree as ET
 
 import jobset
 import watch_dirs
@@ -397,6 +398,8 @@ argp.add_argument('-S', '--stop_on_failure',
                   action='store_const',
                   const=True)
 argp.add_argument('-a', '--antagonists', default=0, type=int)
+argp.add_argument('-x', '--xml_report', default=None, type=str,
+        help='Generates a JUnit-compatible XML report')
 args = argp.parse_args()
 
 # grab config
@@ -493,7 +496,7 @@ class TestCache(object):
         self.parse(json.loads(f.read()))
 
 
-def _build_and_run(check_cancelled, newline_on_success, travis, cache):
+def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_report):
   """Do one pass of building & running tests."""
   # build latest sequentially
   if not jobset.run(build_steps, maxjobs=1,
@@ -519,16 +522,24 @@ def _build_and_run(check_cancelled, newline_on_success, travis, cache):
     runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs
                      else itertools.repeat(massaged_one_run, runs_per_test))
     all_runs = itertools.chain.from_iterable(runs_sequence)
+
+    root = ET.Element('testsuites') if xml_report else None
+    testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None
+
     if not jobset.run(all_runs, check_cancelled,
                       newline_on_success=newline_on_success, travis=travis,
                       infinite_runs=infinite_runs,
                       maxjobs=args.jobs,
                       stop_on_failure=args.stop_on_failure,
-                      cache=cache):
+                      cache=cache if not xml_report else None,
+                      xml_report=testsuite):
       return 2
   finally:
     for antagonist in antagonists:
       antagonist.kill()
+    if xml_report:
+      tree = ET.ElementTree(root)
+      tree.write(xml_report, encoding='UTF-8')
 
   if cache: cache.save()
 
@@ -560,7 +571,8 @@ else:
   result = _build_and_run(check_cancelled=lambda: False,
                           newline_on_success=args.newline_on_success,
                           travis=args.travis,
-                          cache=test_cache)
+                          cache=test_cache,
+                          xml_report=args.xml_report)
   if result == 0:
     jobset.message('SUCCESS', 'All tests passed', do_newline=True)
   else: