From 38bc15552de259fa99c8e9654bed0ff8433902eb Mon Sep 17 00:00:00 2001
From: Josh Haberman <jhaberman@gmail.com>
Date: Tue, 23 Feb 2016 14:28:02 -0800
Subject: [PATCH] Added code to generate XML output file for more granular
 results.

---
 python/tox.ini                    |  4 --
 tools/docker/Dockerfile           |  2 +-
 tools/jenkins/make_test_output.py | 90 +++++++++++++++++++++++++++++++
 tools/jenkins/pull_request.sh     |  1 +
 tools/run_tests/jenkins.sh        | 36 ++++++++++---
 5 files changed, 122 insertions(+), 11 deletions(-)
 create mode 100644 tools/jenkins/make_test_output.py

diff --git a/python/tox.ini b/python/tox.ini
index f663b5387b..cf8d540168 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -6,10 +6,6 @@ envlist =
 usedevelop=true
 passenv = CC
 setenv =
-    # Dummy entry works around bug where tox fails for empty "setenv" section
-    # (since cpp lines aren't used for py builds).
-    # https://bitbucket.org/hpk42/tox/issues/190/generative-setenv-fails-if-there-s-only
-    DUMMY=dummy
     cpp: LD_LIBRARY_PATH={toxinidir}/../src/.libs
     cpp: DYLD_LIBRARY_PATH={toxinidir}/../src/.libs
     cpp: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile
index 5783056528..bceb60449e 100644
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@@ -98,7 +98,7 @@ RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe
 # to get updates from pip.
 
 RUN pip install pip --upgrade
-RUN pip install virtualenv tox
+RUN pip install virtualenv tox yattag
 
 
 ##################
diff --git a/tools/jenkins/make_test_output.py b/tools/jenkins/make_test_output.py
new file mode 100644
index 0000000000..0ba5db3bf3
--- /dev/null
+++ b/tools/jenkins/make_test_output.py
@@ -0,0 +1,90 @@
+"""Gather output from test runs and create an XML file in JUnit format.
+
+The output files from the individual tests have been written in a directory
+structure like:
+
+  $DIR/joblog  (--joblog from "parallel")
+  $DIR/logs/1/cpp/stdout
+  $DIR/logs/1/cpp/stderr
+  $DIR/logs/1/csharp/stdout
+  $DIR/logs/1/csharp/stderr
+  $DIR/logs/1/java_jdk7/stdout
+  $DIR/logs/1/java_jdk7/stderr
+  etc.
+
+This script bundles them into a single output XML file so Jenkins can show
+detailed test results.
+"""
+
+import os;
+import sys;
+from yattag import Doc
+from collections import defaultdict
+
+def readtests(basedir):
+  tests = defaultdict(dict)
+
+  # Sample input (note: separators are tabs).
+  #
+  # Seq	Host	Starttime	Runtime	Send	Receive	Exitval	Signal	Command
+  # 1	:	1456263838.313	0.005	0	0	0	0	echo A
+  with open(basedir + "/joblog") as jobs:
+    firstline = next(jobs)
+    for line in jobs:
+      values = line.split("\t")
+
+      name = values[8].split()[-1]
+      test = tests[name]
+      test["name"] = name
+      test["time"] = values[3]
+
+      exitval = values[6]
+      if int(exitval):
+        # We don't have a more specific message.  User should look at stderr.
+        test["failure"] = "TEST FAILURE"
+      else:
+        test["failure"] = False
+
+  for testname in os.listdir(basedir + "/logs/1"):
+    test = tests[testname]
+
+    with open(basedir + "/logs/1/" + testname + "/stdout") as f:
+      test["stdout"] = f.read()
+
+    with open(basedir + "/logs/1/" + testname + "/stderr") as f:
+      test["stderr"] = f.read()
+
+  # The cpp test is special since it doesn't run under parallel so doesn't show
+  # up in the job log.
+  tests["cpp"]["name"] = "cpp"
+
+  # TODO
+  tests["cpp"]["time"] = "0"
+  tests["cpp"]["failure"] = False
+
+  ret = tests.values()
+  ret.sort(key=lambda x: x["name"])
+
+  return ret
+
+def genxml(tests):
+  doc, tag, text = Doc().tagtext()
+
+  with tag("testsuites"):
+    with tag("testsuite", name="Protobuf Tests"):
+      for test in tests:
+        with tag("testcase", name=test["name"], classname=test["name"],
+                             time=test["time"]):
+          with tag("system-out"):
+            text(test["stdout"])
+          with tag("system-err"):
+            text(test["stderr"])
+          if test["failure"]:
+            with tag("failure"):
+              text(test["failure"])
+
+  return doc.getvalue()
+
+sys.stderr.write("make_test_output.py: writing XML from directory: " +
+                 sys.argv[1] + "\n");
+print genxml(readtests(sys.argv[1]))
diff --git a/tools/jenkins/pull_request.sh b/tools/jenkins/pull_request.sh
index cb0f4072b0..00538b9c4b 100755
--- a/tools/jenkins/pull_request.sh
+++ b/tools/jenkins/pull_request.sh
@@ -3,4 +3,5 @@
 
 export DOCKERFILE_DIR=tools/docker
 export DOCKER_RUN_SCRIPT=tools/run_tests/jenkins.sh
+export OUTPUT_DIR=testoutput
 ./tools/jenkins/build_and_run_docker.sh
diff --git a/tools/run_tests/jenkins.sh b/tools/run_tests/jenkins.sh
index 620c21168c..32050d2547 100755
--- a/tools/run_tests/jenkins.sh
+++ b/tools/run_tests/jenkins.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 
+WORKSPACE_BASE=`pwd`
 MY_DIR="$(dirname "$0")"
 TEST_SCRIPT=$MY_DIR/tests.sh
 BUILD_DIR=/tmp/protobuf
@@ -17,16 +18,30 @@ git clone /var/local/jenkins/protobuf
 cd protobuf
 
 OUTPUT_DIR=`mktemp -d`
-mkdir -p $OUTPUT_DIR/1
+LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
+mkdir -p $LOG_OUTPUT_DIR/1/cpp
 
+################################################################################
 # cpp build needs to run first, non-parallelized, so that protoc is available
 # for other builds.
-$TEST_SCRIPT cpp | tee $OUTPUT_DIR/1/cpp
+
+# Output filenames to follow the overall scheme used by parallel, ie:
+#  $DIR/logs/1/cpp/stdout
+#  $DIR/logs/1/cpp/stderr
+#  $DIR/logs/1/csharp/stdout
+#  $DIR/logs/1/csharp/stderr
+#  $DIR/logs/1/java_jdk7/stdout
+#  $DIR/logs/1/java_jdk7/stderr
+CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
+CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
+$TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
 
 # Other tests are run in parallel.  The overall run fails if any one of them
 # fails.
 
-parallel --results $OUTPUT_DIR $TEST_SCRIPT ::: \
+FAILED=false
+
+parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
   csharp \
   java_jdk7 \
   javanano_jdk7 \
@@ -34,8 +49,17 @@ parallel --results $OUTPUT_DIR $TEST_SCRIPT ::: \
   javanano_oracle7 \
   python \
   python_cpp \
-  ruby21
+  ruby21 \
+  || true  # Process test results even if tests fail.
 
-# java_jdk6 \
+# The directory that is copied from Docker back into the Jenkins workspace.
+COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
+mkdir -p $COPY_FROM_DOCKER
+TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml
 
-find $OUTPUT_DIR
+python $MY_DIR/../jenkins/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
+
+ls -l $TESTOUTPUT_XML_FILE
+
+### disabled tests
+# java_jdk6 \