Joshua Haberman
9 years ago
commit
34d0cc220b
9 changed files with 448 additions and 36 deletions
@ -0,0 +1,6 @@ |
|||||||
|
|
||||||
|
Jenkins Infrastructure |
||||||
|
---------------------- |
||||||
|
|
||||||
|
The scripts in this directory serve as plumbing for running the protobuf |
||||||
|
tests under Jenkins. |
||||||
@ -0,0 +1,56 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
# Builds docker image and runs a command under it. |
||||||
|
# This is a generic script that is configured with the following variables: |
||||||
|
# |
||||||
|
# DOCKERFILE_DIR - Directory in which Dockerfile file is located. |
||||||
|
# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) |
||||||
|
# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. |
||||||
|
# $@ - Extra args to pass to docker run |
||||||
|
|
||||||
|
|
||||||
|
set -ex |
||||||
|
|
||||||
|
cd $(dirname $0)/.. |
||||||
|
git_root=$(pwd) |
||||||
|
cd - |
||||||
|
|
||||||
|
# Use image name based on Dockerfile location checksum |
||||||
|
DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) |
||||||
|
|
||||||
|
# Make sure docker image has been built. Should be instantaneous if so. |
||||||
|
docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR |
||||||
|
|
||||||
|
# Ensure existence of ccache directory |
||||||
|
CCACHE_DIR=/tmp/protobuf-ccache |
||||||
|
mkdir -p $CCACHE_DIR |
||||||
|
|
||||||
|
# Choose random name for docker container |
||||||
|
CONTAINER_NAME="build_and_run_docker_$(uuidgen)" |
||||||
|
|
||||||
|
# Run command inside docker |
||||||
|
docker run \ |
||||||
|
"$@" \ |
||||||
|
-e CCACHE_DIR=$CCACHE_DIR \ |
||||||
|
-e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \ |
||||||
|
-e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ |
||||||
|
-v "$git_root:/var/local/jenkins/protobuf:ro" \ |
||||||
|
-v $CCACHE_DIR:$CCACHE_DIR \ |
||||||
|
-w /var/local/git/protobuf \ |
||||||
|
--name=$CONTAINER_NAME \ |
||||||
|
$DOCKER_IMAGE_NAME \ |
||||||
|
bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" |
||||||
|
|
||||||
|
# Copy output artifacts |
||||||
|
if [ "$OUTPUT_DIR" != "" ] |
||||||
|
then |
||||||
|
docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true" |
||||||
|
fi |
||||||
|
|
||||||
|
# remove the container, possibly killing it first |
||||||
|
docker rm -f $CONTAINER_NAME || true |
||||||
|
|
||||||
|
if [ "$FAILED" != "" ] |
||||||
|
then |
||||||
|
exit 1 |
||||||
|
fi |
||||||
@ -0,0 +1,6 @@ |
|||||||
|
|
||||||
|
Jenkins Build Commands |
||||||
|
---------------------- |
||||||
|
|
||||||
|
The scripts in this directory are designed to be top-level entry points for |
||||||
|
Jenkins projects. |
||||||
@ -0,0 +1,15 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
# This is the top-level script we give to Jenkins as the entry point for |
||||||
|
# running the "pull request" project: |
||||||
|
# |
||||||
|
# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/ |
||||||
|
# |
||||||
|
# This script selects a specific Dockerfile (for building a Docker image) and |
||||||
|
# a script to run inside that image. Then we delegate to the general |
||||||
|
# build_and_run_docker.sh script. |
||||||
|
|
||||||
|
export DOCKERFILE_DIR=jenkins/docker |
||||||
|
export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh |
||||||
|
export OUTPUT_DIR=testoutput |
||||||
|
./jenkins/build_and_run_docker.sh |
||||||
@ -0,0 +1,130 @@ |
|||||||
|
# This Dockerfile specifies the recipe for creating an image for the tests |
||||||
|
# to run in. |
||||||
|
# |
||||||
|
# We install as many test dependencies here as we can, because these setup |
||||||
|
# steps can be cached. They do *not* run every time we run the build. |
||||||
|
# The Docker image is only rebuilt when the Dockerfile (ie. this file) |
||||||
|
# changes. |
||||||
|
|
||||||
|
# Base Dockerfile for gRPC dev images |
||||||
|
FROM debian:latest |
||||||
|
|
||||||
|
# Apt source for old Python versions. |
||||||
|
RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ |
||||||
|
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C |
||||||
|
|
||||||
|
# Apt source for Oracle Java. |
||||||
|
run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ |
||||||
|
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ |
||||||
|
echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections |
||||||
|
|
||||||
|
# Apt source for Mono |
||||||
|
run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ |
||||||
|
echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ |
||||||
|
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
||||||
|
|
||||||
|
# Install dependencies. We start with the basic ones require to build protoc |
||||||
|
# and the C++ build |
||||||
|
RUN apt-get update && apt-get install -y \ |
||||||
|
autoconf \ |
||||||
|
autotools-dev \ |
||||||
|
build-essential \ |
||||||
|
bzip2 \ |
||||||
|
ccache \ |
||||||
|
curl \ |
||||||
|
gcc \ |
||||||
|
git \ |
||||||
|
libc6 \ |
||||||
|
libc6-dbg \ |
||||||
|
libc6-dev \ |
||||||
|
libgtest-dev \ |
||||||
|
libtool \ |
||||||
|
make \ |
||||||
|
parallel \ |
||||||
|
time \ |
||||||
|
wget \ |
||||||
|
# -- For csharp -- |
||||||
|
mono-devel \ |
||||||
|
referenceassemblies-pcl \ |
||||||
|
nunit \ |
||||||
|
# -- For all Java builds -- \ |
||||||
|
maven \ |
||||||
|
# -- For java_jdk6 -- \ |
||||||
|
# oops! not in jessie. too old? openjdk-6-jdk \ |
||||||
|
# -- For java_jdk7 -- \ |
||||||
|
openjdk-7-jdk \ |
||||||
|
# -- For java_oracle7 -- \ |
||||||
|
oracle-java7-installer \ |
||||||
|
# -- For python / python_cpp -- \ |
||||||
|
python-setuptools \ |
||||||
|
python-pip \ |
||||||
|
python-dev \ |
||||||
|
python2.6-dev \ |
||||||
|
python3.3-dev \ |
||||||
|
python3.4-dev \ |
||||||
|
# -- For Ruby -- |
||||||
|
ruby \ |
||||||
|
&& apt-get clean |
||||||
|
|
||||||
|
################## |
||||||
|
# C# dependencies |
||||||
|
|
||||||
|
RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe |
||||||
|
|
||||||
|
################## |
||||||
|
# Python dependencies |
||||||
|
|
||||||
|
# These packages exist in apt-get, but their versions are too old, so we have |
||||||
|
# to get updates from pip. |
||||||
|
|
||||||
|
RUN pip install pip --upgrade |
||||||
|
RUN pip install virtualenv tox yattag |
||||||
|
|
||||||
|
|
||||||
|
################## |
||||||
|
# Ruby dependencies |
||||||
|
|
||||||
|
# Install rvm |
||||||
|
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 |
||||||
|
RUN \curl -sSL https://get.rvm.io | bash -s stable |
||||||
|
|
||||||
|
# Install Ruby 2.1 |
||||||
|
RUN /bin/bash -l -c "rvm install ruby-2.1" |
||||||
|
RUN /bin/bash -l -c "rvm use --default ruby-2.1" |
||||||
|
RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" |
||||||
|
RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" |
||||||
|
RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" |
||||||
|
RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" |
||||||
|
|
||||||
|
################## |
||||||
|
# Java dependencies |
||||||
|
|
||||||
|
# This step requires compiling protoc. :( |
||||||
|
|
||||||
|
ENV MAVEN_REPO /var/maven_local_repository |
||||||
|
ENV MVN mvn --batch-mode |
||||||
|
|
||||||
|
RUN cd /tmp && \ |
||||||
|
git clone https://github.com/google/protobuf.git && \ |
||||||
|
cd protobuf && \ |
||||||
|
./autogen.sh && \ |
||||||
|
./configure && \ |
||||||
|
make -j6 && \ |
||||||
|
cd java && \ |
||||||
|
$MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \ |
||||||
|
$MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ |
||||||
|
cd ../javanano && \ |
||||||
|
$MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO |
||||||
|
|
||||||
|
################## |
||||||
|
# Prepare ccache |
||||||
|
|
||||||
|
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
||||||
|
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
||||||
|
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
||||||
|
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
||||||
|
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
||||||
|
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
||||||
|
|
||||||
|
# Define the default command. |
||||||
|
CMD ["bash"] |
||||||
@ -0,0 +1,91 @@ |
|||||||
|
"""Gathers output from test runs and create an XML file in JUnit format. |
||||||
|
|
||||||
|
The output files from the individual tests have been written in a directory |
||||||
|
structure like: |
||||||
|
|
||||||
|
$DIR/joblog (output from "parallel --joblog joblog") |
||||||
|
$DIR/logs/1/cpp/stdout |
||||||
|
$DIR/logs/1/cpp/stderr |
||||||
|
$DIR/logs/1/csharp/stdout |
||||||
|
$DIR/logs/1/csharp/stderr |
||||||
|
$DIR/logs/1/java_jdk7/stdout |
||||||
|
$DIR/logs/1/java_jdk7/stderr |
||||||
|
etc. |
||||||
|
|
||||||
|
This script bundles them into a single output XML file so Jenkins can show |
||||||
|
detailed test results. It runs as the last step before the Jenkins build |
||||||
|
finishes. |
||||||
|
""" |
||||||
|
|
||||||
|
import os; |
||||||
|
import sys; |
||||||
|
from yattag import Doc |
||||||
|
from collections import defaultdict |
||||||
|
|
||||||
|
def readtests(basedir): |
||||||
|
tests = defaultdict(dict) |
||||||
|
|
||||||
|
# Sample input (note: separators are tabs). |
||||||
|
# |
||||||
|
# Seq Host Starttime Runtime Send Receive Exitval Signal Command |
||||||
|
# 1 : 1456263838.313 0.005 0 0 0 0 echo A |
||||||
|
with open(basedir + "/joblog") as jobs: |
||||||
|
firstline = next(jobs) |
||||||
|
for line in jobs: |
||||||
|
values = line.split("\t") |
||||||
|
|
||||||
|
name = values[8].split()[-1] |
||||||
|
test = tests[name] |
||||||
|
test["name"] = name |
||||||
|
test["time"] = values[3] |
||||||
|
|
||||||
|
exitval = values[6] |
||||||
|
if int(exitval): |
||||||
|
# We don't have a more specific message. User should look at stderr. |
||||||
|
test["failure"] = "TEST FAILURE" |
||||||
|
else: |
||||||
|
test["failure"] = False |
||||||
|
|
||||||
|
for testname in os.listdir(basedir + "/logs/1"): |
||||||
|
test = tests[testname] |
||||||
|
|
||||||
|
with open(basedir + "/logs/1/" + testname + "/stdout") as f: |
||||||
|
test["stdout"] = f.read() |
||||||
|
|
||||||
|
with open(basedir + "/logs/1/" + testname + "/stderr") as f: |
||||||
|
test["stderr"] = f.read() |
||||||
|
|
||||||
|
# The cpp test is special since it doesn't run under parallel so doesn't show |
||||||
|
# up in the job log. |
||||||
|
tests["cpp"]["name"] = "cpp" |
||||||
|
|
||||||
|
with open(basedir + '/logs/1/cpp/build_time', 'r') as f: |
||||||
|
tests["cpp"]["time"] = f.read().strip() |
||||||
|
tests["cpp"]["failure"] = False |
||||||
|
|
||||||
|
ret = tests.values() |
||||||
|
ret.sort(key=lambda x: x["name"]) |
||||||
|
|
||||||
|
return ret |
||||||
|
|
||||||
|
def genxml(tests): |
||||||
|
doc, tag, text = Doc().tagtext() |
||||||
|
|
||||||
|
with tag("testsuites"): |
||||||
|
with tag("testsuite", name="Protobuf Tests"): |
||||||
|
for test in tests: |
||||||
|
with tag("testcase", name=test["name"], classname=test["name"], |
||||||
|
time=test["time"]): |
||||||
|
with tag("system-out"): |
||||||
|
text(test["stdout"]) |
||||||
|
with tag("system-err"): |
||||||
|
text(test["stderr"]) |
||||||
|
if test["failure"]: |
||||||
|
with tag("failure"): |
||||||
|
text(test["failure"]) |
||||||
|
|
||||||
|
return doc.getvalue() |
||||||
|
|
||||||
|
sys.stderr.write("make_test_output.py: writing XML from directory: " + |
||||||
|
sys.argv[1] + "\n"); |
||||||
|
print genxml(readtests(sys.argv[1])) |
||||||
@ -0,0 +1,72 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
# This is the script that runs inside Docker, once the image has been built, |
||||||
|
# to execute all tests for the "pull request" project. |
||||||
|
|
||||||
|
WORKSPACE_BASE=`pwd` |
||||||
|
MY_DIR="$(dirname "$0")" |
||||||
|
TEST_SCRIPT=$MY_DIR/../tests.sh |
||||||
|
BUILD_DIR=/tmp/protobuf |
||||||
|
|
||||||
|
set -e # exit immediately on error |
||||||
|
set -x # display all commands |
||||||
|
|
||||||
|
# The protobuf repository is mounted into our Docker image, but read-only. |
||||||
|
# We clone into a directory inside Docker (this is faster than cp). |
||||||
|
rm -rf $BUILD_DIR |
||||||
|
mkdir -p $BUILD_DIR |
||||||
|
cd $BUILD_DIR |
||||||
|
git clone /var/local/jenkins/protobuf |
||||||
|
cd protobuf |
||||||
|
|
||||||
|
# Set up the directory where our test output is going to go. |
||||||
|
OUTPUT_DIR=`mktemp -d` |
||||||
|
LOG_OUTPUT_DIR=$OUTPUT_DIR/logs |
||||||
|
mkdir -p $LOG_OUTPUT_DIR/1/cpp |
||||||
|
|
||||||
|
################################################################################ |
||||||
|
# cpp build needs to run first, non-parallelized, so that protoc is available |
||||||
|
# for other builds. |
||||||
|
|
||||||
|
# Output filenames to follow the overall scheme used by parallel, ie: |
||||||
|
# $DIR/logs/1/cpp/stdout |
||||||
|
# $DIR/logs/1/cpp/stderr |
||||||
|
# $DIR/logs/1/csharp/stdout |
||||||
|
# $DIR/logs/1/csharp/stderr |
||||||
|
# $DIR/logs/1/java_jdk7/stdout |
||||||
|
# $DIR/logs/1/java_jdk7/stderr |
||||||
|
CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout |
||||||
|
CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr |
||||||
|
|
||||||
|
# Time the C++ build, so we can put this info in the test output. |
||||||
|
# It's important that we get /usr/bin/time (which supports -f and -o) and not |
||||||
|
# the bash builtin "time" which doesn't. |
||||||
|
TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" |
||||||
|
|
||||||
|
$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) |
||||||
|
|
||||||
|
# Other tests are run in parallel. |
||||||
|
|
||||||
|
parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ |
||||||
|
csharp \ |
||||||
|
java_jdk7 \ |
||||||
|
javanano_jdk7 \ |
||||||
|
java_oracle7 \ |
||||||
|
javanano_oracle7 \ |
||||||
|
python \ |
||||||
|
python_cpp \ |
||||||
|
ruby21 \ |
||||||
|
|| true # Process test results even if tests fail. |
||||||
|
|
||||||
|
cat $OUTPUT_DIR/joblog |
||||||
|
|
||||||
|
# The directory that is copied from Docker back into the Jenkins workspace. |
||||||
|
COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput |
||||||
|
mkdir -p $COPY_FROM_DOCKER |
||||||
|
TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml |
||||||
|
|
||||||
|
# Process all the output files from "parallel" and package them into a single |
||||||
|
# .xml file with detailed, broken-down test output. |
||||||
|
python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE |
||||||
|
|
||||||
|
ls -l $TESTOUTPUT_XML_FILE |
||||||
Loading…
Reference in new issue