From 7eba1729f020cd186d8cb77f3a95e48bd205e45b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 19 Jan 2016 08:43:00 -0800 Subject: [PATCH] building artifacts under docker --- tools/jenkins/build_and_run_docker.sh | 79 +++++++++++++++++++ tools/jenkins/docker_run.sh | 41 ++++++++++ .../grpc_artifact_linux_x64/Dockerfile | 64 +++++++++++++++ .../grpc_artifact_linux_x86/Dockerfile | 64 +++++++++++++++ tools/run_tests/build_artifact_csharp.sh | 38 +++++++++ tools/run_tests/build_artifacts.py | 53 ++++++++----- 6 files changed, 318 insertions(+), 21 deletions(-) create mode 100755 tools/jenkins/build_and_run_docker.sh create mode 100755 tools/jenkins/docker_run.sh create mode 100644 tools/jenkins/grpc_artifact_linux_x64/Dockerfile create mode 100644 tools/jenkins/grpc_artifact_linux_x86/Dockerfile create mode 100755 tools/run_tests/build_artifact_csharp.sh diff --git a/tools/jenkins/build_and_run_docker.sh b/tools/jenkins/build_and_run_docker.sh new file mode 100755 index 00000000000..288d9ca74db --- /dev/null +++ b/tools/jenkins/build_and_run_docker.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds docker image and runs a command under it. +# You should never need to call this script on your own. + +set -ex + +cd $(dirname $0)/../.. +git_root=$(pwd) +cd - + +# Create a local branch so the child Docker script won't complain +git branch -f jenkins-docker + +# Inputs +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) +# $@ - Extra args to pass to docker run + +# Use image name based on Dockerfile location checksum +DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + +# Make sure docker image has been built. Should be instantaneous if so. +docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR + +# Choose random name for docker container +CONTAINER_NAME="build_and_run_docker_$(uuidgen)" + +# Run command inside docker +docker run \ + "$@" \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -v "$git_root:/var/local/jenkins/grpc:ro" \ + -w /var/local/git/grpc \ + --name=$CONTAINER_NAME \ + $DOCKER_IMAGE_NAME \ + bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || FAILED="true" + +# Copy output artifacts +if [ "$OUTPUT_DIR" != "" ] +then + docker cp "$CONTAINER_NAME:/var/local/git/grpc/$OUTPUT_DIR" "$git_root" || FAILED="true" +fi + +# remove the container, possibly killing it first +docker rm -f $CONTAINER_NAME || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/jenkins/docker_run.sh b/tools/jenkins/docker_run.sh new file mode 100755 index 00000000000..17073482209 --- /dev/null +++ b/tools/jenkins/docker_run.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by build_docker_* inside a docker +# container. You should never need to call this script on your own. + +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +cd /var/local/git/grpc + +$RUN_COMMAND diff --git a/tools/jenkins/grpc_artifact_linux_x64/Dockerfile b/tools/jenkins/grpc_artifact_linux_x64/Dockerfile new file mode 100644 index 00000000000..d801e6f988c --- /dev/null +++ b/tools/jenkins/grpc_artifact_linux_x64/Dockerfile @@ -0,0 +1,64 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Docker file for building gRPC artifacts. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/jenkins/grpc_artifact_linux_x86/Dockerfile b/tools/jenkins/grpc_artifact_linux_x86/Dockerfile new file mode 100644 index 00000000000..55c2c70dacb --- /dev/null +++ b/tools/jenkins/grpc_artifact_linux_x86/Dockerfile @@ -0,0 +1,64 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Docker file for building gRPC artifacts. + +FROM 32bit/debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/run_tests/build_artifact_csharp.sh b/tools/run_tests/build_artifact_csharp.sh new file mode 100755 index 00000000000..c3811ec628a --- /dev/null +++ b/tools/run_tests/build_artifact_csharp.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../.. + +make grpc_csharp_ext + +mkdir -p artifacts +cp libs/opt/libgrpc_csharp_ext.so artifacts || cp libs/opt/libgrpc_csharp_ext.dylib artifacts diff --git a/tools/run_tests/build_artifacts.py b/tools/run_tests/build_artifacts.py index 47811b23bd2..0d7e3bd56b2 100755 --- a/tools/run_tests/build_artifacts.py +++ b/tools/run_tests/build_artifacts.py @@ -51,10 +51,33 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(ROOT) +def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, + flake_retries=0, timeout_retries=0): + """Creates jobspec for a task running under docker.""" + environ = environ.copy() + environ['RUN_COMMAND'] = shell_command + + #docker_args = ['-v', '%s/artifacts:/var/local/jenkins/grpc/artifacts' % ROOT] + docker_args=[] + for k,v in environ.iteritems(): + docker_args += ['-e', '%s=%s' % (k, v)] + docker_env = {'DOCKERFILE_DIR': dockerfile_dir, + 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh', + 'OUTPUT_DIR': 'artifacts'} + jobspec = jobset.JobSpec( + cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args, + environ=docker_env, + shortname='build_artifact.%s' % (name), + timeout_seconds=30*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries) + return jobspec + + def create_jobspec(name, cmdline, environ=None, shell=False, flake_retries=0, timeout_retries=0): """Creates jobspec.""" - test_job = jobset.JobSpec( + jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, shortname='build_artifact.%s' % (name), @@ -62,7 +85,7 @@ def create_jobspec(name, cmdline, environ=None, shell=False, flake_retries=flake_retries, timeout_retries=timeout_retries, shell=shell) - return test_job + return jobspec def macos_arch_env(arch): @@ -76,23 +99,6 @@ def macos_arch_env(arch): return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg} -class TestDockerArtifact: - """Demo artifact that shows that docker-based artifacts work""" - - def __init__(self): - self.name = 'test_docker_artifact' - self.labels = ['docker', 'linux'] - - def pre_build_jobspecs(self): - return [] - - def build_jobspec(self): - return create_jobspec(self.name, ['sleep', '5']) - - def __str__(self): - return self.name - - class CSharpExtArtifact: """Builds C# native extension library""" @@ -122,12 +128,17 @@ class CSharpExtArtifact: '/p:PlatformToolset=v120', '/p:Platform=%s' % msbuild_platform], shell=True) + if self.platform == 'linux': + environ = {'CONFIG': 'opt'} + return create_docker_jobspec(self.name, + 'tools/jenkins/grpc_artifact_linux_%s' % self.arch, + 'tools/run_tests/build_artifact_csharp.sh') else: environ = {'CONFIG': 'opt'} if self.platform == 'macos': environ.update(macos_arch_env(self.arch)) return create_jobspec(self.name, - ['make', 'grpc_csharp_ext'], + ['tools/run_tests/build_artifact_csharp.sh'], environ=environ) def __str__(self): @@ -135,7 +146,7 @@ class CSharpExtArtifact: _ARTIFACTS = [ - TestDockerArtifact(), + CSharpExtArtifact('linux', 'x86'), CSharpExtArtifact('linux', 'x64'), CSharpExtArtifact('macos', 'x86'), CSharpExtArtifact('macos', 'x64'),