simplify building artifacts on a single machine

pull/11115/head
Jan Tattermusch 8 years ago
parent fa78924340
commit 47c7521960
  1. 57
      tools/run_tests/artifacts/artifact_targets.py
  2. 4
      tools/run_tests/artifacts/build_artifact_csharp.bat
  3. 4
      tools/run_tests/artifacts/build_artifact_csharp.sh
  4. 6
      tools/run_tests/artifacts/build_artifact_node.bat
  5. 6
      tools/run_tests/artifacts/build_artifact_node.sh
  6. 4
      tools/run_tests/artifacts/build_artifact_php.sh
  7. 6
      tools/run_tests/artifacts/build_artifact_protoc.bat
  8. 4
      tools/run_tests/artifacts/build_artifact_protoc.sh
  9. 5
      tools/run_tests/artifacts/build_artifact_python.bat
  10. 5
      tools/run_tests/artifacts/build_artifact_python.sh
  11. 4
      tools/run_tests/artifacts/build_artifact_ruby.sh
  12. 47
      tools/run_tests/artifacts/run_in_workspace.sh
  13. 5
      tools/run_tests/task_runner.py

@ -45,6 +45,7 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
"""Creates jobspec for a task running under docker.""" """Creates jobspec for a task running under docker."""
environ = environ.copy() environ = environ.copy()
environ['RUN_COMMAND'] = shell_command environ['RUN_COMMAND'] = shell_command
environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
docker_args=[] docker_args=[]
for k,v in environ.items(): for k,v in environ.items():
@ -65,9 +66,19 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
return jobspec return jobspec
def create_jobspec(name, cmdline, environ=None, shell=False, def create_jobspec(name, cmdline, environ={}, shell=False,
flake_retries=0, timeout_retries=0, timeout_seconds=30*60): flake_retries=0, timeout_retries=0, timeout_seconds=30*60,
use_workspace=False):
"""Creates jobspec.""" """Creates jobspec."""
environ = environ.copy()
if use_workspace:
environ['WORKSPACE_NAME'] = 'workspace_%s' % name
environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
cmdline = ['bash',
'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline
else:
environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
jobspec = jobset.JobSpec( jobspec = jobset.JobSpec(
cmdline=cmdline, cmdline=cmdline,
environ=environ, environ=environ,
@ -137,13 +148,14 @@ class PythonArtifact:
dir dir
], ],
environ=environ, environ=environ,
shell=True) use_workspace=True)
else: else:
environ['PYTHON'] = self.py_version environ['PYTHON'] = self.py_version
environ['SKIP_PIP_INSTALL'] = 'TRUE' environ['SKIP_PIP_INSTALL'] = 'TRUE'
return create_jobspec(self.name, return create_jobspec(self.name,
['tools/run_tests/artifacts/build_artifact_python.sh'], ['tools/run_tests/artifacts/build_artifact_python.sh'],
environ=environ) environ=environ,
use_workspace=True)
def __str__(self): def __str__(self):
return self.name return self.name
@ -162,20 +174,11 @@ class RubyArtifact:
return [] return []
def build_jobspec(self): def build_jobspec(self):
if self.platform == 'windows': # Ruby build uses docker internally and docker cannot be nested.
raise Exception("Not supported yet") # We are using a custom workspace instead.
else:
if self.platform == 'linux':
environ = {}
if self.arch == 'x86':
environ['SETARCH_CMD'] = 'linux32'
return create_docker_jobspec(self.name,
'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
'tools/run_tests/artifacts/build_artifact_ruby.sh',
environ=environ)
else:
return create_jobspec(self.name, return create_jobspec(self.name,
['tools/run_tests/artifacts/build_artifact_ruby.sh']) ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
use_workspace=True)
class CSharpExtArtifact: class CSharpExtArtifact:
@ -196,7 +199,7 @@ class CSharpExtArtifact:
return create_jobspec(self.name, return create_jobspec(self.name,
['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
cmake_arch_option], cmake_arch_option],
shell=True) use_workspace=True)
else: else:
environ = {'CONFIG': 'opt', environ = {'CONFIG': 'opt',
'EMBED_OPENSSL': 'true', 'EMBED_OPENSSL': 'true',
@ -216,7 +219,8 @@ class CSharpExtArtifact:
environ['LDFLAGS'] += ' %s' % archflag environ['LDFLAGS'] += ' %s' % archflag
return create_jobspec(self.name, return create_jobspec(self.name,
['tools/run_tests/artifacts/build_artifact_csharp.sh'], ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
environ=environ) environ=environ,
use_workspace=True)
def __str__(self): def __str__(self):
return self.name return self.name
@ -245,7 +249,7 @@ class NodeExtArtifact:
return create_jobspec(self.name, return create_jobspec(self.name,
['tools\\run_tests\\artifacts\\build_artifact_node.bat', ['tools\\run_tests\\artifacts\\build_artifact_node.bat',
self.gyp_arch], self.gyp_arch],
shell=True) use_workspace=True)
else: else:
if self.platform == 'linux': if self.platform == 'linux':
return create_docker_jobspec( return create_docker_jobspec(
@ -255,7 +259,8 @@ class NodeExtArtifact:
else: else:
return create_jobspec(self.name, return create_jobspec(self.name,
['tools/run_tests/artifacts/build_artifact_node.sh', ['tools/run_tests/artifacts/build_artifact_node.sh',
self.gyp_arch]) self.gyp_arch],
use_workspace=True)
class PHPArtifact: class PHPArtifact:
"""Builds PHP PECL package""" """Builds PHP PECL package"""
@ -277,7 +282,8 @@ class PHPArtifact:
'tools/run_tests/artifacts/build_artifact_php.sh') 'tools/run_tests/artifacts/build_artifact_php.sh')
else: else:
return create_jobspec(self.name, return create_jobspec(self.name,
['tools/run_tests/artifacts/build_artifact_php.sh']) ['tools/run_tests/artifacts/build_artifact_php.sh'],
use_workspace=True)
class ProtocArtifact: class ProtocArtifact:
"""Builds protoc and protoc-plugin artifacts""" """Builds protoc and protoc-plugin artifacts"""
@ -310,14 +316,16 @@ class ProtocArtifact:
environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
return create_jobspec(self.name, return create_jobspec(self.name,
['tools/run_tests/artifacts/build_artifact_protoc.sh'], ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
environ=environ) environ=environ,
use_workspace=True)
else: else:
generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12'
vcplatform = 'x64' if self.arch == 'x64' else 'Win32' vcplatform = 'x64' if self.arch == 'x64' else 'Win32'
return create_jobspec(self.name, return create_jobspec(self.name,
['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
environ={'generator': generator, environ={'generator': generator,
'Platform': vcplatform}) 'Platform': vcplatform},
use_workspace=True)
def __str__(self): def __str__(self):
return self.name return self.name
@ -351,7 +359,6 @@ def targets():
PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python34'),
PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python35'),
PythonArtifact('windows', 'x64', 'Python36'), PythonArtifact('windows', 'x64', 'Python36'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'), RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'), RubyArtifact('macos', 'x64'),
PHPArtifact('linux', 'x64'), PHPArtifact('linux', 'x64'),

@ -37,8 +37,8 @@ cd cmake\build\%ARCHITECTURE%
cmake --build . --target grpc_csharp_ext --config Release cmake --build . --target grpc_csharp_ext --config Release
cd ..\..\.. cd ..\..\..
mkdir artifacts mkdir -p %ARTIFACTS_OUT%
copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll artifacts || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll artifacts || goto :error copy /Y cmake\build\Win32\Release\grpc_csharp_ext.dll %ARTIFACTS_OUT% || copy /Y cmake\build\x64\Release\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error
goto :EOF goto :EOF

@ -34,5 +34,5 @@ cd $(dirname $0)/../../..
make grpc_csharp_ext make grpc_csharp_ext
mkdir -p artifacts mkdir -p "${ARTIFACTS_OUT}"
cp libs/opt/libgrpc_csharp_ext.so artifacts || cp libs/opt/libgrpc_csharp_ext.dylib artifacts cp libs/opt/libgrpc_csharp_ext.so "${ARTIFACTS_OUT}" || cp libs/opt/libgrpc_csharp_ext.dylib "${ARTIFACTS_OUT}"

@ -37,7 +37,7 @@ del /f /q BUILD || rmdir build /s /q
call npm update || goto :error call npm update || goto :error
mkdir artifacts mkdir -p %ARTIFACTS_OUT%
for %%v in (%node_versions%) do ( for %%v in (%node_versions%) do (
call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1
@ -47,13 +47,13 @@ for %%v in (%node_versions%) do (
rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q
call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error call .\node_modules\.bin\node-pre-gyp.cmd build package testpackage --target=%%v --target_arch=%1 || goto :error
xcopy /Y /I /S build\stage\* artifacts\ || goto :error xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error
) )
for %%v in (%electron_versions%) do ( for %%v in (%electron_versions%) do (
cmd /V /C "set "HOME=%HOMEDRIVE%%HOMEPATH%\electron-gyp" && call .\node_modules\.bin\node-pre-gyp.cmd configure rebuild package testpackage --runtime=electron --target=%%v --target_arch=%1 --disturl=https://atom.io/download/electron" || goto :error cmd /V /C "set "HOME=%HOMEDRIVE%%HOMEPATH%\electron-gyp" && call .\node_modules\.bin\node-pre-gyp.cmd configure rebuild package testpackage --runtime=electron --target=%%v --target_arch=%1 --disturl=https://atom.io/download/electron" || goto :error
xcopy /Y /I /S build\stage\* artifacts\ || goto :error xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error
) )
if %errorlevel% neq 0 exit /b %errorlevel% if %errorlevel% neq 0 exit /b %errorlevel%

@ -38,7 +38,7 @@ cd $(dirname $0)/../../..
rm -rf build || true rm -rf build || true
mkdir -p artifacts mkdir -p "${ARTIFACTS_OUT}"
npm update npm update
@ -49,11 +49,11 @@ electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 )
for version in ${node_versions[@]} for version in ${node_versions[@]}
do do
./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH --grpc_alpine=true ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH --grpc_alpine=true
cp -r build/stage/* artifacts/ cp -r build/stage/* "${ARTIFACTS_OUT}"/
done done
for version in ${electron_versions[@]} for version in ${electron_versions[@]}
do do
HOME=~/.electron-gyp ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --runtime=electron --target=$version --target_arch=$NODE_TARGET_ARCH --disturl=https://atom.io/download/electron HOME=~/.electron-gyp ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --runtime=electron --target=$version --target_arch=$NODE_TARGET_ARCH --disturl=https://atom.io/download/electron
cp -r build/stage/* artifacts/ cp -r build/stage/* "${ARTIFACTS_OUT}"/
done done

@ -33,8 +33,8 @@ set -ex
cd $(dirname $0)/../../.. cd $(dirname $0)/../../..
mkdir -p artifacts mkdir -p "${ARTIFACTS_OUT}"
pear package pear package
cp -r grpc-*.tgz artifacts/ cp -r grpc-*.tgz "${ARTIFACTS_OUT}"/

@ -27,7 +27,7 @@
@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
mkdir artifacts mkdir -p %ARTIFACTS_OUT%
setlocal setlocal
cd third_party/protobuf/cmake cd third_party/protobuf/cmake
@ -39,8 +39,8 @@ endlocal
call vsprojects/build_plugins.bat || goto :error call vsprojects/build_plugins.bat || goto :error
xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe artifacts\ || goto :error xcopy /Y third_party\protobuf\cmake\build\solution\Release\protoc.exe %ARTIFACTS_OUT%\ || goto :error
xcopy /Y vsprojects\Release\*_plugin.exe artifacts\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe artifacts\ || goto :error xcopy /Y vsprojects\Release\*_plugin.exe %ARTIFACTS_OUT%\ || xcopy /Y vsprojects\x64\Release\*_plugin.exe %ARTIFACTS_OUT%\ || goto :error
goto :EOF goto :EOF

@ -37,5 +37,5 @@ cd $(dirname $0)/../../..
make plugins make plugins
mkdir -p artifacts mkdir -p "${ARTIFACTS_OUT}"
cp bins/opt/protobuf/protoc bins/opt/*_plugin artifacts/ cp bins/opt/protobuf/protoc bins/opt/*_plugin "${ARTIFACTS_OUT}"/

@ -38,8 +38,9 @@ set GRPC_PYTHON_BUILD_WITH_CYTHON=1
@rem Multiple builds are running simultaneously, so to avoid distutils @rem Multiple builds are running simultaneously, so to avoid distutils
@rem file collisions, we build everything in a tmp directory @rem file collisions, we build everything in a tmp directory
if not exist "artifacts" mkdir "artifacts" @rem TODO(jtattermusch): it doesn't look like builds are actually running in parallel in the same dir
set ARTIFACT_DIR=%cd%\artifacts mkdir -p %ARTIFACTS_OUT%
set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT%
set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\ set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\
mkdir %BUILD_DIR% mkdir %BUILD_DIR%
xcopy /s/e/q %cd%\* %BUILD_DIR% xcopy /s/e/q %cd%\* %BUILD_DIR%

@ -41,8 +41,9 @@ export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
# Because multiple builds run in parallel, some distutils file # Because multiple builds run in parallel, some distutils file
# operations may collide. To avoid this, each build is run in # operations may collide. To avoid this, each build is run in
# a temp directory # a temp directory
mkdir -p artifacts # TODO(jtattermusch): it doesn't look like builds are actually running in parallel in the same dir
ARTIFACT_DIR="$PWD/artifacts" mkdir -p "${ARTIFACTS_OUT}"
ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}"
BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"` BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"`
trap "rm -rf $BUILD_DIR" EXIT trap "rm -rf $BUILD_DIR" EXIT
cp -r * "$BUILD_DIR" cp -r * "$BUILD_DIR"

@ -63,6 +63,6 @@ if [ "$SYSTEM" == "Darwin" ] ; then
rm `ls pkg/*.gem | grep -v darwin` rm `ls pkg/*.gem | grep -v darwin`
fi fi
mkdir -p artifacts mkdir -p "${ARTIFACTS_OUT}"
cp pkg/*.gem artifacts cp pkg/*.gem "${ARTIFACTS_OUT}"/

@ -0,0 +1,47 @@
#!/bin/bash
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Create a workspace in a subdirectory to allow running multiple builds in isolation.
# WORKSPACE_NAME env variable needs to contain name of the workspace to create.
# All cmdline args will be executed as a command.
set -ex
cd $(dirname $0)/../../..
export repo_root=$(pwd)
rm -rf "${WORKSPACE_NAME}"
git clone . "${WORKSPACE_NAME}"
# clone gRPC submodules, use data from locally cloned submodules where possible
git submodule foreach 'cd "${repo_root}/${WORKSPACE_NAME}" \
&& git submodule update --init --reference ${repo_root}/${name} ${name}'
echo "Running in workspace ${WORKSPACE_NAME}"
cd ${WORKSPACE_NAME}
$@

@ -40,6 +40,7 @@ import artifacts.artifact_targets as artifact_targets
import artifacts.distribtest_targets as distribtest_targets import artifacts.distribtest_targets as distribtest_targets
import artifacts.package_targets as package_targets import artifacts.package_targets as package_targets
import python_utils.jobset as jobset import python_utils.jobset as jobset
import python_utils.report_utils as report_utils
_TARGETS = [] _TARGETS = []
_TARGETS += artifact_targets.targets() _TARGETS += artifact_targets.targets()
@ -116,8 +117,10 @@ if not build_jobs:
sys.exit(1) sys.exit(1)
jobset.message('START', 'Building targets.', do_newline=True) jobset.message('START', 'Building targets.', do_newline=True)
num_failures, _ = jobset.run( num_failures, resultset = jobset.run(
build_jobs, newline_on_success=True, maxjobs=args.jobs) build_jobs, newline_on_success=True, maxjobs=args.jobs)
report_utils.render_junit_xml_report(resultset, 'report_taskrunner_sponge_log.xml',
suite_name='tasks')
if num_failures == 0: if num_failures == 0:
jobset.message('SUCCESS', 'All targets built successfully.', jobset.message('SUCCESS', 'All targets built successfully.',
do_newline=True) do_newline=True)

Loading…
Cancel
Save