|
|
|
|
@ -12,7 +12,6 @@ |
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
|
|
|
# See the License for the specific language governing permissions and |
|
|
|
|
# limitations under the License. |
|
|
|
|
|
|
|
|
|
"""Definition of targets to build artifacts.""" |
|
|
|
|
|
|
|
|
|
import os.path |
|
|
|
|
@ -24,9 +23,15 @@ sys.path.insert(0, os.path.abspath('..')) |
|
|
|
|
import python_utils.jobset as jobset |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, |
|
|
|
|
flake_retries=0, timeout_retries=0, timeout_seconds=30*60, |
|
|
|
|
docker_base_image=None, extra_docker_args=None): |
|
|
|
|
def create_docker_jobspec(name, |
|
|
|
|
dockerfile_dir, |
|
|
|
|
shell_command, |
|
|
|
|
environ={}, |
|
|
|
|
flake_retries=0, |
|
|
|
|
timeout_retries=0, |
|
|
|
|
timeout_seconds=30 * 60, |
|
|
|
|
docker_base_image=None, |
|
|
|
|
extra_docker_args=None): |
|
|
|
|
"""Creates jobspec for a task running under docker.""" |
|
|
|
|
environ = environ.copy() |
|
|
|
|
environ['RUN_COMMAND'] = shell_command |
|
|
|
|
@ -35,16 +40,19 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, |
|
|
|
|
docker_args = [] |
|
|
|
|
for k, v in environ.items(): |
|
|
|
|
docker_args += ['-e', '%s=%s' % (k, v)] |
|
|
|
|
docker_env = {'DOCKERFILE_DIR': dockerfile_dir, |
|
|
|
|
docker_env = { |
|
|
|
|
'DOCKERFILE_DIR': dockerfile_dir, |
|
|
|
|
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', |
|
|
|
|
'OUTPUT_DIR': 'artifacts'} |
|
|
|
|
'OUTPUT_DIR': 'artifacts' |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if docker_base_image is not None: |
|
|
|
|
docker_env['DOCKER_BASE_IMAGE'] = docker_base_image |
|
|
|
|
if extra_docker_args is not None: |
|
|
|
|
docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args |
|
|
|
|
jobspec = jobset.JobSpec( |
|
|
|
|
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, |
|
|
|
|
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + |
|
|
|
|
docker_args, |
|
|
|
|
environ=docker_env, |
|
|
|
|
shortname='build_artifact.%s' % (name), |
|
|
|
|
timeout_seconds=timeout_seconds, |
|
|
|
|
@ -53,8 +61,13 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, |
|
|
|
|
return jobspec |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_jobspec(name, cmdline, environ={}, shell=False, |
|
|
|
|
flake_retries=0, timeout_retries=0, timeout_seconds=30*60, |
|
|
|
|
def create_jobspec(name, |
|
|
|
|
cmdline, |
|
|
|
|
environ={}, |
|
|
|
|
shell=False, |
|
|
|
|
flake_retries=0, |
|
|
|
|
timeout_retries=0, |
|
|
|
|
timeout_seconds=30 * 60, |
|
|
|
|
use_workspace=False, |
|
|
|
|
cpu_cost=1.0): |
|
|
|
|
"""Creates jobspec.""" |
|
|
|
|
@ -62,8 +75,8 @@ def create_jobspec(name, cmdline, environ={}, shell=False, |
|
|
|
|
if use_workspace: |
|
|
|
|
environ['WORKSPACE_NAME'] = 'workspace_%s' % name |
|
|
|
|
environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) |
|
|
|
|
cmdline = ['bash', |
|
|
|
|
'tools/run_tests/artifacts/run_in_workspace.sh'] + cmdline |
|
|
|
|
cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' |
|
|
|
|
] + cmdline |
|
|
|
|
else: |
|
|
|
|
environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) |
|
|
|
|
|
|
|
|
|
@ -81,10 +94,7 @@ def create_jobspec(name, cmdline, environ={}, shell=False, |
|
|
|
|
|
|
|
|
|
_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' |
|
|
|
|
|
|
|
|
|
_ARCH_FLAG_MAP = { |
|
|
|
|
'x86': '-m32', |
|
|
|
|
'x64': '-m64' |
|
|
|
|
} |
|
|
|
|
_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PythonArtifact: |
|
|
|
|
@ -104,13 +114,15 @@ class PythonArtifact: |
|
|
|
|
environ = {} |
|
|
|
|
if self.platform == 'linux_extra': |
|
|
|
|
# Raspberry Pi build |
|
|
|
|
environ['PYTHON'] = '/usr/local/bin/python{}'.format(self.py_version) |
|
|
|
|
environ['PYTHON'] = '/usr/local/bin/python{}'.format( |
|
|
|
|
self.py_version) |
|
|
|
|
environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) |
|
|
|
|
# https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 |
|
|
|
|
# A QEMU bug causes submodule update to hang, so we copy directly |
|
|
|
|
environ['RELATIVE_COPY_PATH'] = '.' |
|
|
|
|
extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' |
|
|
|
|
return create_docker_jobspec(self.name, |
|
|
|
|
return create_docker_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), |
|
|
|
|
'tools/run_tests/artifacts/build_artifact_python.sh', |
|
|
|
|
environ=environ, |
|
|
|
|
@ -122,7 +134,8 @@ class PythonArtifact: |
|
|
|
|
environ['SETARCH_CMD'] = 'linux32' |
|
|
|
|
# Inside the manylinux container, the python installations are located in |
|
|
|
|
# special places... |
|
|
|
|
environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version) |
|
|
|
|
environ['PYTHON'] = '/opt/python/{}/bin/python'.format( |
|
|
|
|
self.py_version) |
|
|
|
|
environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) |
|
|
|
|
# Platform autodetection for the manylinux1 image breaks so we set the |
|
|
|
|
# defines ourselves. |
|
|
|
|
@ -131,12 +144,15 @@ class PythonArtifact: |
|
|
|
|
environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' |
|
|
|
|
environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' |
|
|
|
|
environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' |
|
|
|
|
return create_docker_jobspec(self.name, |
|
|
|
|
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, |
|
|
|
|
return create_docker_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % |
|
|
|
|
self.arch, |
|
|
|
|
'tools/run_tests/artifacts/build_artifact_python.sh', |
|
|
|
|
environ=environ, |
|
|
|
|
timeout_seconds=60 * 60, |
|
|
|
|
docker_base_image='quay.io/pypa/manylinux1_i686' if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') |
|
|
|
|
docker_base_image='quay.io/pypa/manylinux1_i686' |
|
|
|
|
if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') |
|
|
|
|
elif self.platform == 'windows': |
|
|
|
|
if 'Python27' in self.py_version or 'Python34' in self.py_version: |
|
|
|
|
environ['EXT_COMPILER'] = 'mingw32' |
|
|
|
|
@ -144,18 +160,21 @@ class PythonArtifact: |
|
|
|
|
environ['EXT_COMPILER'] = 'msvc' |
|
|
|
|
# For some reason, the batch script %random% always runs with the same |
|
|
|
|
# seed. We create a random temp-dir here |
|
|
|
|
dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
['tools\\run_tests\\artifacts\\build_artifact_python.bat', |
|
|
|
|
self.py_version, |
|
|
|
|
'32' if self.arch == 'x86' else '64'], |
|
|
|
|
dir = ''.join( |
|
|
|
|
random.choice(string.ascii_uppercase) for _ in range(10)) |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, [ |
|
|
|
|
'tools\\run_tests\\artifacts\\build_artifact_python.bat', |
|
|
|
|
self.py_version, '32' if self.arch == 'x86' else '64' |
|
|
|
|
], |
|
|
|
|
environ=environ, |
|
|
|
|
timeout_seconds=45 * 60, |
|
|
|
|
use_workspace=True) |
|
|
|
|
else: |
|
|
|
|
environ['PYTHON'] = self.py_version |
|
|
|
|
environ['SKIP_PIP_INSTALL'] = 'TRUE' |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
['tools/run_tests/artifacts/build_artifact_python.sh'], |
|
|
|
|
environ=environ, |
|
|
|
|
timeout_seconds=60 * 60, |
|
|
|
|
@ -180,8 +199,8 @@ class RubyArtifact: |
|
|
|
|
def build_jobspec(self): |
|
|
|
|
# Ruby build uses docker internally and docker cannot be nested. |
|
|
|
|
# We are using a custom workspace instead. |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
['tools/run_tests/artifacts/build_artifact_ruby.sh'], |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], |
|
|
|
|
use_workspace=True, |
|
|
|
|
timeout_seconds=45 * 60) |
|
|
|
|
|
|
|
|
|
@ -201,19 +220,24 @@ class CSharpExtArtifact: |
|
|
|
|
def build_jobspec(self): |
|
|
|
|
if self.platform == 'windows': |
|
|
|
|
cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
['tools\\run_tests\\artifacts\\build_artifact_csharp.bat', |
|
|
|
|
cmake_arch_option], |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, [ |
|
|
|
|
'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', |
|
|
|
|
cmake_arch_option |
|
|
|
|
], |
|
|
|
|
use_workspace=True) |
|
|
|
|
else: |
|
|
|
|
environ = {'CONFIG': 'opt', |
|
|
|
|
environ = { |
|
|
|
|
'CONFIG': 'opt', |
|
|
|
|
'EMBED_OPENSSL': 'true', |
|
|
|
|
'EMBED_ZLIB': 'true', |
|
|
|
|
'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', |
|
|
|
|
'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', |
|
|
|
|
'LDFLAGS': ''} |
|
|
|
|
'LDFLAGS': '' |
|
|
|
|
} |
|
|
|
|
if self.platform == 'linux': |
|
|
|
|
return create_docker_jobspec(self.name, |
|
|
|
|
return create_docker_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, |
|
|
|
|
'tools/run_tests/artifacts/build_artifact_csharp.sh', |
|
|
|
|
environ=environ) |
|
|
|
|
@ -222,7 +246,8 @@ class CSharpExtArtifact: |
|
|
|
|
environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) |
|
|
|
|
environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) |
|
|
|
|
environ['LDFLAGS'] += ' %s' % archflag |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
['tools/run_tests/artifacts/build_artifact_csharp.sh'], |
|
|
|
|
environ=environ, |
|
|
|
|
use_workspace=True) |
|
|
|
|
@ -230,6 +255,7 @@ class CSharpExtArtifact: |
|
|
|
|
def __str__(self): |
|
|
|
|
return self.name |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PHPArtifact: |
|
|
|
|
"""Builds PHP PECL package""" |
|
|
|
|
|
|
|
|
|
@ -249,10 +275,11 @@ class PHPArtifact: |
|
|
|
|
'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), |
|
|
|
|
'tools/run_tests/artifacts/build_artifact_php.sh') |
|
|
|
|
else: |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
['tools/run_tests/artifacts/build_artifact_php.sh'], |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], |
|
|
|
|
use_workspace=True) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ProtocArtifact: |
|
|
|
|
"""Builds protoc and protoc-plugin artifacts""" |
|
|
|
|
|
|
|
|
|
@ -271,24 +298,30 @@ class ProtocArtifact: |
|
|
|
|
ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] |
|
|
|
|
if self.platform != 'macos': |
|
|
|
|
ldflags += ' -static-libgcc -static-libstdc++ -s' |
|
|
|
|
environ={'CONFIG': 'opt', |
|
|
|
|
environ = { |
|
|
|
|
'CONFIG': 'opt', |
|
|
|
|
'CXXFLAGS': cxxflags, |
|
|
|
|
'LDFLAGS': ldflags, |
|
|
|
|
'PROTOBUF_LDFLAGS_EXTRA': ldflags} |
|
|
|
|
'PROTOBUF_LDFLAGS_EXTRA': ldflags |
|
|
|
|
} |
|
|
|
|
if self.platform == 'linux': |
|
|
|
|
return create_docker_jobspec(self.name, |
|
|
|
|
return create_docker_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
'tools/dockerfile/grpc_artifact_protoc', |
|
|
|
|
'tools/run_tests/artifacts/build_artifact_protoc.sh', |
|
|
|
|
environ=environ) |
|
|
|
|
else: |
|
|
|
|
environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
environ[ |
|
|
|
|
'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
['tools/run_tests/artifacts/build_artifact_protoc.sh'], |
|
|
|
|
environ=environ, |
|
|
|
|
use_workspace=True) |
|
|
|
|
else: |
|
|
|
|
generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
return create_jobspec( |
|
|
|
|
self.name, |
|
|
|
|
['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], |
|
|
|
|
environ={'generator': generator}, |
|
|
|
|
use_workspace=True) |
|
|
|
|
@ -299,41 +332,41 @@ class ProtocArtifact: |
|
|
|
|
|
|
|
|
|
def targets(): |
|
|
|
|
"""Gets list of supported targets""" |
|
|
|
|
return ([Cls(platform, arch) |
|
|
|
|
return ([ |
|
|
|
|
Cls(platform, arch) |
|
|
|
|
for Cls in (CSharpExtArtifact, ProtocArtifact) |
|
|
|
|
for platform in ('linux', 'macos', 'windows') |
|
|
|
|
for arch in ('x86', 'x64')] + |
|
|
|
|
[PythonArtifact('linux', 'x86', 'cp27-cp27m'), |
|
|
|
|
PythonArtifact('linux', 'x86', 'cp27-cp27mu'), |
|
|
|
|
PythonArtifact('linux', 'x86', 'cp34-cp34m'), |
|
|
|
|
PythonArtifact('linux', 'x86', 'cp35-cp35m'), |
|
|
|
|
PythonArtifact('linux', 'x86', 'cp36-cp36m'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv7', '2.7'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv7', '3.4'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv7', '3.5'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv7', '3.6'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv6', '2.7'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv6', '3.4'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv6', '3.5'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv6', '3.6'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp27-cp27m'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp27-cp27mu'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp34-cp34m'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp35-cp35m'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp36-cp36m'), |
|
|
|
|
PythonArtifact('macos', 'x64', 'python2.7'), |
|
|
|
|
PythonArtifact('macos', 'x64', 'python3.4'), |
|
|
|
|
PythonArtifact('macos', 'x64', 'python3.5'), |
|
|
|
|
PythonArtifact('macos', 'x64', 'python3.6'), |
|
|
|
|
PythonArtifact('windows', 'x86', 'Python27_32bits'), |
|
|
|
|
PythonArtifact('windows', 'x86', 'Python34_32bits'), |
|
|
|
|
PythonArtifact('windows', 'x86', 'Python35_32bits'), |
|
|
|
|
PythonArtifact('windows', 'x86', 'Python36_32bits'), |
|
|
|
|
PythonArtifact('windows', 'x64', 'Python27'), |
|
|
|
|
PythonArtifact('windows', 'x64', 'Python34'), |
|
|
|
|
PythonArtifact('windows', 'x64', 'Python35'), |
|
|
|
|
PythonArtifact('windows', 'x64', 'Python36'), |
|
|
|
|
RubyArtifact('linux', 'x64'), |
|
|
|
|
RubyArtifact('macos', 'x64'), |
|
|
|
|
PHPArtifact('linux', 'x64'), |
|
|
|
|
PHPArtifact('macos', 'x64')]) |
|
|
|
|
for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') |
|
|
|
|
] + [ |
|
|
|
|
PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact( |
|
|
|
|
'linux', 'x86', 'cp27-cp27mu'), PythonArtifact( |
|
|
|
|
'linux', 'x86', 'cp34-cp34m'), PythonArtifact( |
|
|
|
|
'linux', 'x86', 'cp35-cp35m'), PythonArtifact( |
|
|
|
|
'linux', 'x86', 'cp36-cp36m'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv7', '2.7'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv7', '3.4'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv7', '3.5'), |
|
|
|
|
PythonArtifact('linux_extra', 'armv7', '3.6'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv6', '2.7'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv6', '3.4'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv6', '3.5'), PythonArtifact( |
|
|
|
|
'linux_extra', 'armv6', '3.6'), PythonArtifact( |
|
|
|
|
'linux', 'x64', 'cp27-cp27m'), PythonArtifact( |
|
|
|
|
'linux', 'x64', 'cp27-cp27mu'), PythonArtifact( |
|
|
|
|
'linux', 'x64', 'cp34-cp34m'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp35-cp35m'), PythonArtifact( |
|
|
|
|
'linux', 'x64', 'cp36-cp36m'), PythonArtifact( |
|
|
|
|
'macos', 'x64', 'python2.7'), PythonArtifact( |
|
|
|
|
'macos', 'x64', 'python3.4'), PythonArtifact('macos', 'x64', |
|
|
|
|
'python3.5'), |
|
|
|
|
PythonArtifact('macos', 'x64', 'python3.6'), PythonArtifact( |
|
|
|
|
'windows', 'x86', 'Python27_32bits'), PythonArtifact( |
|
|
|
|
'windows', 'x86', 'Python34_32bits'), PythonArtifact( |
|
|
|
|
'windows', 'x86', 'Python35_32bits'), PythonArtifact( |
|
|
|
|
'windows', 'x86', 'Python36_32bits'), PythonArtifact( |
|
|
|
|
'windows', 'x64', 'Python27'), |
|
|
|
|
PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact( |
|
|
|
|
'windows', 'x64', 'Python35'), PythonArtifact( |
|
|
|
|
'windows', 'x64', 'Python36'), RubyArtifact( |
|
|
|
|
'linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact( |
|
|
|
|
'linux', 'x64'), PHPArtifact('macos', 'x64') |
|
|
|
|
]) |
|
|
|
|
|
ncteisen
7 years ago