|
|
|
@ -30,11 +30,14 @@ |
|
|
|
|
|
|
|
|
|
"""Definition of targets to build artifacts.""" |
|
|
|
|
|
|
|
|
|
import os.path |
|
|
|
|
import sys |
|
|
|
|
|
|
|
|
|
import jobset |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, |
|
|
|
|
flake_retries=0, timeout_retries=0): |
|
|
|
|
flake_retries=0, timeout_retries=0, timeout_seconds=30*60): |
|
|
|
|
"""Creates jobspec for a task running under docker.""" |
|
|
|
|
environ = environ.copy() |
|
|
|
|
environ['RUN_COMMAND'] = shell_command |
|
|
|
@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, |
|
|
|
|
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, |
|
|
|
|
environ=docker_env, |
|
|
|
|
shortname='build_artifact.%s' % (name), |
|
|
|
|
timeout_seconds=30*60, |
|
|
|
|
timeout_seconds=timeout_seconds, |
|
|
|
|
flake_retries=flake_retries, |
|
|
|
|
timeout_retries=timeout_retries) |
|
|
|
|
return jobspec |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_jobspec(name, cmdline, environ=None, shell=False, |
|
|
|
|
flake_retries=0, timeout_retries=0): |
|
|
|
|
flake_retries=0, timeout_retries=0, timeout_seconds=30*60): |
|
|
|
|
"""Creates jobspec.""" |
|
|
|
|
jobspec = jobset.JobSpec( |
|
|
|
|
cmdline=cmdline, |
|
|
|
|
environ=environ, |
|
|
|
|
shortname='build_artifact.%s' % (name), |
|
|
|
|
timeout_seconds=30*60, |
|
|
|
|
timeout_seconds=timeout_seconds, |
|
|
|
|
flake_retries=flake_retries, |
|
|
|
|
timeout_retries=timeout_retries, |
|
|
|
|
shell=shell) |
|
|
|
@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = { |
|
|
|
|
'x64': '-m64' |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
python_version_arch_map = { |
|
|
|
|
'x86': 'Python27_32bits', |
|
|
|
|
'x64': 'Python27' |
|
|
|
|
python_windows_version_arch_map = { |
|
|
|
|
('x86', '2.7'): 'Python27_32bits', |
|
|
|
|
('x64', '2.7'): 'Python27', |
|
|
|
|
('x86', '3.4'): 'Python34_32bits', |
|
|
|
|
('x64', '3.4'): 'Python34', |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
class PythonArtifact: |
|
|
|
|
"""Builds Python artifacts.""" |
|
|
|
|
|
|
|
|
|
def __init__(self, platform, arch, manylinux_build=None): |
|
|
|
|
def __init__(self, platform, arch, python_version, manylinux_build=None): |
|
|
|
|
if manylinux_build: |
|
|
|
|
self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build) |
|
|
|
|
self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build) |
|
|
|
|
else: |
|
|
|
|
self.name = 'python_%s_%s' % (platform, arch) |
|
|
|
|
self.name = 'python%s_%s_%s' % (python_version, platform, arch) |
|
|
|
|
self.platform = platform |
|
|
|
|
self.arch = arch |
|
|
|
|
self.labels = ['artifact', 'python', platform, arch] |
|
|
|
|
self.python_version = python_version_arch_map[arch] |
|
|
|
|
self.labels = ['artifact', 'python', python_version, platform, arch] |
|
|
|
|
self.python_version = python_version |
|
|
|
|
self.python_windows_prefix = python_windows_version_arch_map[arch, python_version] |
|
|
|
|
self.manylinux_build = manylinux_build |
|
|
|
|
|
|
|
|
|
def pre_build_jobspecs(self): |
|
|
|
|
return [] |
|
|
|
|
return [] |
|
|
|
|
|
|
|
|
|
def build_jobspec(self): |
|
|
|
|
environ = {} |
|
|
|
@ -107,7 +113,6 @@ class PythonArtifact: |
|
|
|
|
# special places... |
|
|
|
|
environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build) |
|
|
|
|
environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build) |
|
|
|
|
# Our docker image has all the prerequisites pip-installed already. |
|
|
|
|
environ['SKIP_PIP_INSTALL'] = '1' |
|
|
|
|
# Platform autodetection for the manylinux1 image breaks so we set the |
|
|
|
|
# defines ourselves. |
|
|
|
@ -117,16 +122,18 @@ class PythonArtifact: |
|
|
|
|
return create_docker_jobspec(self.name, |
|
|
|
|
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, |
|
|
|
|
'tools/run_tests/build_artifact_python.sh', |
|
|
|
|
environ=environ) |
|
|
|
|
environ=environ, |
|
|
|
|
timeout_seconds=60*60) |
|
|
|
|
elif self.platform == 'windows': |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
['tools\\run_tests\\build_artifact_python.bat', |
|
|
|
|
self.python_version, |
|
|
|
|
self.python_windows_prefix, |
|
|
|
|
'32' if self.arch == 'x86' else '64' |
|
|
|
|
], |
|
|
|
|
shell=True) |
|
|
|
|
else: |
|
|
|
|
environ['SKIP_PIP_INSTALL'] = 'TRUE' |
|
|
|
|
environ['PYTHON'] = 'python{}'.format(self.python_version) |
|
|
|
|
return create_jobspec(self.name, |
|
|
|
|
['tools/run_tests/build_artifact_python.sh'], |
|
|
|
|
environ=environ) |
|
|
|
@ -323,13 +330,18 @@ def targets(): |
|
|
|
|
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) |
|
|
|
|
for platform in ('linux', 'macos', 'windows') |
|
|
|
|
for arch in ('x86', 'x64')] + |
|
|
|
|
[PythonArtifact('linux', 'x86', 'cp27-cp27m'), |
|
|
|
|
PythonArtifact('linux', 'x86', 'cp27-cp27mu'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp27-cp27m'), |
|
|
|
|
PythonArtifact('linux', 'x64', 'cp27-cp27mu'), |
|
|
|
|
PythonArtifact('macos', 'x64'), |
|
|
|
|
PythonArtifact('windows', 'x86'), |
|
|
|
|
PythonArtifact('windows', 'x64'), |
|
|
|
|
[PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'), |
|
|
|
|
PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'), |
|
|
|
|
PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'), |
|
|
|
|
PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'), |
|
|
|
|
PythonArtifact('macos', 'x64', '2.7'), |
|
|
|
|
PythonArtifact('windows', 'x86', '2.7'), |
|
|
|
|
PythonArtifact('windows', 'x64', '2.7'), |
|
|
|
|
PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'), |
|
|
|
|
PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'), |
|
|
|
|
PythonArtifact('macos', 'x64', '3.4'), |
|
|
|
|
PythonArtifact('windows', 'x86', '3.4'), |
|
|
|
|
PythonArtifact('windows', 'x64', '3.4'), |
|
|
|
|
RubyArtifact('linux', 'x86'), |
|
|
|
|
RubyArtifact('linux', 'x64'), |
|
|
|
|
RubyArtifact('macos', 'x64'), |
|
|
|
|