#!/usr/bin/env python3 # Copyright 2016 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Definition of targets to build artifacts.""" import os.path import random import string import sys sys.path.insert(0, os.path.abspath("..")) import python_utils.jobset as jobset _LATEST_MANYLINUX = "manylinux2014" def create_docker_jobspec( name, dockerfile_dir, shell_command, environ={}, flake_retries=0, timeout_retries=0, timeout_seconds=30 * 60, extra_docker_args=None, verbose_success=False, ): """Creates jobspec for a task running under docker.""" environ = environ.copy() environ["ARTIFACTS_OUT"] = "artifacts/%s" % name docker_args = [] for k, v in list(environ.items()): docker_args += ["-e", "%s=%s" % (k, v)] docker_env = { "DOCKERFILE_DIR": dockerfile_dir, "DOCKER_RUN_SCRIPT": "tools/run_tests/dockerize/docker_run.sh", "DOCKER_RUN_SCRIPT_COMMAND": shell_command, "OUTPUT_DIR": "artifacts", } if extra_docker_args is not None: docker_env["EXTRA_DOCKER_ARGS"] = extra_docker_args jobspec = jobset.JobSpec( cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] + docker_args, environ=docker_env, shortname="build_artifact.%s" % (name), timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries, verbose_success=verbose_success, ) return jobspec def create_jobspec( name, cmdline, environ={}, shell=False, flake_retries=0, timeout_retries=0, timeout_seconds=30 * 60, use_workspace=False, cpu_cost=1.0, verbose_success=False, ): """Creates jobspec.""" environ = environ.copy() if use_workspace: environ["WORKSPACE_NAME"] = "workspace_%s" % name environ["ARTIFACTS_OUT"] = os.path.join("..", "artifacts", name) cmdline = [ "bash", "tools/run_tests/artifacts/run_in_workspace.sh", ] + cmdline else: environ["ARTIFACTS_OUT"] = os.path.join("artifacts", name) jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, shortname="build_artifact.%s" % (name), timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries, shell=shell, cpu_cost=cpu_cost, verbose_success=verbose_success, ) return jobspec _MACOS_COMPAT_FLAG = "-mmacosx-version-min=10.10" _ARCH_FLAG_MAP = {"x86": "-m32", "x64": "-m64"} class PythonArtifact: """Builds Python artifacts.""" def __init__(self, platform, arch, py_version, presubmit=False): self.name = "python_%s_%s_%s" % (platform, arch, py_version) self.platform = platform self.arch = arch self.labels = ["artifact", "python", platform, arch, py_version] if presubmit: self.labels.append("presubmit") self.py_version = py_version if platform == _LATEST_MANYLINUX: self.labels.append("latest-manylinux") if "manylinux" in platform: self.labels.append("linux") if "linux_extra" in platform: # linux_extra wheels used to be built by a separate kokoro job. # Their build is now much faster, so they can be included # in the regular artifact build. self.labels.append("linux") if "musllinux" in platform: self.labels.append("linux") def pre_build_jobspecs(self): return [] def build_jobspec(self, inner_jobs=None): environ = {} if inner_jobs is not None: # set number of parallel jobs when building native extension # building the native extension is the most time-consuming part of the build environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"] = str(inner_jobs) if self.platform == "macos": environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64" environ["GRPC_UNIVERSAL2_REPAIR"] = "true" if self.platform == "linux_extra": # Crosscompilation build for armv7 (e.g. Raspberry Pi) environ["PYTHON"] = "/opt/python/{}/bin/python3".format( self.py_version ) environ["PIP"] = "/opt/python/{}/bin/pip3".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" return create_docker_jobspec( self.name, "tools/dockerfile/grpc_artifact_python_linux_{}".format( self.arch ), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, timeout_seconds=60 * 60, ) elif "manylinux" in self.platform: if self.arch == "x86": environ["SETARCH_CMD"] = "linux32" # Inside the manylinux container, the python installations are located in # special places... environ["PYTHON"] = "/opt/python/{}/bin/python".format( self.py_version ) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" if self.arch == "aarch64": environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" # As we won't strip the binary with auditwheel (see below), strip # it at link time. environ["LDFLAGS"] = "-s" else: # only run auditwheel if we're not crosscompiling environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" # only build the packages that depend on grpcio-tools # if we're not crosscompiling. # - they require protoc to run on current architecture # - they only have sdist packages anyway, so it's useless to build them again environ["GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS"] = "TRUE" return create_docker_jobspec( self.name, "tools/dockerfile/grpc_artifact_python_%s_%s" % (self.platform, self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, timeout_seconds=60 * 60 * 2, ) elif "musllinux" in self.platform: environ["PYTHON"] = "/opt/python/{}/bin/python".format( self.py_version ) environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE" return create_docker_jobspec( self.name, "tools/dockerfile/grpc_artifact_python_%s_%s" % (self.platform, self.arch), "tools/run_tests/artifacts/build_artifact_python.sh", environ=environ, timeout_seconds=60 * 60 * 2, ) elif self.platform == "windows": environ["EXT_COMPILER"] = "msvc" # For some reason, the batch script %random% always runs with the same # seed. We create a random temp-dir here dir = "".join( random.choice(string.ascii_uppercase) for _ in range(10) ) return create_jobspec( self.name, [ "tools\\run_tests\\artifacts\\build_artifact_python.bat", self.py_version, "32" if self.arch == "x86" else "64", ], environ=environ, timeout_seconds=45 * 60, use_workspace=True, ) else: environ["PYTHON"] = self.py_version environ["SKIP_PIP_INSTALL"] = "TRUE" return create_jobspec( self.name, ["tools/run_tests/artifacts/build_artifact_python.sh"], environ=environ, timeout_seconds=60 * 60 * 2, use_workspace=True, ) def __str__(self): return self.name class RubyArtifact: """Builds ruby native gem.""" def __init__(self, platform, gem_platform, presubmit=False): self.name = "ruby_native_gem_%s_%s" % (platform, gem_platform) self.platform = platform self.gem_platform = gem_platform self.labels = ["artifact", "ruby", platform, gem_platform] if presubmit: self.labels.append("presubmit") def pre_build_jobspecs(self): return [] def build_jobspec(self, inner_jobs=None): environ = {} if inner_jobs is not None: # set number of parallel jobs when building native extension environ["GRPC_RUBY_BUILD_PROCS"] = str(inner_jobs) # Ruby build uses docker internally and docker cannot be nested. # We are using a custom workspace instead. return create_jobspec( self.name, [ "tools/run_tests/artifacts/build_artifact_ruby.sh", self.gem_platform, ], use_workspace=True, timeout_seconds=90 * 60, environ=environ, ) class PHPArtifact: """Builds PHP PECL package""" def __init__(self, platform, arch, presubmit=False): self.name = "php_pecl_package_{0}_{1}".format(platform, arch) self.platform = platform self.arch = arch self.labels = ["artifact", "php", platform, arch] if presubmit: self.labels.append("presubmit") def pre_build_jobspecs(self): return [] def build_jobspec(self, inner_jobs=None): del inner_jobs # arg unused as PHP artifact build is basically just packing an archive if self.platform == "linux": return create_docker_jobspec( self.name, "tools/dockerfile/test/php73_zts_debian11_{}".format(self.arch), "tools/run_tests/artifacts/build_artifact_php.sh", ) else: return create_jobspec( self.name, ["tools/run_tests/artifacts/build_artifact_php.sh"], use_workspace=True, ) class ProtocArtifact: """Builds protoc and protoc-plugin artifacts""" def __init__(self, platform, arch, presubmit=False): self.name = "protoc_%s_%s" % (platform, arch) self.platform = platform self.arch = arch self.labels = ["artifact", "protoc", platform, arch] if presubmit: self.labels.append("presubmit") def pre_build_jobspecs(self): return [] def build_jobspec(self, inner_jobs=None): environ = {} if inner_jobs is not None: # set number of parallel jobs when building protoc environ["GRPC_PROTOC_BUILD_COMPILER_JOBS"] = str(inner_jobs) if self.platform != "windows": environ["CXXFLAGS"] = "" environ["LDFLAGS"] = "" if self.platform == "linux": dockerfile_dir = ( "tools/dockerfile/grpc_artifact_centos6_{}".format( self.arch ) ) if self.arch == "aarch64": # for aarch64, use a dockcross manylinux image that will # give us both ready to use crosscompiler and sufficient backward compatibility dockerfile_dir = ( "tools/dockerfile/grpc_artifact_protoc_aarch64" ) environ["LDFLAGS"] += " -static-libgcc -static-libstdc++ -s" return create_docker_jobspec( self.name, dockerfile_dir, "tools/run_tests/artifacts/build_artifact_protoc.sh", environ=environ, ) else: environ["CXXFLAGS"] += ( " -std=c++14 -stdlib=libc++ %s" % _MACOS_COMPAT_FLAG ) return create_jobspec( self.name, ["tools/run_tests/artifacts/build_artifact_protoc.sh"], environ=environ, timeout_seconds=60 * 60, use_workspace=True, ) else: vs_tools_architecture = ( self.arch ) # architecture selector passed to vcvarsall.bat environ["ARCHITECTURE"] = vs_tools_architecture return create_jobspec( self.name, ["tools\\run_tests\\artifacts\\build_artifact_protoc.bat"], environ=environ, use_workspace=True, ) def __str__(self): return self.name def _reorder_targets_for_build_speed(targets): """Reorder targets to achieve optimal build speed""" # ruby artifact build builds multiple artifacts at once, so make sure # we start building ruby artifacts first, so that they don't end up # being a long tail once everything else finishes. return list( sorted( targets, key=lambda target: 0 if target.name.startswith("ruby_") else 1, ) ) def targets(): """Gets list of supported targets""" return _reorder_targets_for_build_speed( [ ProtocArtifact("linux", "x64", presubmit=True), ProtocArtifact("linux", "x86", presubmit=True), ProtocArtifact("linux", "aarch64", presubmit=True), ProtocArtifact("macos", "x64", presubmit=True), ProtocArtifact("windows", "x64", presubmit=True), ProtocArtifact("windows", "x86", presubmit=True), PythonArtifact( "manylinux2014", "x64", "cp37-cp37m", presubmit=True ), PythonArtifact("manylinux2014", "x64", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "x64", "cp39-cp39"), PythonArtifact("manylinux2014", "x64", "cp310-cp310"), PythonArtifact( "manylinux2014", "x64", "cp311-cp311", presubmit=True ), PythonArtifact( "manylinux2014", "x86", "cp37-cp37m", presubmit=True ), PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp39-cp39"), PythonArtifact("manylinux2014", "x86", "cp310-cp310"), PythonArtifact( "manylinux2014", "x86", "cp311-cp311", presubmit=True ), PythonArtifact( "manylinux2014", "aarch64", "cp37-cp37m", presubmit=True ), PythonArtifact( "manylinux2014", "aarch64", "cp38-cp38", presubmit=True ), PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"), PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"), PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"), PythonArtifact( "linux_extra", "armv7", "cp37-cp37m", presubmit=True ), PythonArtifact("linux_extra", "armv7", "cp38-cp38"), PythonArtifact("linux_extra", "armv7", "cp39-cp39"), PythonArtifact("linux_extra", "armv7", "cp310-cp310"), PythonArtifact( "linux_extra", "armv7", "cp311-cp311", presubmit=True ), PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"), PythonArtifact( "musllinux_1_1", "x64", "cp311-cp311", presubmit=True ), PythonArtifact( "musllinux_1_1", "x64", "cp37-cp37m", presubmit=True ), PythonArtifact("musllinux_1_1", "x64", "cp38-cp38"), PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"), PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"), PythonArtifact( "musllinux_1_1", "x86", "cp311-cp311", presubmit=True ), PythonArtifact( "musllinux_1_1", "x86", "cp37-cp37m", presubmit=True ), PythonArtifact("musllinux_1_1", "x86", "cp38-cp38"), PythonArtifact("musllinux_1_1", "x86", "cp39-cp39"), PythonArtifact("macos", "x64", "python3.7", presubmit=True), PythonArtifact("macos", "x64", "python3.8"), PythonArtifact("macos", "x64", "python3.9"), PythonArtifact("macos", "x64", "python3.10", presubmit=True), PythonArtifact("macos", "x64", "python3.11", presubmit=True), PythonArtifact("windows", "x86", "Python37_32bit", presubmit=True), PythonArtifact("windows", "x86", "Python38_32bit"), PythonArtifact("windows", "x86", "Python39_32bit"), PythonArtifact("windows", "x86", "Python310_32bit"), PythonArtifact("windows", "x86", "Python311_32bit", presubmit=True), PythonArtifact("windows", "x64", "Python37", presubmit=True), PythonArtifact("windows", "x64", "Python38"), PythonArtifact("windows", "x64", "Python39"), PythonArtifact("windows", "x64", "Python310"), PythonArtifact("windows", "x64", "Python311", presubmit=True), RubyArtifact("linux", "x86-mingw32", presubmit=True), RubyArtifact("linux", "x64-mingw32", presubmit=True), RubyArtifact("linux", "x64-mingw-ucrt", presubmit=True), RubyArtifact("linux", "x86_64-linux", presubmit=True), RubyArtifact("linux", "x86-linux", presubmit=True), RubyArtifact("linux", "aarch64-linux", presubmit=True), RubyArtifact("linux", "x86_64-darwin", presubmit=True), RubyArtifact("linux", "arm64-darwin", presubmit=True), PHPArtifact("linux", "x64", presubmit=True), PHPArtifact("macos", "x64", presubmit=True), ] )