grpc/tools/run_tests/artifacts/artifact_targets.py

#!/usr/bin/env python3
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Definition of targets to build artifacts."""

import os.path
import random
import string
import sys

sys.path.insert(0, os.path.abspath('..'))
import python_utils.jobset as jobset

_LATEST_MANYLINUX = "manylinux2014"


def create_docker_jobspec(name,
                          dockerfile_dir,
                          shell_command,
                          environ={},
                          flake_retries=0,
                          timeout_retries=0,
                          timeout_seconds=30 * 60,
                          extra_docker_args=None,
                          verbose_success=False):
    """Creates jobspec for a task running under docker."""
    environ = environ.copy()
    environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name

    docker_args = []
    for k, v in list(environ.items()):
        docker_args += ['-e', '%s=%s' % (k, v)]
    docker_env = {
        'DOCKERFILE_DIR': dockerfile_dir,
        'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
        'DOCKER_RUN_SCRIPT_COMMAND': shell_command,
        'OUTPUT_DIR': 'artifacts'
    }
    if extra_docker_args is not None:
        docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
    jobspec = jobset.JobSpec(
        cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
        docker_args,
        environ=docker_env,
        shortname='build_artifact.%s' % (name),
        timeout_seconds=timeout_seconds,
        flake_retries=flake_retries,
        timeout_retries=timeout_retries,
        verbose_success=verbose_success)
    return jobspec


def create_jobspec(name,
                   cmdline,
                   environ={},
                   shell=False,
                   flake_retries=0,
                   timeout_retries=0,
                   timeout_seconds=30 * 60,
                   use_workspace=False,
                   cpu_cost=1.0,
                   verbose_success=False):
    """Creates jobspec."""
    environ = environ.copy()
    if use_workspace:
        environ['WORKSPACE_NAME'] = 'workspace_%s' % name
        environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
        cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
                  ] + cmdline
    else:
        environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)

    jobspec = jobset.JobSpec(cmdline=cmdline,
                             environ=environ,
                             shortname='build_artifact.%s' % (name),
                             timeout_seconds=timeout_seconds,
                             flake_retries=flake_retries,
                             timeout_retries=timeout_retries,
                             shell=shell,
                             cpu_cost=cpu_cost,
                             verbose_success=verbose_success)
    return jobspec


_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10'

_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'}


class PythonArtifact:
    """Builds Python artifacts."""

    def __init__(self, platform, arch, py_version, presubmit=False):
        self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
        self.platform = platform
        self.arch = arch
        self.labels = ['artifact', 'python', platform, arch, py_version]
        if presubmit:
            self.labels.append('presubmit')
        self.py_version = py_version
        if platform == _LATEST_MANYLINUX:
            self.labels.append('latest-manylinux')
        if 'manylinux' in platform:
            self.labels.append('linux')
        if 'linux_extra' in platform:
            # linux_extra wheels used to be built by a separate kokoro job.
            # Their build is now much faster, so they can be included
            # in the regular artifact build.
            self.labels.append('linux')
        if 'musllinux' in platform:
            self.labels.append('linux')

    def pre_build_jobspecs(self):
        return []

    def build_jobspec(self, inner_jobs=None):
        environ = {}
        if inner_jobs is not None:
            # set number of parallel jobs when building native extension
            # building the native extension is the most time-consuming part of the build
            environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = str(inner_jobs)

        # This is necessary due to https://github.com/pypa/wheel/issues/406.
        # distutils incorrectly generates a universal2 artifact that only contains
        # x86_64 libraries.
        if self.platform == "macos" and self.arch == "x64":
            environ["GRPC_UNIVERSAL2_REPAIR"] = "true"

        if self.platform == 'linux_extra':
            # Crosscompilation build for armv7 (e.g. Raspberry Pi)
            environ['PYTHON'] = '/opt/python/{}/bin/python3'.format(
                self.py_version)
            environ['PIP'] = '/opt/python/{}/bin/pip3'.format(self.py_version)
            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
            environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
            return create_docker_jobspec(
                self.name,
                'tools/dockerfile/grpc_artifact_python_linux_{}'.format(
                    self.arch),
                'tools/run_tests/artifacts/build_artifact_python.sh',
                environ=environ,
                timeout_seconds=60 * 60)
        elif 'manylinux' in self.platform:
            if self.arch == 'x86':
                environ['SETARCH_CMD'] = 'linux32'
            # Inside the manylinux container, the python installations are located in
            # special places...
            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
                self.py_version)
            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
            if self.arch == 'aarch64':
                environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
            else:
                # only run auditwheel if we're not crosscompiling
                environ['GRPC_RUN_AUDITWHEEL_REPAIR'] = 'TRUE'
                # only build the packages that depend on grpcio-tools
                # if we're not crosscompiling.
                # - they require protoc to run on current architecture
                # - they only have sdist packages anyway, so it's useless to build them again
                environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
            return create_docker_jobspec(
                self.name,
                'tools/dockerfile/grpc_artifact_python_%s_%s' %
                (self.platform, self.arch),
                'tools/run_tests/artifacts/build_artifact_python.sh',
                environ=environ,
                timeout_seconds=60 * 60 * 2)
        elif 'musllinux' in self.platform:
            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
                self.py_version)
            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
            environ['GRPC_RUN_AUDITWHEEL_REPAIR'] = 'TRUE'
            environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE'
            return create_docker_jobspec(
                self.name,
                'tools/dockerfile/grpc_artifact_python_%s_%s' %
                (self.platform, self.arch),
                'tools/run_tests/artifacts/build_artifact_python.sh',
                environ=environ,
                timeout_seconds=60 * 60 * 2)
        elif self.platform == 'windows':
            environ['EXT_COMPILER'] = 'msvc'
            # For some reason, the batch script %random% always runs with the same
            # seed.  We create a random temp-dir here
            dir = ''.join(
                random.choice(string.ascii_uppercase) for _ in range(10))
            return create_jobspec(self.name, [
                'tools\\run_tests\\artifacts\\build_artifact_python.bat',
                self.py_version, '32' if self.arch == 'x86' else '64'
            ],
                                  environ=environ,
                                  timeout_seconds=45 * 60,
                                  use_workspace=True)
        else:
            environ['PYTHON'] = self.py_version
            environ['SKIP_PIP_INSTALL'] = 'TRUE'
            return create_jobspec(
                self.name,
                ['tools/run_tests/artifacts/build_artifact_python.sh'],
                environ=environ,
                timeout_seconds=60 * 60 * 2,
                use_workspace=True)

    def __str__(self):
        return self.name


class RubyArtifact:
    """Builds ruby native gem."""

    def __init__(self, platform, gem_platform, presubmit=False):
        self.name = 'ruby_native_gem_%s_%s' % (platform, gem_platform)
        self.platform = platform
        self.gem_platform = gem_platform
        self.labels = ['artifact', 'ruby', platform, gem_platform]
        if presubmit:
            self.labels.append('presubmit')

    def pre_build_jobspecs(self):
        return []

    def build_jobspec(self, inner_jobs=None):
        environ = {}
        if inner_jobs is not None:
            # set number of parallel jobs when building native extension
            environ['GRPC_RUBY_BUILD_PROCS'] = str(inner_jobs)
        # Ruby build uses docker internally and docker cannot be nested.
        # We are using a custom workspace instead.
        return create_jobspec(self.name, [
            'tools/run_tests/artifacts/build_artifact_ruby.sh',
            self.gem_platform
        ],
                              use_workspace=True,
                              timeout_seconds=90 * 60,
                              environ=environ)


class CSharpExtArtifact:
    """Builds C# native extension library"""

    def __init__(self, platform, arch, arch_abi=None, presubmit=False):
        self.name = 'csharp_ext_%s_%s' % (platform, arch)
        self.platform = platform
        self.arch = arch
        self.arch_abi = arch_abi
        self.labels = ['artifact', 'csharp', platform, arch]
        if arch_abi:
            self.name += '_%s' % arch_abi
            self.labels.append(arch_abi)
        if presubmit:
            self.labels.append('presubmit')

    def pre_build_jobspecs(self):
        return []

    def build_jobspec(self, inner_jobs=None):
        environ = {}
        if inner_jobs is not None:
            # set number of parallel jobs when building native extension
            environ['GRPC_CSHARP_BUILD_EXT_COMPILER_JOBS'] = str(inner_jobs)

        if self.arch == 'android':
            environ['ANDROID_ABI'] = self.arch_abi
            return create_docker_jobspec(
                self.name,
                'tools/dockerfile/grpc_artifact_android_ndk',
                'tools/run_tests/artifacts/build_artifact_csharp_android.sh',
                environ=environ)
        elif self.arch == 'ios':
            return create_jobspec(
                self.name,
                ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'],
                timeout_seconds=60 * 60,
                use_workspace=True,
                environ=environ)
        elif self.platform == 'windows':
            return create_jobspec(self.name, [
                'tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
                self.arch
            ],
                                  timeout_seconds=45 * 60,
                                  use_workspace=True,
                                  environ=environ)
        else:
            if self.platform == 'linux':
                dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format(
                    self.arch)
                if self.arch == 'aarch64':
                    # for aarch64, use a dockcross manylinux image that will
                    # give us both ready to use crosscompiler and sufficient backward compatibility
                    dockerfile_dir = 'tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64'
                return create_docker_jobspec(
                    self.name,
                    dockerfile_dir,
                    'tools/run_tests/artifacts/build_artifact_csharp.sh',
                    environ=environ)
            else:
                return create_jobspec(
                    self.name,
                    ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
                    timeout_seconds=45 * 60,
                    use_workspace=True,
                    environ=environ)

    def __str__(self):
        return self.name


class PHPArtifact:
    """Builds PHP PECL package"""

    def __init__(self, platform, arch, presubmit=False):
        self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
        self.platform = platform
        self.arch = arch
        self.labels = ['artifact', 'php', platform, arch]
        if presubmit:
            self.labels.append('presubmit')

    def pre_build_jobspecs(self):
        return []

    def build_jobspec(self, inner_jobs=None):
        del inner_jobs  # arg unused as PHP artifact build is basically just packing an archive
        if self.platform == 'linux':
            return create_docker_jobspec(
                self.name,
                'tools/dockerfile/test/php73_zts_debian11_{}'.format(self.arch),
                'tools/run_tests/artifacts/build_artifact_php.sh')
        else:
            return create_jobspec(
                self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'],
                use_workspace=True)


class ProtocArtifact:
    """Builds protoc and protoc-plugin artifacts"""

    def __init__(self, platform, arch, presubmit=False):
        self.name = 'protoc_%s_%s' % (platform, arch)
        self.platform = platform
        self.arch = arch
        self.labels = ['artifact', 'protoc', platform, arch]
        if presubmit:
            self.labels.append('presubmit')

    def pre_build_jobspecs(self):
        return []

    def build_jobspec(self, inner_jobs=None):
        environ = {}
        if inner_jobs is not None:
            # set number of parallel jobs when building protoc
            environ['GRPC_PROTOC_BUILD_COMPILER_JOBS'] = str(inner_jobs)

        if self.platform != 'windows':
            environ['CXXFLAGS'] = ''
            environ['LDFLAGS'] = ''
            if self.platform == 'linux':
                dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format(
                    self.arch)
                if self.arch == 'aarch64':
                    # for aarch64, use a dockcross manylinux image that will
                    # give us both ready to use crosscompiler and sufficient backward compatibility
                    dockerfile_dir = 'tools/dockerfile/grpc_artifact_protoc_aarch64'
                environ['LDFLAGS'] += ' -static-libgcc -static-libstdc++ -s'
                return create_docker_jobspec(
                    self.name,
                    dockerfile_dir,
                    'tools/run_tests/artifacts/build_artifact_protoc.sh',
                    environ=environ)
            else:
                environ[
                    'CXXFLAGS'] += ' -std=c++14 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
                return create_jobspec(
                    self.name,
                    ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
                    environ=environ,
                    timeout_seconds=60 * 60,
                    use_workspace=True)
        else:
            vs_tools_architecture = self.arch  # architecture selector passed to vcvarsall.bat
            environ['ARCHITECTURE'] = vs_tools_architecture
            return create_jobspec(
                self.name,
                ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
                environ=environ,
                use_workspace=True)

    def __str__(self):
        return self.name


def _reorder_targets_for_build_speed(targets):
    """Reorder targets to achieve optimal build speed"""
    # ruby artifact build builds multiple artifacts at once, so make sure
    # we start building ruby artifacts first, so that they don't end up
    # being a long tail once everything else finishes.
    return list(
        sorted(targets,
               key=lambda target: 0 if target.name.startswith('ruby_') else 1))


def targets():
    """Gets list of supported targets"""
    return _reorder_targets_for_build_speed([
        ProtocArtifact('linux', 'x64', presubmit=True),
        ProtocArtifact('linux', 'x86', presubmit=True),
        ProtocArtifact('linux', 'aarch64', presubmit=True),
        ProtocArtifact('macos', 'x64', presubmit=True),
        ProtocArtifact('windows', 'x64', presubmit=True),
        ProtocArtifact('windows', 'x86', presubmit=True),
        CSharpExtArtifact('linux', 'x64', presubmit=True),
        CSharpExtArtifact('linux', 'aarch64', presubmit=True),
        CSharpExtArtifact('macos', 'x64', presubmit=True),
        CSharpExtArtifact('windows', 'x64', presubmit=True),
        CSharpExtArtifact('windows', 'x86', presubmit=True),
        CSharpExtArtifact('linux',
                          'android',
                          arch_abi='arm64-v8a',
                          presubmit=True),
        CSharpExtArtifact('linux',
                          'android',
                          arch_abi='armeabi-v7a',
                          presubmit=True),
        CSharpExtArtifact('linux', 'android', arch_abi='x86', presubmit=True),
        CSharpExtArtifact('macos', 'ios', presubmit=True),
        PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m', presubmit=True),
        PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m', presubmit=True),
        PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'),
        PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'),
        PythonArtifact('manylinux2014', 'x64', 'cp310-cp310', presubmit=True),
        PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m', presubmit=True),
        PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m', presubmit=True),
        PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'),
        PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'),
        PythonArtifact('manylinux2014', 'x86', 'cp310-cp310', presubmit=True),
        PythonArtifact('manylinux2014', 'aarch64', 'cp36-cp36m',
                       presubmit=True),
        PythonArtifact('manylinux2014', 'aarch64', 'cp37-cp37m'),
        PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38', presubmit=True),
        PythonArtifact('manylinux2014', 'aarch64', 'cp39-cp39'),
        PythonArtifact('manylinux2014', 'aarch64', 'cp310-cp310'),
        PythonArtifact('linux_extra', 'armv7', 'cp36-cp36m', presubmit=True),
        PythonArtifact('linux_extra', 'armv7', 'cp37-cp37m'),
        PythonArtifact('linux_extra', 'armv7', 'cp38-cp38'),
        PythonArtifact('linux_extra', 'armv7', 'cp39-cp39'),
        PythonArtifact('linux_extra', 'armv7', 'cp310-cp310', presubmit=True),
        PythonArtifact('musllinux_1_1', 'x64', 'cp310-cp310', presubmit=True),
        PythonArtifact('musllinux_1_1', 'x64', 'cp36-cp36m', presubmit=True),
        PythonArtifact('musllinux_1_1', 'x64', 'cp37-cp37m'),
        PythonArtifact('musllinux_1_1', 'x64', 'cp38-cp38'),
        PythonArtifact('musllinux_1_1', 'x64', 'cp39-cp39'),
        PythonArtifact('musllinux_1_1', 'x86', 'cp310-cp310', presubmit=True),
        PythonArtifact('musllinux_1_1', 'x86', 'cp36-cp36m', presubmit=True),
        PythonArtifact('musllinux_1_1', 'x86', 'cp37-cp37m'),
        PythonArtifact('musllinux_1_1', 'x86', 'cp38-cp38'),
        PythonArtifact('musllinux_1_1', 'x86', 'cp39-cp39'),
        PythonArtifact('macos', 'x64', 'python3.6', presubmit=True),
        PythonArtifact('macos', 'x64', 'python3.7'),
        PythonArtifact('macos', 'x64', 'python3.8'),
        PythonArtifact('macos', 'x64', 'python3.9'),
        PythonArtifact('macos', 'x64', 'python3.10', presubmit=True),
        PythonArtifact('windows', 'x86', 'Python36_32bit', presubmit=True),
        PythonArtifact('windows', 'x86', 'Python37_32bit'),
        PythonArtifact('windows', 'x86', 'Python38_32bit'),
        PythonArtifact('windows', 'x86', 'Python39_32bit'),
        PythonArtifact('windows', 'x86', 'Python310_32bit', presubmit=True),
        PythonArtifact('windows', 'x64', 'Python36', presubmit=True),
        PythonArtifact('windows', 'x64', 'Python37'),
        PythonArtifact('windows', 'x64', 'Python38'),
        PythonArtifact('windows', 'x64', 'Python39'),
        PythonArtifact('windows', 'x64', 'Python310', presubmit=True),
        RubyArtifact('linux', 'x86-mingw32', presubmit=True),
        RubyArtifact('linux', 'x64-mingw32', presubmit=True),
        RubyArtifact('linux', 'x64-mingw-ucrt', presubmit=True),
        RubyArtifact('linux', 'x86_64-linux', presubmit=True),
        RubyArtifact('linux', 'x86-linux', presubmit=True),
        RubyArtifact('linux', 'x86_64-darwin', presubmit=True),
        RubyArtifact('linux', 'arm64-darwin', presubmit=True),
        PHPArtifact('linux', 'x64', presubmit=True),
        PHPArtifact('macos', 'x64', presubmit=True),
    ])