From 48d80bb6cfbc464bf5e9d146ad86210bb4e9891e Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Fri, 19 Feb 2021 13:36:20 -0500 Subject: [PATCH] Use CIPD Go packages. These are ultimately just the upstream tarballs, but it's one less ad-hoc script to maintain. Change-Id: Ia93a7a9d4944d482e4e4137587998790e8e59294 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/45784 Commit-Queue: David Benjamin Reviewed-by: Adam Langley --- util/bot/DEPS | 18 ++- util/bot/UPDATING | 3 - util/bot/go/bootstrap.py | 296 --------------------------------------- util/bot/go/env.py | 68 +++++---- 4 files changed, 54 insertions(+), 331 deletions(-) delete mode 100755 util/bot/go/bootstrap.py diff --git a/util/bot/DEPS b/util/bot/DEPS index db7dfa111..e710c0f5f 100644 --- a/util/bot/DEPS +++ b/util/bot/DEPS @@ -21,14 +21,16 @@ vars = { 'checkout_libcxx': False, 'vs_version': 'env', - # Run the following commands to see the latest builds in CIPD: - # - # cipd describe infra/3pp/tools/cmake/linux-amd64 -version latest - # cipd describe infra/3pp/tools/cmake/mac-amd64 -version latest + # Run the following command to see the latest builds in CIPD: + # cipd describe PACKAGE_NAME -version latest + + # infra/3pp/tools/cmake/linux-amd64 # # TODO(https://crbug.com/1176531): Update to a newer CMake when available # from CIPD. This is currently blocked on the linked bug. 'cmake_version': 'version:3.13.5', + # infra/3pp/tools/go/linux-amd64 + 'go_version': 'version:1.16', # Update the following from # https://chromium.googlesource.com/chromium/src/+/master/DEPS @@ -79,6 +81,14 @@ deps = { 'dep_type': 'cipd', }, + 'boringssl/util/bot/golang': { + 'packages': [{ + 'package': 'infra/3pp/tools/go/${{platform}}', + 'version': Var('go_version'), + }], + 'dep_type': 'cipd', + }, + 'boringssl/util/bot/libFuzzer': { 'url': Var('chromium_git') + '/chromium/llvm-project/compiler-rt/lib/fuzzer.git' +'@' + Var('libfuzzer_revision'), 'condition': 'checkout_fuzzer', diff --git a/util/bot/UPDATING b/util/bot/UPDATING index 533eef81a..2e6b91445 100644 --- a/util/bot/UPDATING +++ b/util/bot/UPDATING @@ -10,9 +10,6 @@ To update to newer revisions, follow these instructions: DEPS: Update the variables as described in the comments. -go/bootstrap.py: Set TOOLSET_VERSION to the latest release of Go, found at - https://golang.org/dl/. - update_clang.py: Set CLANG_REVISION and CLANG_SUB_REVISION to the values used in Chromium, found at https://chromium.googlesource.com/chromium/src/+/master/tools/clang/scripts/update.py diff --git a/util/bot/go/bootstrap.py b/util/bot/go/bootstrap.py deleted file mode 100755 index 0765bb522..000000000 --- a/util/bot/go/bootstrap.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python -# Copyright 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Modified from go/bootstrap.py in Chromium infrastructure's repository to patch -# out everything but the core toolchain. -# -# https://chromium.googlesource.com/infra/infra/ - -"""Prepares a local hermetic Go installation. - -- Downloads and unpacks the Go toolset in ../golang. -""" - -import contextlib -import logging -import os -import platform -import shutil -import stat -import subprocess -import sys -import tarfile -import tempfile -import urllib -import zipfile - -# TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to -# git completely. - -LOGGER = logging.getLogger(__name__) - - -# /path/to/util/bot -ROOT = os.path.dirname(os.path.abspath(__file__)) - -# Where to install Go toolset to. GOROOT would be /go. -TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang') - -# Default workspace with infra go code. -WORKSPACE = os.path.join(ROOT, 'go') - -# Platform depended suffix for executable files. -EXE_SFX = '.exe' if sys.platform == 'win32' else '' - -# Pinned version of Go toolset to download. -TOOLSET_VERSION = 'go1.16' - -# Platform dependent portion of a download URL. See http://golang.org/dl/. -TOOLSET_VARIANTS = { - ('darwin', 'x86-64'): 'darwin-amd64.tar.gz', - ('linux2', 'x86-32'): 'linux-386.tar.gz', - ('linux2', 'x86-64'): 'linux-amd64.tar.gz', - ('win32', 'x86-32'): 'windows-386.zip', - ('win32', 'x86-64'): 'windows-amd64.zip', -} - -# Download URL root. -DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang' - - -class Failure(Exception): - """Bootstrap failed.""" - - -def get_toolset_url(): - """URL of a platform specific Go toolset archive.""" - # TODO(vadimsh): Support toolset for cross-compilation. - arch = { - 'amd64': 'x86-64', - 'x86_64': 'x86-64', - 'i386': 'x86-32', - 'x86': 'x86-32', - }.get(platform.machine().lower()) - variant = TOOLSET_VARIANTS.get((sys.platform, arch)) - if not variant: - # TODO(vadimsh): Compile go lang from source. - raise Failure('Unrecognized platform') - return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant) - - -def read_file(path): - """Returns contents of a given file or None if not readable.""" - assert isinstance(path, (list, tuple)) - try: - with open(os.path.join(*path), 'r') as f: - return f.read() - except IOError: - return None - - -def write_file(path, data): - """Writes |data| to a file.""" - assert isinstance(path, (list, tuple)) - with open(os.path.join(*path), 'w') as f: - f.write(data) - - -def remove_directory(path): - """Recursively removes a directory.""" - assert isinstance(path, (list, tuple)) - p = os.path.join(*path) - if not os.path.exists(p): - return - LOGGER.info('Removing %s', p) - # Crutch to remove read-only file (.git/* in particular) on Windows. - def onerror(func, path, _exc_info): - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise - shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None) - - -def install_toolset(toolset_root, url): - """Downloads and installs Go toolset. - - GOROOT would be /go/. - """ - if not os.path.exists(toolset_root): - os.makedirs(toolset_root) - pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:]) - - LOGGER.info('Downloading %s...', url) - download_file(url, pkg_path) - - LOGGER.info('Extracting...') - if pkg_path.endswith('.zip'): - with zipfile.ZipFile(pkg_path, 'r') as f: - f.extractall(toolset_root) - elif pkg_path.endswith('.tar.gz'): - with tarfile.open(pkg_path, 'r:gz') as f: - f.extractall(toolset_root) - else: - raise Failure('Unrecognized archive format') - - LOGGER.info('Validating...') - if not check_hello_world(toolset_root): - raise Failure('Something is not right, test program doesn\'t work') - - -def download_file(url, path): - """Fetches |url| to |path|.""" - last_progress = [0] - def report(a, b, c): - progress = int(a * b * 100.0 / c) - if progress != last_progress[0]: - print >> sys.stderr, 'Downloading... %d%%' % progress - last_progress[0] = progress - # TODO(vadimsh): Use something less crippled, something that validates SSL. - urllib.urlretrieve(url, path, reporthook=report) - - -@contextlib.contextmanager -def temp_dir(path): - """Creates a temporary directory, then deletes it.""" - tmp = tempfile.mkdtemp(dir=path) - try: - yield tmp - finally: - remove_directory([tmp]) - - -def check_hello_world(toolset_root): - """Compiles and runs 'hello world' program to verify that toolset works.""" - with temp_dir(toolset_root) as tmp: - path = os.path.join(tmp, 'hello.go') - write_file([path], r""" - package main - func main() { println("hello, world\n") } - """) - out = subprocess.check_output( - [get_go_exe(toolset_root), 'run', path], - env=get_go_environ(toolset_root, tmp), - stderr=subprocess.STDOUT) - if out.strip() != 'hello, world': - LOGGER.error('Failed to run sample program:\n%s', out) - return False - return True - - -def ensure_toolset_installed(toolset_root): - """Installs or updates Go toolset if necessary. - - Returns True if new toolset was installed. - """ - installed = read_file([toolset_root, 'INSTALLED_TOOLSET']) - available = get_toolset_url() - if installed == available: - LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION) - return False - - LOGGER.info('Installing Go toolset.') - LOGGER.info(' Old toolset is %s', installed) - LOGGER.info(' New toolset is %s', available) - remove_directory([toolset_root]) - install_toolset(toolset_root, available) - LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION) - write_file([toolset_root, 'INSTALLED_TOOLSET'], available) - return True - - -def get_go_environ( - toolset_root, - workspace=None): - """Returns a copy of os.environ with added GO* environment variables. - - Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent. - - Args: - toolset_root: GOROOT would be /go. - workspace: main workspace directory or None if compiling in GOROOT. - """ - env = os.environ.copy() - env['GOROOT'] = os.path.join(toolset_root, 'go') - if workspace: - env['GOBIN'] = os.path.join(workspace, 'bin') - else: - env.pop('GOBIN', None) - - all_go_paths = [] - if workspace: - all_go_paths.append(workspace) - env['GOPATH'] = os.pathsep.join(all_go_paths) - - # New PATH entries. - paths_to_add = [ - os.path.join(env['GOROOT'], 'bin'), - env.get('GOBIN'), - ] - - # Make sure not to add duplicates entries to PATH over and over again when - # get_go_environ is invoked multiple times. - path = env['PATH'].split(os.pathsep) - paths_to_add = [p for p in paths_to_add if p and p not in path] - env['PATH'] = os.pathsep.join(paths_to_add + path) - - return env - - -def get_go_exe(toolset_root): - """Returns path to go executable.""" - return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX) - - -def bootstrap(logging_level): - """Installs all dependencies in default locations. - - Supposed to be called at the beginning of some script (it modifies logger). - - Args: - logging_level: logging level of bootstrap process. - """ - logging.basicConfig() - LOGGER.setLevel(logging_level) - ensure_toolset_installed(TOOLSET_ROOT) - - -def prepare_go_environ(): - """Returns dict with environment variables to set to use Go toolset. - - Installs or updates the toolset if necessary. - """ - bootstrap(logging.INFO) - return get_go_environ(TOOLSET_ROOT, WORKSPACE) - - -def find_executable(name, workspaces): - """Returns full path to an executable in some bin/ (in GOROOT or GOBIN).""" - basename = name - if EXE_SFX and basename.endswith(EXE_SFX): - basename = basename[:-len(EXE_SFX)] - roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')] - for path in workspaces: - roots.extend([ - os.path.join(path, 'bin'), - ]) - for root in roots: - full_path = os.path.join(root, basename + EXE_SFX) - if os.path.exists(full_path): - return full_path - return name - - -def main(args): - if args: - print >> sys.stderr, sys.modules[__name__].__doc__, - return 2 - bootstrap(logging.DEBUG) - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/util/bot/go/env.py b/util/bot/go/env.py index 820968c9b..91eace098 100755 --- a/util/bot/go/env.py +++ b/util/bot/go/env.py @@ -8,42 +8,54 @@ # # https://chromium.googlesource.com/infra/infra/ -"""Can be used to point environment variable to hermetic Go toolset. - -Usage (on linux and mac): -$ eval `./env.py` -$ go version - -Or it can be used to wrap a command: +"""Used to wrap a command: $ ./env.py go version """ assert __name__ == '__main__' -import imp import os import subprocess import sys -# Do not want to mess with sys.path, load the module directly. -bootstrap = imp.load_source( - 'bootstrap', os.path.join(os.path.dirname(__file__), 'bootstrap.py')) - -old = os.environ.copy() -new = bootstrap.prepare_go_environ() - -if len(sys.argv) == 1: - for key, value in sorted(new.iteritems()): - if old.get(key) != value: - print 'export %s="%s"' % (key, value) +# /path/to/util/bot +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Platform depended suffix for executable files. +EXE_SFX = '.exe' if sys.platform == 'win32' else '' + +def get_go_environ(goroot): + """Returns a copy of os.environ with added GOROOT and PATH variables.""" + env = os.environ.copy() + env['GOROOT'] = goroot + gobin = os.path.join(goroot, 'bin') + path = env['PATH'].split(os.pathsep) + if gobin not in path: + env['PATH'] = os.pathsep.join([gobin] + path) + return env + +def find_executable(name, goroot): + """Returns full path to an executable in GOROOT.""" + basename = name + if EXE_SFX and basename.endswith(EXE_SFX): + basename = basename[:-len(EXE_SFX)] + full_path = os.path.join(goroot, 'bin', basename + EXE_SFX) + if os.path.exists(full_path): + return full_path + return name + +# TODO(davidben): Now that we use CIPD to fetch Go, this script does not do +# much. Switch to setting up GOROOT and PATH in the recipe? +goroot = os.path.join(ROOT, 'golang') +new = get_go_environ(goroot) + +exe = sys.argv[1] +if exe == 'python': + exe = sys.executable else: - exe = sys.argv[1] - if exe == 'python': - exe = sys.executable - else: - # Help Windows to find the executable in new PATH, do it only when - # executable is referenced by name (and not by path). - if os.sep not in exe: - exe = bootstrap.find_executable(exe, [bootstrap.WORKSPACE]) - sys.exit(subprocess.call([exe] + sys.argv[2:], env=new)) + # Help Windows to find the executable in new PATH, do it only when + # executable is referenced by name (and not by path). + if os.sep not in exe: + exe = find_executable(exe, goroot) +sys.exit(subprocess.call([exe] + sys.argv[2:], env=new))