From 73b69308acd6257f4056bd2f1e91e9cf6b259a26 Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Mon, 19 Oct 2020 22:05:49 -0400 Subject: [PATCH] Rework vs_toolchain.py and vs_env.py. Chromium's VS toolchains now maintain JSON files with the expected environment, so we don't need to pull in gyp to figure out the batch file to run. This drops a long obsolete dependency and will make it possible to handle other VS architectures. (gyp internally only handled x86 and x64.) Also trim away the logic in vs_toolchain.py to account for non-depot_tools toolchains. Unlike Chromium, we don't use these scripts outside of CI/CQ. Change-Id: I2c9fddac52eef7b4895731d78c637fdcf9c85033 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/43504 Reviewed-by: Adam Langley --- .gitignore | 1 - util/bot/DEPS | 3 -- util/bot/vs_env.py | 13 +---- util/bot/vs_toolchain.py | 113 +++++++++++++++++---------------------- 4 files changed, 51 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 8ce23d1ca..83a64740d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ util/bot/cmake-mac.tar.gz util/bot/cmake-win32 util/bot/cmake-win32.zip util/bot/golang -util/bot/gyp util/bot/libcxx util/bot/libcxxabi util/bot/llvm-build diff --git a/util/bot/DEPS b/util/bot/DEPS index 8fe7b3597..8544fe9c3 100644 --- a/util/bot/DEPS +++ b/util/bot/DEPS @@ -40,9 +40,6 @@ deps = { 'dep_type': 'cipd', }, - 'boringssl/util/bot/gyp': - Var('chromium_git') + '/external/gyp.git' + '@' + 'd61a9397e668fa9843c4aa7da9e79460fe590bfb', - 'boringssl/util/bot/libFuzzer': { 'url': Var('chromium_git') + '/chromium/llvm-project/compiler-rt/lib/fuzzer.git' +'@' + 'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'condition': 'checkout_fuzzer', diff --git a/util/bot/vs_env.py b/util/bot/vs_env.py index 184750063..13217cadd 100644 --- a/util/bot/vs_env.py +++ b/util/bot/vs_env.py @@ -16,8 +16,6 @@ import subprocess import sys import vs_toolchain -# vs_toolchain adds gyp to sys.path. -import gyp.MSVSVersion if len(sys.argv) < 2: print >>sys.stderr, "Usage: vs_env.py TARGET_ARCH CMD..." @@ -26,12 +24,5 @@ if len(sys.argv) < 2: target_arch = sys.argv[1] cmd = sys.argv[2:] -vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() -vs_version = gyp.MSVSVersion.SelectVisualStudioVersion() - -# Using shell=True is somewhat ugly, but the alternative is to pull in a copy -# of the Chromium GN build's setup_toolchain.py which runs the setup script, -# then 'set', and then parses the environment variables out. (GYP internally -# does the same thing.) -sys.exit(subprocess.call(vs_version.SetupScript(target_arch) + ["&&"] + cmd, - shell=True)) +vs_toolchain.SetEnvironmentForCPU(target_arch) +sys.exit(subprocess.call(cmd)) diff --git a/util/bot/vs_toolchain.py b/util/bot/vs_toolchain.py index eb3e04973..6e94df351 100644 --- a/util/bot/vs_toolchain.py +++ b/util/bot/vs_toolchain.py @@ -4,65 +4,54 @@ import json import os -import pipes -import shutil +import os.path import subprocess import sys script_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(script_dir, 'gyp', 'pylib')) json_data_file = os.path.join(script_dir, 'win_toolchain.json') -import gyp - - # Use MSVS2015 as the default toolchain. CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2015' -def SetEnvironmentAndGetRuntimeDllDirs(): - """Sets up os.environ to use the depot_tools VS toolchain with gyp, and - returns the location of the VS runtime DLLs so they can be copied into - the output directory after gyp generation. - """ - vs_runtime_dll_dirs = None - depot_tools_win_toolchain = \ - bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) - if sys.platform in ('win32', 'cygwin') and depot_tools_win_toolchain: - if not os.path.exists(json_data_file): - Update() - with open(json_data_file, 'r') as tempf: - toolchain_data = json.load(tempf) - - toolchain = toolchain_data['path'] - version = toolchain_data['version'] - win_sdk = toolchain_data.get('win_sdk') - if not win_sdk: - win_sdk = toolchain_data['win8sdk'] - wdk = toolchain_data['wdk'] - # TODO(scottmg): The order unfortunately matters in these. They should be - # split into separate keys for x86 and x64. (See CopyVsRuntimeDlls call - # below). http://crbug.com/345992 - vs_runtime_dll_dirs = toolchain_data['runtime_dirs'] - - os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain - os.environ['GYP_MSVS_VERSION'] = version - # We need to make sure windows_sdk_path is set to the automated - # toolchain values in GYP_DEFINES, but don't want to override any - # otheroptions.express - # values there. - gyp_defines_dict = gyp.NameValueListToDict(gyp.ShlexEnv('GYP_DEFINES')) - gyp_defines_dict['windows_sdk_path'] = win_sdk - os.environ['GYP_DEFINES'] = ' '.join('%s=%s' % (k, pipes.quote(str(v))) - for k, v in gyp_defines_dict.iteritems()) - os.environ['WINDOWSSDKDIR'] = win_sdk - os.environ['WDK_DIR'] = wdk - # Include the VS runtime in the PATH in case it's not machine-installed. - runtime_path = ';'.join(vs_runtime_dll_dirs) - os.environ['PATH'] = runtime_path + ';' + os.environ['PATH'] - return vs_runtime_dll_dirs +def SetEnvironmentForCPU(cpu): + """Sets the environment to build with the selected toolchain for |cpu|.""" + with open(json_data_file, 'r') as tempf: + toolchain_data = json.load(tempf) + sdk_dir = toolchain_data['win_sdk'] + os.environ['WINDOWSSDKDIR'] = sdk_dir + os.environ['WDK_DIR'] = toolchain_data['wdk'] + # Include the VS runtime in the PATH in case it's not machine-installed. + vs_runtime_dll_dirs = toolchain_data['runtime_dirs'] + runtime_path = os.pathsep.join(vs_runtime_dll_dirs) + os.environ['PATH'] = runtime_path + os.pathsep + os.environ['PATH'] + + # Set up the architecture-specific environment from the SetEnv files. See + # _LoadToolchainEnv() from setup_toolchain.py in Chromium. + assert cpu in ('x86', 'x64', 'arm', 'arm64') + with open(os.path.join(sdk_dir, 'bin', 'SetEnv.%s.json' % cpu)) as f: + env = json.load(f)['env'] + if env['VSINSTALLDIR'] == [["..", "..\\"]]: + # Old-style paths were relative to the win_sdk\bin directory. + json_relative_dir = os.path.join(sdk_dir, 'bin') + else: + # New-style paths are relative to the toolchain directory, which is the + # parent of the SDK directory. + json_relative_dir = os.path.split(sdk_dir)[0] + for k in env: + entries = [os.path.join(*([json_relative_dir] + e)) for e in env[k]] + # clang-cl wants INCLUDE to be ;-separated even on non-Windows, + # lld-link wants LIB to be ;-separated even on non-Windows. Path gets :. + sep = os.pathsep if k == 'PATH' else ';' + env[k] = sep.join(entries) + # PATH is a bit of a special case, it's in addition to the current PATH. + env['PATH'] = env['PATH'] + os.pathsep + os.environ['PATH'] + + for k, v in env.items(): + os.environ[k] = v def FindDepotTools(): @@ -76,6 +65,8 @@ def FindDepotTools(): def GetVisualStudioVersion(): """Return GYP_MSVS_VERSION of Visual Studio. """ + # TODO(davidben): Replace this with a command-line argument. The depot_tools + # script no longer needs this environment variable. return os.environ.get('GYP_MSVS_VERSION', CURRENT_DEFAULT_TOOLCHAIN_VERSION) @@ -97,24 +88,18 @@ def _GetDesiredVsToolchainHashes(): def Update(): """Requests an update of the toolchain to the specific hashes we have at this revision. The update outputs a .json of the various configuration - information required to pass to gyp which we use in |GetToolchainDir()|. + information required to pass to vs_env.py which we use in + |SetEnvironmentForCPU()|. """ - depot_tools_win_toolchain = \ - bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) - if sys.platform in ('win32', 'cygwin') and depot_tools_win_toolchain: - depot_tools_path = FindDepotTools() - # Necessary so that get_toolchain_if_necessary.py will put the VS toolkit - # in the correct directory. - os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion() - get_toolchain_args = [ - sys.executable, - os.path.join(depot_tools_path, - 'win_toolchain', - 'get_toolchain_if_necessary.py'), - '--output-json', json_data_file, - ] + _GetDesiredVsToolchainHashes() - subprocess.check_call(get_toolchain_args) - + depot_tools_path = FindDepotTools() + get_toolchain_args = [ + sys.executable, + os.path.join(depot_tools_path, + 'win_toolchain', + 'get_toolchain_if_necessary.py'), + '--output-json', json_data_file, + ] + _GetDesiredVsToolchainHashes() + subprocess.check_call(get_toolchain_args) return 0