Rework vs_toolchain.py and vs_env.py.

Chromium's VS toolchains now maintain JSON files with the expected
environment, so we don't need to pull in gyp to figure out the batch
file to run. This drops a long obsolete dependency and will make it
possible to handle other VS architectures. (gyp internally only handled
x86 and x64.)

Also trim away the logic in vs_toolchain.py to account for
non-depot_tools toolchains. Unlike Chromium, we don't use these scripts
outside of CI/CQ.

Change-Id: I2c9fddac52eef7b4895731d78c637fdcf9c85033
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/43504
Reviewed-by: Adam Langley <agl@google.com>

.gitignore

@@ -17,7 +17,6 @@ util/bot/cmake-mac.tar.gz
 util/bot/cmake-win32
 util/bot/cmake-win32.zip
 util/bot/golang
-util/bot/gyp
 util/bot/libcxx
 util/bot/libcxxabi
 util/bot/llvm-build

util/bot/DEPS

@@ -40,9 +40,6 @@ deps = {
       'dep_type': 'cipd',
   },
 
-  'boringssl/util/bot/gyp':
-    Var('chromium_git') + '/external/gyp.git' + '@' + 'd61a9397e668fa9843c4aa7da9e79460fe590bfb',
-
   'boringssl/util/bot/libFuzzer': {
     'url': Var('chromium_git') + '/chromium/llvm-project/compiler-rt/lib/fuzzer.git' +'@' + 'debe7d2d1982e540fbd6bd78604bf001753f9e74',
     'condition': 'checkout_fuzzer',

util/bot/vs_env.py

@@ -16,8 +16,6 @@ import subprocess
 import sys
 
 import vs_toolchain
-# vs_toolchain adds gyp to sys.path.
-import gyp.MSVSVersion
 
 if len(sys.argv) < 2:
   print >>sys.stderr, "Usage: vs_env.py TARGET_ARCH CMD..."
@@ -26,12 +24,5 @@ if len(sys.argv) < 2:
 target_arch = sys.argv[1]
 cmd = sys.argv[2:]
 
-vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
-vs_version = gyp.MSVSVersion.SelectVisualStudioVersion()
-
-# Using shell=True is somewhat ugly, but the alternative is to pull in a copy
-# of the Chromium GN build's setup_toolchain.py which runs the setup script,
-# then 'set', and then parses the environment variables out. (GYP internally
-# does the same thing.)
-sys.exit(subprocess.call(vs_version.SetupScript(target_arch) + ["&&"] + cmd,
-                         shell=True))
+vs_toolchain.SetEnvironmentForCPU(target_arch)
+sys.exit(subprocess.call(cmd))

util/bot/vs_toolchain.py

@@ -4,65 +4,54 @@
 
 import json
 import os
-import pipes
-import shutil
+import os.path
 import subprocess
 import sys
 
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-sys.path.insert(0, os.path.join(script_dir, 'gyp', 'pylib'))
 json_data_file = os.path.join(script_dir, 'win_toolchain.json')
 
 
-import gyp
-
-
 # Use MSVS2015 as the default toolchain.
 CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2015'
 
 
-def SetEnvironmentAndGetRuntimeDllDirs():
-  """Sets up os.environ to use the depot_tools VS toolchain with gyp, and
-  returns the location of the VS runtime DLLs so they can be copied into
-  the output directory after gyp generation.
-  """
-  vs_runtime_dll_dirs = None
-  depot_tools_win_toolchain = \
-      bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1')))
-  if sys.platform in ('win32', 'cygwin') and depot_tools_win_toolchain:
-    if not os.path.exists(json_data_file):
-      Update()
+def SetEnvironmentForCPU(cpu):
+  """Sets the environment to build with the selected toolchain for |cpu|."""
   with open(json_data_file, 'r') as tempf:
     toolchain_data = json.load(tempf)
-
-    toolchain = toolchain_data['path']
-    version = toolchain_data['version']
-    win_sdk = toolchain_data.get('win_sdk')
-    if not win_sdk:
-      win_sdk = toolchain_data['win8sdk']
-    wdk = toolchain_data['wdk']
-    # TODO(scottmg): The order unfortunately matters in these. They should be
-    # split into separate keys for x86 and x64. (See CopyVsRuntimeDlls call
-    # below). http://crbug.com/345992
-    vs_runtime_dll_dirs = toolchain_data['runtime_dirs']
-
-    os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain
-    os.environ['GYP_MSVS_VERSION'] = version
-    # We need to make sure windows_sdk_path is set to the automated
-    # toolchain values in GYP_DEFINES, but don't want to override any
-    # otheroptions.express
-    # values there.
-    gyp_defines_dict = gyp.NameValueListToDict(gyp.ShlexEnv('GYP_DEFINES'))
-    gyp_defines_dict['windows_sdk_path'] = win_sdk
-    os.environ['GYP_DEFINES'] = ' '.join('%s=%s' % (k, pipes.quote(str(v)))
-        for k, v in gyp_defines_dict.iteritems())
-    os.environ['WINDOWSSDKDIR'] = win_sdk
-    os.environ['WDK_DIR'] = wdk
+  sdk_dir = toolchain_data['win_sdk']
+  os.environ['WINDOWSSDKDIR'] = sdk_dir
+  os.environ['WDK_DIR'] = toolchain_data['wdk']
   # Include the VS runtime in the PATH in case it's not machine-installed.
-    runtime_path = ';'.join(vs_runtime_dll_dirs)
-    os.environ['PATH'] = runtime_path + ';' + os.environ['PATH']
-  return vs_runtime_dll_dirs
+  vs_runtime_dll_dirs = toolchain_data['runtime_dirs']
+  runtime_path = os.pathsep.join(vs_runtime_dll_dirs)
+  os.environ['PATH'] = runtime_path + os.pathsep + os.environ['PATH']
+
+  # Set up the architecture-specific environment from the SetEnv files. See
+  # _LoadToolchainEnv() from setup_toolchain.py in Chromium.
+  assert cpu in ('x86', 'x64', 'arm', 'arm64')
+  with open(os.path.join(sdk_dir, 'bin', 'SetEnv.%s.json' % cpu)) as f:
+    env = json.load(f)['env']
+  if env['VSINSTALLDIR'] == [["..", "..\\"]]:
+    # Old-style paths were relative to the win_sdk\bin directory.
+    json_relative_dir = os.path.join(sdk_dir, 'bin')
+  else:
+    # New-style paths are relative to the toolchain directory, which is the
+    # parent of the SDK directory.
+    json_relative_dir = os.path.split(sdk_dir)[0]
+  for k in env:
+    entries = [os.path.join(*([json_relative_dir] + e)) for e in env[k]]
+    # clang-cl wants INCLUDE to be ;-separated even on non-Windows,
+    # lld-link wants LIB to be ;-separated even on non-Windows.  Path gets :.
+    sep = os.pathsep if k == 'PATH' else ';'
+    env[k] = sep.join(entries)
+  # PATH is a bit of a special case, it's in addition to the current PATH.
+  env['PATH'] = env['PATH'] + os.pathsep + os.environ['PATH']
+
+  for k, v in env.items():
+    os.environ[k] = v
 
 
 def FindDepotTools():
@@ -76,6 +65,8 @@ def FindDepotTools():
 def GetVisualStudioVersion():
   """Return GYP_MSVS_VERSION of Visual Studio.
   """
+  # TODO(davidben): Replace this with a command-line argument. The depot_tools
+  # script no longer needs this environment variable.
   return os.environ.get('GYP_MSVS_VERSION', CURRENT_DEFAULT_TOOLCHAIN_VERSION)
 
 
@@ -97,15 +88,10 @@ def _GetDesiredVsToolchainHashes():
 def Update():
   """Requests an update of the toolchain to the specific hashes we have at
   this revision. The update outputs a .json of the various configuration
-  information required to pass to gyp which we use in |GetToolchainDir()|.
+  information required to pass to vs_env.py which we use in
+  |SetEnvironmentForCPU()|.
   """
-  depot_tools_win_toolchain = \
-      bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1')))
-  if sys.platform in ('win32', 'cygwin') and depot_tools_win_toolchain:
   depot_tools_path = FindDepotTools()
-    # Necessary so that get_toolchain_if_necessary.py will put the VS toolkit
-    # in the correct directory.
-    os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion()
   get_toolchain_args = [
       sys.executable,
       os.path.join(depot_tools_path,
@@ -114,7 +100,6 @@ def Update():
       '--output-json', json_data_file,
     ] + _GetDesiredVsToolchainHashes()
   subprocess.check_call(get_toolchain_args)
-
   return 0