Speed up windows C/C++ builds by using cmake Ninja generator (#28833)

* support building C/C++ with ninja

* make cmake_ninja_vs2015 build the default for C/C++

* fix vcvarsall location for vs2017

tools/internal_ci/helper_scripts/prepare_build_windows.bat

@@ -36,6 +36,13 @@ netsh interface ip set dns "Local Area Connection 8" static 169.254.169.254 prim
 netsh interface ip add dnsservers "Local Area Connection 8" 8.8.8.8 index=2
 netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3
 
+@rem Install nasm (required for boringssl assembly optimized build as boringssl no long supports yasm)
+@rem Downloading from GCS should be very reliables when on a GCP VM.
+mkdir C:\nasm
+curl -sSL -o C:\nasm\nasm.exe https://storage.googleapis.com/grpc-build-helper/nasm-2.15.05/nasm.exe || goto :error
+set PATH=C:\nasm;%PATH%
+nasm
+
 @rem Only install C# dependencies if we are running C# tests
 If "%PREPARE_BUILD_INSTALL_DEPS_CSHARP%" == "true" (
   @rem C# prerequisites: Install dotnet SDK

tools/internal_ci/windows/grpc_build_artifacts.bat

@@ -22,10 +22,6 @@ IF "%cd%"=="T:\src" (
 )
 endlocal
 
-@rem Boringssl build no longer supports yasm
-choco uninstall yasm -y --limit-output
-choco install nasm -y --limit-output || exit /b 1
-
 @rem enter repo root
 cd /d %~dp0\..\..\..
 

tools/run_tests/helper_scripts/build_cxx.bat

@@ -21,10 +21,43 @@ cd cmake
 mkdir build
 cd build
 
-cmake -DgRPC_BUILD_TESTS=ON %* ../.. || goto :error
+If "%GRPC_BUILD_ACTIVATE_VS_TOOLS%" == "2015" (
+  @rem set cl.exe build environment to build with VS2015 tooling
+  @rem this is required for Ninja build to work
+  call "%VS140COMNTOOLS%..\..\VC\vcvarsall.bat" %GRPC_BUILD_VS_TOOLS_ARCHITECTURE%
+  @rem restore command echo
+  echo on
+)
 
-@rem GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX will be set to either "c" or "cxx"
-cmake --build . --target buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% --config %MSBUILD_CONFIG% || goto :error
+@rem Workaround a bug where VS150COMNTOOLS is not set due to a bug in VS 2017 installer
+@rem see https://developercommunity.visualstudio.com/t/installing-visualstudio-build-tools-doesnt-add-env/17435
+set "VS150COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\"
+
+If "%GRPC_BUILD_ACTIVATE_VS_TOOLS%" == "2017" (
+  @rem set cl.exe build environment to build with VS2017 tooling
+  @rem this is required for Ninja build to work
+  call "%VS150COMNTOOLS%..\..\VC\Auxiliary\Build\vcvarsall.bat" %GRPC_BUILD_VS_TOOLS_ARCHITECTURE%
+  @rem restore command echo
+  echo on
+)
+
+If "%GRPC_CMAKE_GENERATOR%" == "Ninja" (
+  @rem Use ninja
+
+  @rem Select MSVC compiler (cl.exe) explicitly to make sure we don't end up gcc from mingw or cygwin
+  @rem (both are on path in kokoro win workers)
+  cmake -G "%GRPC_CMAKE_GENERATOR%" -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE="%MSBUILD_CONFIG%" %* ../.. || goto :error
+
+  ninja -j%GRPC_RUN_TESTS_JOBS% buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% || goto :error
+
+) else (
+  @rem Use one of the Visual Studio generators.
+
+  cmake -G "%GRPC_CMAKE_GENERATOR%" -A "%GRPC_CMAKE_ARCHITECTURE%" -DgRPC_BUILD_TESTS=ON -DgRPC_BUILD_MSVC_MP_COUNT=%GRPC_RUN_TESTS_JOBS% %* ../.. || goto :error
+
+  @rem GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX will be set to either "c" or "cxx"
+  cmake --build . --target buildtests_%GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX% --config %MSBUILD_CONFIG% || goto :error
+)
 
 endlocal
 

tools/run_tests/run_tests.py

@@ -250,20 +250,43 @@ class CLanguage(object):
         self.args = args
         if self.platform == 'windows':
             _check_compiler(self.args.compiler, [
-                'default', 'cmake', 'cmake_vs2015', 'cmake_vs2017',
-                'cmake_vs2019'
+                'default',
+                'cmake',
+                'cmake_ninja_vs2015',
+                'cmake_ninja_vs2017',
+                'cmake_vs2015',
+                'cmake_vs2017',
+                'cmake_vs2019',
             ])
             _check_arch(self.args.arch, ['default', 'x64', 'x86'])
-            if self.args.compiler == 'cmake_vs2019':
-                cmake_generator_option = 'Visual Studio 16 2019'
+
+            activate_vs_tools = ''
+            if self.args.compiler == 'cmake_ninja_vs2015' or self.args.compiler == 'cmake' or self.args.compiler == 'default':
+                # cmake + ninja build is the default because it is faster and supports boringssl assembly optimizations
+                # the compiler used is exactly the same as for cmake_vs2015
+                cmake_generator = 'Ninja'
+                activate_vs_tools = '2015'
+            elif self.args.compiler == 'cmake_ninja_vs2017':
+                cmake_generator = 'Ninja'
+                activate_vs_tools = '2017'
+            elif self.args.compiler == 'cmake_vs2015':
+                cmake_generator = 'Visual Studio 14 2015'
             elif self.args.compiler == 'cmake_vs2017':
-                cmake_generator_option = 'Visual Studio 15 2017'
+                cmake_generator = 'Visual Studio 15 2017'
+            elif self.args.compiler == 'cmake_vs2019':
+                cmake_generator = 'Visual Studio 16 2019'
             else:
-                cmake_generator_option = 'Visual Studio 14 2015'
-            cmake_arch_option = 'x64' if self.args.arch == 'x64' else 'Win32'
-            self._cmake_configure_extra_args = [
-                '-G', cmake_generator_option, '-A', cmake_arch_option
-            ]
+                print('should never reach here.')
+                sys.exit(1)
+
+            self._cmake_configure_extra_args = []
+            self._cmake_generator_windows = cmake_generator
+            # required to pass as cmake "-A" configuration for VS builds (but not for Ninja)
+            self._cmake_architecture_windows = 'x64' if self.args.arch == 'x64' else 'Win32'
+            # when builing with Ninja, the VS common tools need to be activated first
+            self._activate_vs_tools_windows = activate_vs_tools
+            self._vs_tools_architecture_windows = 'x64' if self.args.arch == 'x64' else 'x86'
+
         else:
             if self.platform == 'linux':
                 # Allow all the known architectures. _check_arch_option has already checked that we're not doing
@@ -429,7 +452,16 @@ class CLanguage(object):
 
     def build_steps_environ(self):
         """Extra environment variables set for pre_build_steps and build_steps jobs."""
-        return {'GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX': self.lang_suffix}
+        environ = {'GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX': self.lang_suffix}
+        if self.platform == 'windows':
+            environ['GRPC_CMAKE_GENERATOR'] = self._cmake_generator_windows
+            environ[
+                'GRPC_CMAKE_ARCHITECTURE'] = self._cmake_architecture_windows
+            environ[
+                'GRPC_BUILD_ACTIVATE_VS_TOOLS'] = self._activate_vs_tools_windows
+            environ[
+                'GRPC_BUILD_VS_TOOLS_ARCHITECTURE'] = self._vs_tools_architecture_windows
+        return environ
 
     def post_tests_steps(self):
         if self.platform == 'windows':
@@ -1547,6 +1579,8 @@ argp.add_argument(
         'electron1.6',
         'coreclr',
         'cmake',
+        'cmake_ninja_vs2015',
+        'cmake_ninja_vs2017',
         'cmake_vs2015',
         'cmake_vs2017',
         'cmake_vs2019',

tools/run_tests/run_tests_matrix.py

@@ -289,6 +289,17 @@ def _create_portability_test_jobs(extra_args=[],
                                 extra_args=extra_args,
                                 inner_jobs=inner_jobs)
 
+    # portability C on Windows with the "Visual Studio" cmake
+    # generator, i.e. not using Ninja (to verify that we can still build with msbuild)
+    test_jobs += _generate_jobs(languages=['c'],
+                                configs=['dbg'],
+                                platforms=['windows'],
+                                arch='default',
+                                compiler='cmake_vs2015',
+                                labels=['portability', 'corelang'],
+                                extra_args=extra_args,
+                                inner_jobs=inner_jobs)
+
     # portability C++ on Windows
     # TODO(jtattermusch): some of the tests are failing, so we force --build_only
     test_jobs += _generate_jobs(languages=['c++'],
@@ -307,7 +318,7 @@ def _create_portability_test_jobs(extra_args=[],
                                 configs=['dbg'],
                                 platforms=['windows'],
                                 arch='x64',
-                                compiler='cmake_vs2017',
+                                compiler='cmake_ninja_vs2017',
                                 labels=['portability', 'corelang'],
                                 extra_args=extra_args + ['--build_only'],
                                 inner_jobs=inner_jobs,