From c98bde60cb4d23b431c3203621775ce655aab210 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jan 2017 19:12:11 +0100 Subject: [PATCH 1/2] add cmake support to run_tests.py --- .../helper_scripts/pre_build_cmake.bat | 48 +++++++++++++++ .../helper_scripts/pre_build_cmake.sh | 39 ++++++++++++ tools/run_tests/run_tests.py | 61 +++++++++++++++---- 3 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 tools/run_tests/helper_scripts/pre_build_cmake.bat create mode 100755 tools/run_tests/helper_scripts/pre_build_cmake.sh diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.bat b/tools/run_tests/helper_scripts/pre_build_cmake.bat new file mode 100644 index 00000000000..c937b9e09f8 --- /dev/null +++ b/tools/run_tests/helper_scripts/pre_build_cmake.bat @@ -0,0 +1,48 @@ +@rem Copyright 2017, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +setlocal + +cd /d %~dp0\..\..\.. + +mkdir cmake +cd cmake +mkdir build +cd build + +@rem TODO(jtattermusch): Stop hardcoding path to yasm once Jenkins workers can locate yasm correctly +cmake -G "Visual Studio 14 2015" -DgRPC_BUILD_TESTS=ON -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../.. || goto :error + +endlocal + +goto :EOF + +:error +echo Failed! +exit /b %errorlevel% diff --git a/tools/run_tests/helper_scripts/pre_build_cmake.sh b/tools/run_tests/helper_scripts/pre_build_cmake.sh new file mode 100755 index 00000000000..49083f0ede5 --- /dev/null +++ b/tools/run_tests/helper_scripts/pre_build_cmake.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +mkdir -p cmake/build +cd cmake/build + +# MSBUILD_CONFIG's values are suitable for cmake as well +cmake -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=${MSBUILD_CONFIG} ../.. diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9d767258102..a7278ed4072 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -197,10 +197,17 @@ class CLanguage(object): def configure(self, config, args): self.config = config self.args = args - if self.platform == 'windows': + if self.args.compiler == 'cmake': + _check_arch(self.args.arch, ['default']) + self._use_cmake = True + self._docker_distro = 'jessie' + self._make_options = [] + elif self.platform == 'windows': + self._use_cmake = False self._make_options = [_windows_toolset_option(self.args.compiler), _windows_arch_option(self.args.arch)] else: + self._use_cmake = False self._docker_distro, self._make_options = self._compiler_options(self.args.use_docker, self.args.compiler) if args.iomgr_platform == "uv": @@ -220,6 +227,9 @@ class CLanguage(object): out = [] binaries = get_c_tests(self.args.travis, self.test_lang) for target in binaries: + if self._use_cmake and target.get('boringssl', False): + # cmake doesn't build boringssl tests + continue polling_strategies = (_POLLING_STRATEGIES.get(self.platform, ['all']) if target.get('uses_polling', True) else ['all']) @@ -253,12 +263,18 @@ class CLanguage(object): if self.args.iomgr_platform in target.get('exclude_iomgrs', []): continue if self.platform == 'windows': - binary = 'vsprojects/%s%s/%s.exe' % ( - 'x64/' if self.args.arch == 'x64' else '', - _MSBUILD_CONFIG[self.config.build_config], - target['name']) + if self._use_cmake: + binary = 'cmake/build/%s/%s.exe' % (_MSBUILD_CONFIG[self.config.build_config], target['name']) + else: + binary = 'vsprojects/%s%s/%s.exe' % ( + 'x64/' if self.args.arch == 'x64' else '', + _MSBUILD_CONFIG[self.config.build_config], + target['name']) else: - binary = 'bins/%s/%s' % (self.config.build_config, target['name']) + if self._use_cmake: + binary = 'cmake/build/%s' % target['name'] + else: + binary = 'bins/%s/%s' % (self.config.build_config, target['name']) cpu_cost = target['cpu_cost'] if cpu_cost == 'capacity': cpu_cost = multiprocessing.cpu_count() @@ -313,10 +329,16 @@ class CLanguage(object): return self._make_options; def pre_build_steps(self): - if self.platform == 'windows': - return [['tools\\run_tests\\helper_scripts\\pre_build_c.bat']] + if self._use_cmake: + if self.platform == 'windows': + return [['tools\\run_tests\\helper_scripts\\pre_build_cmake.bat']] + else: + return [['tools/run_tests/helper_scripts/pre_build_cmake.sh']] else: - return [] + if self.platform == 'windows': + return [['tools\\run_tests\\helper_scripts\\pre_build_c.bat']] + else: + return [] def build_steps(self): return [] @@ -328,7 +350,10 @@ class CLanguage(object): return [['tools/run_tests/helper_scripts/post_tests_c.sh']] def makefile_name(self): - return 'Makefile' + if self._use_cmake: + return 'cmake/build/Makefile' + else: + return 'Makefile' def _clang_make_options(self, version_suffix=''): return ['CC=clang%s' % version_suffix, @@ -1112,7 +1137,8 @@ argp.add_argument('--compiler', 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'node0.12', 'node4', 'node5', 'node6', 'node7', 'electron1.3', - 'coreclr'], + 'coreclr', + 'cmake'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') argp.add_argument('--iomgr_platform', @@ -1248,6 +1274,12 @@ _check_arch_option(args.arch) def make_jobspec(cfg, targets, makefile='Makefile'): if platform_string() == 'windows': + if makefile.startswith('cmake/build/'): + return [jobset.JobSpec(['cmake', '--build', '.', + '--target', '%s' % target, + '--config', _MSBUILD_CONFIG[cfg]], + cwd='cmake/build', + timeout_seconds=None) for target in targets] extra_args = [] # better do parallel compilation # empirically /m:2 gives the best performance/price and should prevent @@ -1264,6 +1296,13 @@ def make_jobspec(cfg, targets, makefile='Makefile'): shell=True, timeout_seconds=None) for target in targets] else: + if targets and makefile.startswith('cmake/build/'): + # With cmake, we've passed all the build configuration in the pre-build step already + return [jobset.JobSpec([os.getenv('MAKE', 'make'), + '-j', '%d' % args.jobs] + + targets, + cwd='cmake/build', + timeout_seconds=None)] if targets: return [jobset.JobSpec([os.getenv('MAKE', 'make'), '-f', makefile, From dfb03bb94ebf522cba7880d6f83740925803fa8e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Jan 2017 19:27:58 +0100 Subject: [PATCH 2/2] add cmake build to portability tests --- tools/run_tests/run_tests_matrix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 8f70a6d2ea9..a428fb48537 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -214,6 +214,18 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) extra_args=extra_args, inner_jobs=inner_jobs) + # cmake build for C and C++ + # TODO(jtattermusch): some of the tests are failing, so we force --build_only + # to make sure it's buildable at least. + test_jobs += _generate_jobs(languages=['c', 'c++'], + configs=['dbg'], + platforms=['linux', 'windows'], + arch='default', + compiler='cmake', + labels=['portability'], + extra_args=extra_args + ['--build_only'], + inner_jobs=inner_jobs) + test_jobs += _generate_jobs(languages=['python'], configs=['dbg'], platforms=['linux'],