From 17328e7019b8b7406a82a508beedf3138ad1681b Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 7 May 2017 03:17:42 -0400 Subject: [PATCH 1/6] Add coverage export for tests. --- .coveragerc | 17 +++++++++++++ .gitignore | 1 + docs/markdown/Release-notes-for-0.41.0.md | 7 +++++- run_tests.py | 29 ++++++++++++++++++----- 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..a345f4684 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,17 @@ +[run] +branch = True +concurrency = + multiprocessing +data_file = .coverage/coverage +parallel = True +source = + meson.py + mesonbuild/ + mesonconf.py + mesonintrospect.py + mesonrewriter.py + mesontest.py + run_cross_test.py + run_project_tests.py + run_tests.py + run_unittests.py diff --git a/.gitignore b/.gitignore index b3506894f..f16854a48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /.idea __pycache__ +.coverage /install dir /work area diff --git a/docs/markdown/Release-notes-for-0.41.0.md b/docs/markdown/Release-notes-for-0.41.0.md index a96ded0c3..7da9ed7cc 100644 --- a/docs/markdown/Release-notes-for-0.41.0.md +++ b/docs/markdown/Release-notes-for-0.41.0.md @@ -52,7 +52,12 @@ installed. This is roughly equivalent to the `distcheck` target in other build systems. Currently this only works for projects using Git and only with the Ninja backend. - ## Support for passing arguments to Rust compiler Targets for building rust now take a `rust_args` keyword. + +## Code coverage export for tests + +Code coverage can be generated for tests by passing the `--cov` argument to +the `run_tests.py` test runner. Note, since multiple processes are used, +coverage must be combined before producing a report (`coverage3 combine`.) diff --git a/run_tests.py b/run_tests.py index 7ef9cf47e..00c259503 100755 --- a/run_tests.py +++ b/run_tests.py @@ -19,6 +19,7 @@ import sys import time import shutil import subprocess +import tempfile import platform from mesonbuild import mesonlib from mesonbuild.environment import detect_ninja @@ -125,6 +126,13 @@ class FakeEnvironment(object): return False if __name__ == '__main__': + # Enable coverage early... + enable_coverage = '--cov' in sys.argv + if enable_coverage: + os.makedirs('.coverage', exist_ok=True) + sys.argv.remove('--cov') + import coverage + coverage.process_startup() returncode = 0 # Iterate over list in reverse order to find the last --backend arg backend = Backend.ninja @@ -164,10 +172,19 @@ if __name__ == '__main__': # Can't pass arguments to unit tests, so set the backend to use in the environment env = os.environ.copy() env['MESON_UNIT_TEST_BACKEND'] = backend.name - returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'] + units, env=env) - # Ubuntu packages do not have a binary without -6 suffix. - if should_run_linux_cross_tests(): - print('Running cross compilation tests.\n') - returncode += subprocess.call([sys.executable, 'run_cross_test.py', 'cross/ubuntu-armhf.txt']) - returncode += subprocess.call([sys.executable, 'run_project_tests.py'] + sys.argv[1:]) + with tempfile.TemporaryDirectory() as td: + # Enable coverage on all subsequent processes. + if enable_coverage: + with open(os.path.join(td, 'usercustomize.py'), 'w') as f: + f.write('import coverage\n' + 'coverage.process_startup()\n') + env['COVERAGE_PROCESS_START'] = '.coveragerc' + env['PYTHONPATH'] = os.pathsep.join([td] + env.get('PYTHONPATH', [])) + + returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'] + units, env=env) + # Ubuntu packages do not have a binary without -6 suffix. + if should_run_linux_cross_tests(): + print('Running cross compilation tests.\n') + returncode += subprocess.call([sys.executable, 'run_cross_test.py', 'cross/ubuntu-armhf.txt'], env=env) + returncode += subprocess.call([sys.executable, 'run_project_tests.py'] + sys.argv[1:], env=env) sys.exit(returncode) From 8a9479f8eaab73676ef98c570dfafaff2b5453ec Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 7 May 2017 03:22:36 -0400 Subject: [PATCH 2/6] Enable code coverage on Travis. --- .coveragerc | 33 +++++++++++++++++++++++++++++++++ .travis.yml | 16 ++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index a345f4684..c3d41d4b8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -15,3 +15,36 @@ source = run_project_tests.py run_tests.py run_unittests.py + +# Aliases to /root/ are needed because Travis runs in docker at /root/. +[paths] +meson = + meson.py + /root/meson.py +mesonbuild = + mesonbuild/ + /root/mesonbuild/ +mesonconf = + mesonconf.py + /root/mesonconf.py +mesonintrospect = + mesonintrospect.py + /root/mesonintrospect.py +mesonrewriter = + mesonrewriter.py + /root/mesonrewriter.py +mesontest = + mesontest.py + /root/mesontest.py +run_cross_test = + run_cross_test.py + /root/run_cross_test.py +run_project_tests = + run_project_tests.py + /root/run_project_tests.py +run_tests = + run_tests.py + /root/run_tests.py +run_unittests = + run_unittests.py + /root/run_unittests.py diff --git a/.travis.yml b/.travis.yml index 5f4318d83..5a7339729 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,8 @@ before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install ninja python3; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker pull jpakkane/mesonci:zesty; fi + - pip3 install codecov + - mkdir .coverage # We need to copy the current checkout inside the Docker container, # because it has the MR id to be tested checked out. @@ -40,5 +42,15 @@ script: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo FROM jpakkane/mesonci:zesty > Dockerfile; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo ADD . /root >> Dockerfile; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker build -t withgit .; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker run withgit /bin/sh -c "cd /root && TRAVIS=true CC=$CC CXX=$CXX OBJC=$CC OBJCXX=$CXX ./run_tests.py -- $MESON_ARGS"; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then SDKROOT=$(xcodebuild -version -sdk macosx Path) OBJC=$CC OBJCXX=$CXX ./run_tests.py --backend=ninja -- $MESON_ARGS ; fi + - | + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + ci_env=`bash <(curl -s https://codecov.io/env)` + docker run $ci_env -v ${PWD}/.coverage:/root/.coverage \ + withgit \ + /bin/sh -c "cd /root && CC=$CC CXX=$CXX OBJC=$CC OBJCXX=$CXX ./run_tests.py --cov -- $MESON_ARGS; chmod -R a+rwX .coverage" + fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then SDKROOT=$(xcodebuild -version -sdk macosx Path) OBJC=$CC OBJCXX=$CXX ./run_tests.py --cov --backend=ninja -- $MESON_ARGS ; fi + +after_success: + - coverage3 combine + - codecov From fcfe3c32316ade43b65eeed4337f305f03e0687b Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 7 May 2017 03:36:23 -0400 Subject: [PATCH 3/6] Use a wrapper script to run tests in Cygwin. --- .appveyor.yml | 13 +++++++++---- ci/appveyor-test.sh | 6 ------ ci/run-in-cygwin.bat | 6 ++++++ 3 files changed, 15 insertions(+), 10 deletions(-) delete mode 100755 ci/appveyor-test.sh create mode 100644 ci/run-in-cygwin.bat diff --git a/.appveyor.yml b/.appveyor.yml index 708e1b176..8ab193184 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -54,26 +54,31 @@ skip_commits: - docs/**/* install: + - cmd: set "ORIG_PATH=%PATH%" # Use the x86 python only when building for x86 for the cpython tests. # For all other archs (including, say, arm), use the x64 python. - ps: (new-object net.webclient).DownloadFile('https://www.dropbox.com/s/bbzvepq85hv47x1/ninja.exe?dl=1', 'C:\projects\meson\ninja.exe') - cmd: if %arch%==x86 (set MESON_PYTHON_PATH=C:\python34) else (set MESON_PYTHON_PATH=C:\python34-x64) - cmd: echo Using Python at %MESON_PYTHON_PATH% + - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% ) - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% ) - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% ) + - cmd: if %compiler%==msys2-mingw (if %arch%==x86 (set "PATH=C:\msys64\mingw32\bin;%PATH%") else (set "PATH=C:\msys64\mingw64\bin;%PATH%")) + - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat ) + - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" ) + - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat ) + - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python ) build_script: - cmd: echo No build step. - - cmd: if not %compiler%==cygwin if %backend%==ninja ( ninja.exe --version ) else ( MSBuild /version & echo. ) + - cmd: if %backend%==ninja ( %WRAPPER% ninja.exe --version ) else ( MSBuild /version & echo. ) test_script: - cmd: echo Running tests for %arch% and %compiler% with the %backend% backend - - cmd: set "ORIG_PATH=%PATH%" - - cmd: if %compiler%==cygwin ( set "PATH=%CYGWIN_ROOT%\bin;%SYSTEMROOT%\system32" && bash -lc "cd $APPVEYOR_BUILD_FOLDER && ci/appveyor-test.sh" ) - - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" && python run_tests.py --backend=%backend% ) + - cmd: "%WRAPPER% %PYTHON% run_tests.py --backend=%backend%" on_finish: - set "PATH=%ORIG_PATH%" diff --git a/ci/appveyor-test.sh b/ci/appveyor-test.sh deleted file mode 100755 index 2f2963054..000000000 --- a/ci/appveyor-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -echo ninja $(ninja --version) -python3 --version -V - -python3 run_tests.py --backend=${backend} diff --git a/ci/run-in-cygwin.bat b/ci/run-in-cygwin.bat new file mode 100644 index 000000000..146d28ecd --- /dev/null +++ b/ci/run-in-cygwin.bat @@ -0,0 +1,6 @@ +if _%arch%_ == _x64_ set CYGWIN_ROOT=C:\cygwin64 +if _%arch%_ == _x86_ set CYGWIN_ROOT=C:\cygwin + +set PATH=%CYGWIN_ROOT%\bin;%SYSTEMROOT%\system32 + +env.exe -- %* From 6e2fff1f4421309a182ca5cf55d971ff31fda7d1 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 7 May 2017 03:43:04 -0400 Subject: [PATCH 4/6] Enable code coverage on AppVeyor. * Install pip on msys (which doesn't have it) * Install codecov/coverage everywhere. * Generate coverage XML file manually because codecov tries to call the coverage script itself and it's in a weird place on the path. --- .appveyor.yml | 24 ++++++++++++++++++------ ci/appveyor-install.bat | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 8ab193184..dba0bef8c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -59,18 +59,24 @@ install: # For all other archs (including, say, arm), use the x64 python. - ps: (new-object net.webclient).DownloadFile('https://www.dropbox.com/s/bbzvepq85hv47x1/ninja.exe?dl=1', 'C:\projects\meson\ninja.exe') - cmd: if %arch%==x86 (set MESON_PYTHON_PATH=C:\python34) else (set MESON_PYTHON_PATH=C:\python34-x64) - - cmd: echo Using Python at %MESON_PYTHON_PATH% + # Set paths and config for each build type. - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% ) - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% ) - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% ) - - cmd: if %compiler%==msys2-mingw (if %arch%==x86 (set "PATH=C:\msys64\mingw32\bin;%PATH%") else (set "PATH=C:\msys64\mingw64\bin;%PATH%")) - - - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat ) - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" ) - - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat ) - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python ) + - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat ) + - cmd: if %compiler%==cygwin ( %WRAPPER% which %PYTHON% ) else ( where %PYTHON% ) + + # Install additional packages needed for specific builds. + - ps: If($Env:compiler -eq 'msys2-mingw') {(new-object Net.WebClient).DownloadFile('https://bootstrap.pypa.io/get-pip.py', 'C:\projects\meson\get-pip.py')} + - cmd: if %compiler%==msys2-mingw ( %PYTHON% "C:\projects\meson\get-pip.py" ) + - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat ) + + # Install additional packages needed for all builds. + - cmd: "%WRAPPER% %PYTHON% -m pip install codecov" build_script: - cmd: echo No build step. @@ -78,7 +84,13 @@ build_script: test_script: - cmd: echo Running tests for %arch% and %compiler% with the %backend% backend - - cmd: "%WRAPPER% %PYTHON% run_tests.py --backend=%backend%" + - cmd: "%WRAPPER% %PYTHON% run_tests.py --cov --backend=%backend%" + +after_test: + - cmd: "%WRAPPER% %PYTHON% -m coverage combine" + # Generate XML report manually because codecov module doesn't know how to run it properly. + - cmd: "%WRAPPER% %PYTHON% -m coverage xml" + - cmd: "%WRAPPER% %PYTHON% -m codecov -X search pycov -f coverage.xml" on_finish: - set "PATH=%ORIG_PATH%" diff --git a/ci/appveyor-install.bat b/ci/appveyor-install.bat index 0c1ce440b..becc80a1f 100644 --- a/ci/appveyor-install.bat +++ b/ci/appveyor-install.bat @@ -7,5 +7,5 @@ if _%arch%_ == _x86_ set SETUP=setup-x86.exe && set CYGWIN_ROOT=C:\cygwin if not exist %CACHE% mkdir %CACHE% echo Updating Cygwin and installing ninja and test prerequisites -%CYGWIN_ROOT%\%SETUP% -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -l "%CACHE%" -g -P "ninja,gcc-objc,gcc-objc++,libglib2.0-devel,zlib-devel" +%CYGWIN_ROOT%\%SETUP% -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -l "%CACHE%" -g -P "ninja,gcc-objc,gcc-objc++,libglib2.0-devel,zlib-devel,python3-pip" echo Install done From 617a0da42690f65fe68e49ab6930d8908c2826f3 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Mon, 8 May 2017 02:39:16 -0400 Subject: [PATCH 5/6] Give each coverage build a unique name. Travis builds are currently build number (123.1, 123.2, etc.) and AppVeyor is some random string, making it hard to determine which builds cover what. --- .appveyor.yml | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index dba0bef8c..1e20a372c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -90,7 +90,7 @@ after_test: - cmd: "%WRAPPER% %PYTHON% -m coverage combine" # Generate XML report manually because codecov module doesn't know how to run it properly. - cmd: "%WRAPPER% %PYTHON% -m coverage xml" - - cmd: "%WRAPPER% %PYTHON% -m codecov -X search pycov -f coverage.xml" + - cmd: "%WRAPPER% %PYTHON% -m codecov -X search pycov -f coverage.xml -n windows-%arch%-%compiler%-%backend%" on_finish: - set "PATH=%ORIG_PATH%" diff --git a/.travis.yml b/.travis.yml index 5a7339729..fb86292d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,4 +53,4 @@ script: after_success: - coverage3 combine - - codecov + - codecov -n "${TRAVIS_OS_NAME}-${CC}-{$(echo ${MESON_ARGS} | sed -e 's/^--//g' -e 's/ --/,/g')}" From 083b2756d1f1c41a06e9b1d2c60b2b2751d1070f Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Wed, 10 May 2017 21:31:25 -0400 Subject: [PATCH 6/6] Add codecov badge. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c57568096..73a5bfdad 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ build system. [![PyPI](https://img.shields.io/pypi/v/meson.svg)](https://pypi.python.org/pypi/meson) [![Travis](https://travis-ci.org/mesonbuild/meson.svg?branch=master)](https://travis-ci.org/mesonbuild/meson) [![Appveyor](https://ci.appveyor.com/api/projects/status/l5c8v71ninew2i3p?svg=true)](https://ci.appveyor.com/project/jpakkane/meson) +[![Codecov](https://codecov.io/gh/mesonbuild/meson/coverage.svg?branch=master)](https://codecov.io/gh/mesonbuild/meson/branch/master) #### Dependencies