Merge pull request #1664 from QuLogic/codecov

Enable code coverage

.appveyor.yml

@@ -54,26 +54,43 @@ skip_commits:
     - docs/**/*
 
 install:
+  - cmd: set "ORIG_PATH=%PATH%"
   # Use the x86 python only when building for x86 for the cpython tests.
   # For all other archs (including, say, arm), use the x64 python.
   - ps: (new-object net.webclient).DownloadFile('https://www.dropbox.com/s/bbzvepq85hv47x1/ninja.exe?dl=1', 'C:\projects\meson\ninja.exe')
   - cmd: if %arch%==x86 (set MESON_PYTHON_PATH=C:\python34) else (set MESON_PYTHON_PATH=C:\python34-x64)
-  - cmd: echo Using Python at %MESON_PYTHON_PATH%
+
+  # Set paths and config for each build type.
   - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% )
   - cmd: if %compiler%==msys2-mingw (if %arch%==x86 (set "PATH=C:\msys64\mingw32\bin;%PATH%") else (set "PATH=C:\msys64\mingw64\bin;%PATH%"))
+  - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" )
+  - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python )
+  - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat )
+  - cmd: if %compiler%==cygwin ( %WRAPPER% which %PYTHON% ) else ( where %PYTHON% )
+
+  # Install additional packages needed for specific builds.
+  - ps: If($Env:compiler -eq 'msys2-mingw') {(new-object Net.WebClient).DownloadFile('https://bootstrap.pypa.io/get-pip.py', 'C:\projects\meson\get-pip.py')}
+  - cmd: if %compiler%==msys2-mingw ( %PYTHON% "C:\projects\meson\get-pip.py" )
   - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat )
 
+  # Install additional packages needed for all builds.
+  - cmd: "%WRAPPER% %PYTHON% -m pip install codecov"
+
 build_script:
   - cmd: echo No build step.
-  - cmd: if not %compiler%==cygwin if %backend%==ninja ( ninja.exe --version ) else ( MSBuild /version & echo. )
+  - cmd: if %backend%==ninja ( %WRAPPER% ninja.exe --version ) else ( MSBuild /version & echo. )
 
 test_script:
   - cmd: echo Running tests for %arch% and %compiler% with the %backend% backend
-  - cmd: set "ORIG_PATH=%PATH%"
-  - cmd: if %compiler%==cygwin ( set "PATH=%CYGWIN_ROOT%\bin;%SYSTEMROOT%\system32" && bash -lc "cd $APPVEYOR_BUILD_FOLDER && ci/appveyor-test.sh" )
-  - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" && python run_tests.py --backend=%backend% )
+  - cmd: "%WRAPPER% %PYTHON% run_tests.py --cov --backend=%backend%"
+
+after_test:
+  - cmd: "%WRAPPER% %PYTHON% -m coverage combine"
+  # Generate XML report manually because codecov module doesn't know how to run it properly.
+  - cmd: "%WRAPPER% %PYTHON% -m coverage xml"
+  - cmd: "%WRAPPER% %PYTHON% -m codecov -X search pycov -f coverage.xml -n windows-%arch%-%compiler%-%backend%"
 
 on_finish:
   - set "PATH=%ORIG_PATH%"

.coveragerc

@@ -0,0 +1,50 @@
+[run]
+branch = True
+concurrency =
+    multiprocessing
+data_file = .coverage/coverage
+parallel = True
+source =
+    meson.py
+    mesonbuild/
+    mesonconf.py
+    mesonintrospect.py
+    mesonrewriter.py
+    mesontest.py
+    run_cross_test.py
+    run_project_tests.py
+    run_tests.py
+    run_unittests.py
+
+# Aliases to /root/ are needed because Travis runs in docker at /root/.
+[paths]
+meson =
+    meson.py
+    /root/meson.py
+mesonbuild =
+    mesonbuild/
+    /root/mesonbuild/
+mesonconf =
+    mesonconf.py
+    /root/mesonconf.py
+mesonintrospect =
+    mesonintrospect.py
+    /root/mesonintrospect.py
+mesonrewriter =
+    mesonrewriter.py
+    /root/mesonrewriter.py
+mesontest =
+    mesontest.py
+    /root/mesontest.py
+run_cross_test =
+    run_cross_test.py
+    /root/run_cross_test.py
+run_project_tests =
+    run_project_tests.py
+    /root/run_project_tests.py
+run_tests =
+    run_tests.py
+    /root/run_tests.py
+run_unittests =
+    run_unittests.py
+    /root/run_unittests.py

.gitignore

@@ -5,6 +5,7 @@
 /.idea
 
 __pycache__
+.coverage
 /install dir
 /work area
 

.travis.yml

@@ -32,6 +32,8 @@ before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install ninja python3; fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker pull jpakkane/mesonci:zesty; fi
+  - pip3 install codecov
+  - mkdir .coverage
 
 # We need to copy the current checkout inside the Docker container,
 # because it has the MR id to be tested checked out.
@@ -40,5 +42,15 @@ script:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo FROM jpakkane/mesonci:zesty > Dockerfile; fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo ADD . /root >> Dockerfile; fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker build -t withgit .; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then docker run withgit /bin/sh -c "cd /root && TRAVIS=true CC=$CC CXX=$CXX OBJC=$CC OBJCXX=$CXX ./run_tests.py -- $MESON_ARGS"; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then SDKROOT=$(xcodebuild -version -sdk macosx Path) OBJC=$CC OBJCXX=$CXX ./run_tests.py --backend=ninja -- $MESON_ARGS ; fi
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      ci_env=`bash <(curl -s https://codecov.io/env)`
+      docker run $ci_env -v ${PWD}/.coverage:/root/.coverage \
+        withgit \
+          /bin/sh -c "cd /root && CC=$CC CXX=$CXX OBJC=$CC OBJCXX=$CXX ./run_tests.py --cov -- $MESON_ARGS; chmod -R a+rwX .coverage"
+    fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then SDKROOT=$(xcodebuild -version -sdk macosx Path) OBJC=$CC OBJCXX=$CXX ./run_tests.py --cov --backend=ninja -- $MESON_ARGS ; fi
+
+after_success:
+  - coverage3 combine
+  - codecov -n "${TRAVIS_OS_NAME}-${CC}-{$(echo ${MESON_ARGS} | sed -e 's/^--//g' -e 's/ --/,/g')}"

README.md

@@ -9,6 +9,7 @@ build system.
 [![PyPI](https://img.shields.io/pypi/v/meson.svg)](https://pypi.python.org/pypi/meson)
 [![Travis](https://travis-ci.org/mesonbuild/meson.svg?branch=master)](https://travis-ci.org/mesonbuild/meson)
 [![Appveyor](https://ci.appveyor.com/api/projects/status/l5c8v71ninew2i3p?svg=true)](https://ci.appveyor.com/project/jpakkane/meson)
+[![Codecov](https://codecov.io/gh/mesonbuild/meson/coverage.svg?branch=master)](https://codecov.io/gh/mesonbuild/meson/branch/master)
 
 #### Dependencies
 

ci/appveyor-install.bat

@@ -7,5 +7,5 @@ if _%arch%_ == _x86_ set SETUP=setup-x86.exe && set CYGWIN_ROOT=C:\cygwin
 if not exist %CACHE% mkdir %CACHE%
 
 echo Updating Cygwin and installing ninja and test prerequisites
-%CYGWIN_ROOT%\%SETUP% -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -l "%CACHE%" -g -P "ninja,gcc-objc,gcc-objc++,libglib2.0-devel,zlib-devel"
+%CYGWIN_ROOT%\%SETUP% -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -l "%CACHE%" -g -P "ninja,gcc-objc,gcc-objc++,libglib2.0-devel,zlib-devel,python3-pip"
 echo Install done

ci/appveyor-test.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-echo ninja $(ninja --version)
-python3 --version -V
-
-python3 run_tests.py --backend=${backend}

ci/run-in-cygwin.bat

@@ -0,0 +1,6 @@
+if _%arch%_ == _x64_ set CYGWIN_ROOT=C:\cygwin64
+if _%arch%_ == _x86_ set CYGWIN_ROOT=C:\cygwin
+
+set PATH=%CYGWIN_ROOT%\bin;%SYSTEMROOT%\system32
+
+env.exe -- %*

docs/markdown/Release-notes-for-0.41.0.md

@@ -52,7 +52,12 @@ installed. This is roughly equivalent to the `distcheck` target in
 other build systems. Currently this only works for projects using Git
 and only with the Ninja backend.
 
-
 ## Support for passing arguments to Rust compiler
 
 Targets for building rust now take a `rust_args` keyword.
+
+## Code coverage export for tests
+
+Code coverage can be generated for tests by passing the `--cov` argument to
+the `run_tests.py` test runner. Note, since multiple processes are used,
+coverage must be combined before producing a report (`coverage3 combine`.)

run_tests.py

@@ -19,6 +19,7 @@ import sys
 import time
 import shutil
 import subprocess
+import tempfile
 import platform
 from mesonbuild import mesonlib
 from mesonbuild.environment import detect_ninja
@@ -125,6 +126,13 @@ class FakeEnvironment(object):
         return False
 
 if __name__ == '__main__':
+    # Enable coverage early...
+    enable_coverage = '--cov' in sys.argv
+    if enable_coverage:
+        os.makedirs('.coverage', exist_ok=True)
+        sys.argv.remove('--cov')
+        import coverage
+        coverage.process_startup()
     returncode = 0
     # Iterate over list in reverse order to find the last --backend arg
     backend = Backend.ninja
@@ -164,10 +172,19 @@ if __name__ == '__main__':
     # Can't pass arguments to unit tests, so set the backend to use in the environment
     env = os.environ.copy()
     env['MESON_UNIT_TEST_BACKEND'] = backend.name
-    returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'] + units, env=env)
-    # Ubuntu packages do not have a binary without -6 suffix.
-    if should_run_linux_cross_tests():
-        print('Running cross compilation tests.\n')
-        returncode += subprocess.call([sys.executable, 'run_cross_test.py', 'cross/ubuntu-armhf.txt'])
-    returncode += subprocess.call([sys.executable, 'run_project_tests.py'] + sys.argv[1:])
+    with tempfile.TemporaryDirectory() as td:
+        # Enable coverage on all subsequent processes.
+        if enable_coverage:
+            with open(os.path.join(td, 'usercustomize.py'), 'w') as f:
+                f.write('import coverage\n'
+                        'coverage.process_startup()\n')
+            env['COVERAGE_PROCESS_START'] = '.coveragerc'
+            env['PYTHONPATH'] = os.pathsep.join([td] + env.get('PYTHONPATH', []))
+
+        returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'] + units, env=env)
+        # Ubuntu packages do not have a binary without -6 suffix.
+        if should_run_linux_cross_tests():
+            print('Running cross compilation tests.\n')
+            returncode += subprocess.call([sys.executable, 'run_cross_test.py', 'cross/ubuntu-armhf.txt'], env=env)
+        returncode += subprocess.call([sys.executable, 'run_project_tests.py'] + sys.argv[1:], env=env)
     sys.exit(returncode)