coverage: Enable coverage reports

pull/8921/head
Daniel Mensinger 4 years ago committed by Jussi Pakkanen
parent edfe24178d
commit 765aff5a42
  1. 50
      .coveragerc
  2. 7
      .github/workflows/cygwin.yml
  3. 13
      .github/workflows/macos.yml
  4. 7
      .github/workflows/msys2.yml
  5. 5
      .github/workflows/nonative.yml
  6. 9
      .github/workflows/os_comp.yml
  7. 12
      .github/workflows/unusedargs_missingreturn.yml
  8. 3
      .gitignore
  9. 1
      azure-pipelines.yml
  10. 19
      ci/run.ps1
  11. 13
      ci/upload_cov.sh
  12. 19
      ci/usercustomize.py
  13. 25
      data/.coveragerc.in
  14. 78
      run_tests.py
  15. 7
      run_unittests.py
  16. 53
      tools/run_with_cov.py

@ -1,50 +0,0 @@
[run]
branch = True
concurrency =
multiprocessing
data_file = .coverage/coverage
parallel = True
source =
meson.py
mesonbuild/
mesonconf.py
mesonintrospect.py
mesonrewriter.py
mesontest.py
run_cross_test.py
run_project_tests.py
run_tests.py
run_unittests.py
# Aliases to /root/ are needed because Travis runs in docker at /root/.
[paths]
meson =
meson.py
/root/meson.py
mesonbuild =
mesonbuild/
/root/mesonbuild/
mesonconf =
mesonconf.py
/root/mesonconf.py
mesonintrospect =
mesonintrospect.py
/root/mesonintrospect.py
mesonrewriter =
mesonrewriter.py
/root/mesonrewriter.py
mesontest =
mesontest.py
/root/mesontest.py
run_cross_test =
run_cross_test.py
/root/run_cross_test.py
run_project_tests =
run_project_tests.py
/root/run_project_tests.py
run_tests =
run_tests.py
/root/run_tests.py
run_unittests =
run_unittests.py
/root/run_unittests.py

@ -68,13 +68,13 @@ jobs:
- name: Run pip - name: Run pip
run: | run: |
export PATH=/usr/bin:/usr/local/bin:$(cygpath ${SYSTEMROOT})/system32 export PATH=/usr/bin:/usr/local/bin:$(cygpath ${SYSTEMROOT})/system32
python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist coverage codecov
shell: C:\cygwin\bin\bash.exe --noprofile --norc -o igncr -eo pipefail '{0}' shell: C:\cygwin\bin\bash.exe --noprofile --norc -o igncr -eo pipefail '{0}'
- name: Run tests - name: Run tests
run: | run: |
export PATH=/usr/bin:/usr/local/bin:$(cygpath ${SYSTEMROOT})/system32 export PATH=/usr/bin:/usr/local/bin:$(cygpath ${SYSTEMROOT})/system32
python3 run_tests.py --backend=ninja python3 ./tools/run_with_cov.py run_tests.py --backend=ninja
env: env:
# Cygwin's static boost installation is broken (some static library # Cygwin's static boost installation is broken (some static library
# variants such as boost_thread are not present) # variants such as boost_thread are not present)
@ -87,3 +87,6 @@ jobs:
path: meson-test-run.* path: meson-test-run.*
# test log should be saved on failure # test log should be saved on failure
if: ${{ !cancelled() }} if: ${{ !cancelled() }}
- name: Upload coverage report
run: ./ci/upload_cov.sh "${{ matrix.NAME }}"

@ -24,7 +24,7 @@ jobs:
python-version: '3.x' python-version: '3.x'
- run: | - run: |
python -m pip install --upgrade pip python -m pip install --upgrade pip
python -m pip install pytest pytest-xdist jsonschema python -m pip install pytest pytest-xdist jsonschema coverage codecov
- run: brew install pkg-config ninja llvm qt@5 - run: brew install pkg-config ninja llvm qt@5
- env: - env:
CPPFLAGS: "-I/usr/local/include" CPPFLAGS: "-I/usr/local/include"
@ -35,7 +35,10 @@ jobs:
export SDKROOT="$(xcodebuild -version -sdk macosx Path)" export SDKROOT="$(xcodebuild -version -sdk macosx Path)"
export PATH="$HOME/tools:/usr/local/opt/qt@5/bin:$PATH:$(brew --prefix llvm)/bin" export PATH="$HOME/tools:/usr/local/opt/qt@5/bin:$PATH:$(brew --prefix llvm)/bin"
export PKG_CONFIG_PATH="/usr/local/opt/qt@5/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH="/usr/local/opt/qt@5/lib/pkgconfig:$PKG_CONFIG_PATH"
./run_unittests.py ./tools/run_with_cov.py ./run_unittests.py
- name: Upload coverage report
run: ./ci/upload_cov.sh "appleclang [unit tests]"
project-tests-appleclang: project-tests-appleclang:
runs-on: macos-latest runs-on: macos-latest
@ -50,7 +53,7 @@ jobs:
- run: | - run: |
python3 -m pip install --upgrade setuptools python3 -m pip install --upgrade setuptools
python3 -m pip install --upgrade pip python3 -m pip install --upgrade pip
python3 -m pip install cython python3 -m pip install cython coverage codecov
- env: - env:
CPPFLAGS: "-I/usr/local/include" CPPFLAGS: "-I/usr/local/include"
LDFLAGS: "-L/usr/local/lib" LDFLAGS: "-L/usr/local/lib"
@ -61,7 +64,9 @@ jobs:
export SDKROOT="$(xcodebuild -version -sdk macosx Path)" export SDKROOT="$(xcodebuild -version -sdk macosx Path)"
export PATH="$HOME/tools:/usr/local/opt/qt@5/bin:$PATH:$(brew --prefix llvm)/bin" export PATH="$HOME/tools:/usr/local/opt/qt@5/bin:$PATH:$(brew --prefix llvm)/bin"
export PKG_CONFIG_PATH="/usr/local/opt/qt@5/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH="/usr/local/opt/qt@5/lib/pkgconfig:$PKG_CONFIG_PATH"
./run_project_tests.py --backend=ninja ./tools/run_with_cov.py ./run_project_tests.py --backend=ninja
- name: Upload coverage report
run: ./ci/upload_cov.sh "appleclang [project tests; unity=${{ matrix.unity }}]"
Qt4macos: Qt4macos:
runs-on: macos-latest runs-on: macos-latest

@ -71,7 +71,7 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist python3 -m pip --disable-pip-version-check install gcovr jsonschema pefile pytest pytest-xdist coverage codecov
- name: Run Tests - name: Run Tests
run: | run: |
@ -89,9 +89,12 @@ jobs:
pacman --noconfirm --needed -S mingw-w64-${{ matrix.MSYS2_ARCH }}-${{ matrix.MSYS2_CURSES }} pacman --noconfirm --needed -S mingw-w64-${{ matrix.MSYS2_ARCH }}-${{ matrix.MSYS2_CURSES }}
fi fi
MSYSTEM= python3 run_tests.py --backend=ninja MSYSTEM= python3 ./tools/run_with_cov.py run_tests.py --backend=ninja
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: ${{ matrix.NAME }} name: ${{ matrix.NAME }}
path: meson-test-run.* path: meson-test-run.*
- name: Upload coverage report
run: ./ci/upload_cov.sh "${{ matrix.NAME }}"

@ -26,6 +26,9 @@ jobs:
- run: | - run: |
apt-get -y purge clang gcc gdc apt-get -y purge clang gcc gdc
apt-get -y autoremove apt-get -y autoremove
python3 -m pip install coverage codecov
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: Run tests - name: Run tests
run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS --cross ubuntu-armhf.json --cross-only' run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./tools/run_with_cov.py ./run_tests.py $CI_ARGS --cross ubuntu-armhf.json --cross-only'
- name: Upload coverage report
run: ./ci/upload_cov.sh "Ubuntu nonnative"

@ -40,7 +40,9 @@ jobs:
# All environment variables are stored inside the docker image in /ci/env_vars.sh # All environment variables are stored inside the docker image in /ci/env_vars.sh
# They are defined in the `env` section in each image.json. CI_ARGS should be set # They are defined in the `env` section in each image.json. CI_ARGS should be set
# via the `args` array ub the image.json # via the `args` array ub the image.json
run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS' run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./tools/run_with_cov.py ./run_tests.py $CI_ARGS'
- name: Upload coverage report
run: ./ci/upload_cov.sh "OS Comp [${{ matrix.cfg.name }}]"
ubuntu-rolling: ubuntu-rolling:
name: 'Ubuntu Rolling' name: 'Ubuntu Rolling'
@ -106,4 +108,7 @@ jobs:
update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix
update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix
./run_tests.py $RUN_TESTS_ARGS -- $MESON_ARGS ./tools/run_with_cov.py ./run_tests.py $RUN_TESTS_ARGS -- $MESON_ARGS
- name: Upload coverage report
run: ./ci/upload_cov.sh "Ubuntu [${{ matrix.cfg.CC }} ${{ matrix.cfg.RUN_TESTS_ARGS }} ${{ matrix.cfg.MESON_ARGS }}]"

@ -45,7 +45,10 @@ jobs:
run: | run: |
sudo apt update -yq sudo apt update -yq
sudo apt install -yq --no-install-recommends g++ gfortran ninja-build gobjc gobjc++ sudo apt install -yq --no-install-recommends g++ gfortran ninja-build gobjc gobjc++
- run: python run_project_tests.py --only cmake common fortran platform-linux "objective c" "objective c++" python -m pip install coverage codecov
- run: ./tools/run_with_cov.py run_project_tests.py --only cmake common fortran platform-linux "objective c" "objective c++"
- name: Upload coverage report
run: ./ci/upload_cov.sh "UnusedMissingReturn"
windows: windows:
runs-on: windows-latest runs-on: windows-latest
@ -55,10 +58,13 @@ jobs:
with: with:
python-version: '3.x' python-version: '3.x'
- run: pip install ninja pefile - run: pip install ninja pefile coverage codecov
- run: python run_project_tests.py --only platform-windows - run: python ./tools/run_with_cov.py run_project_tests.py --only platform-windows
env: env:
CC: gcc CC: gcc
CXX: g++ CXX: g++
FC: gfortran FC: gfortran
- name: Upload coverage report
run: ./ci/upload_cov.sh "UnusedMissingReturn Windows"

3
.gitignore vendored

@ -8,7 +8,8 @@
/.vscode /.vscode
__pycache__ __pycache__
.coverage /.coverage/
/.coveragerc
/install dir /install dir
/work area /work area

@ -9,6 +9,7 @@ trigger:
variables: variables:
CI: 1 CI: 1
SOURCE_VERSION: $(Build.SourceVersion)
jobs: jobs:

@ -68,11 +68,26 @@ python --version
# Needed for running unit tests in parallel. # Needed for running unit tests in parallel.
echo "" echo ""
python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist jsonschema python -m pip --disable-pip-version-check install --upgrade pefile pytest-xdist jsonschema coverage
echo "" echo ""
echo "=== Start running tests ===" echo "=== Start running tests ==="
# Starting from VS2019 Powershell(?) will fail the test run # Starting from VS2019 Powershell(?) will fail the test run
# if it prints anything to stderr. Python's test runner # if it prints anything to stderr. Python's test runner
# does that by default so we need to forward it. # does that by default so we need to forward it.
cmd /c "python 2>&1 run_tests.py --backend $env:backend $env:extraargs" cmd /c "python 2>&1 ./tools/run_with_cov.py run_tests.py --backend $env:backend $env:extraargs"
echo ""
echo ""
echo "=== Gathering coverage report ==="
echo ""
python3 -m coverage combine
python3 -m coverage xml
python3 -m coverage report
# Currently codecov.py does not handle Azure, use this fork of a fork to get it
# working without requireing a token
git clone https://github.com/mensinda/codecov-python
python3 -m pip install --ignore-installed ./codecov-python
python3 -m codecov -f .coverage/coverage.xml -n "VS$env:compiler $env:arch $env:backend" -c $env:SOURCE_VERSION

@ -0,0 +1,13 @@
#!/bin/bash
echo "Combining coverage reports..."
coverage combine
echo "Generating XML report..."
coverage xml
echo "Printing report"
coverage report
echo "Uploading to codecov..."
codecov -f .coverage/coverage.xml -n "$1"

@ -0,0 +1,19 @@
# Copyright 2021 The Meson development team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is used by coverage (see tools/run_with_cov.py) to enable coverage
# reports in python subprocesses
import coverage
coverage.process_startup()

@ -0,0 +1,25 @@
[run]
branch = True
parallel = True
concurrency = multiprocessing
data_file = @ROOT@/.coverage/coverage
source = @ROOT@/mesonbuild/
[report]
exclude_lines =
if T.TYPE_CHECKING:
[paths]
mesonbuild =
mesonbuild/
__w/meson/meson/mesonbuild/
@ROOT@/mesonbuild/
[html]
directory = @ROOT@/.coverage/html
[xml]
output = @ROOT@/.coverage/coverage.xml
[json]
output = @ROOT@/.coverage/coverage.json

@ -20,7 +20,6 @@ import sys
import time import time
import shutil import shutil
import subprocess import subprocess
import tempfile
import platform import platform
import argparse import argparse
import traceback import traceback
@ -311,7 +310,6 @@ def print_system_info():
def main(): def main():
print_system_info() print_system_info()
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--cov', action='store_true')
parser.add_argument('--backend', default=None, dest='backend', parser.add_argument('--backend', default=None, dest='backend',
choices=backendlist) choices=backendlist)
parser.add_argument('--cross', default=[], dest='cross', action='append') parser.add_argument('--cross', default=[], dest='cross', action='append')
@ -319,13 +317,6 @@ def main():
parser.add_argument('--failfast', action='store_true') parser.add_argument('--failfast', action='store_true')
parser.add_argument('--no-unittests', action='store_true', default=False) parser.add_argument('--no-unittests', action='store_true', default=False)
(options, _) = parser.parse_known_args() (options, _) = parser.parse_known_args()
# Enable coverage early...
enable_coverage = options.cov
if enable_coverage:
os.makedirs('.coverage', exist_ok=True)
sys.argv.remove('--cov')
import coverage
coverage.process_startup()
returncode = 0 returncode = 0
backend, _ = guess_backend(options.backend, shutil.which('msbuild')) backend, _ = guess_backend(options.backend, shutil.which('msbuild'))
no_unittests = options.no_unittests no_unittests = options.no_unittests
@ -349,52 +340,41 @@ def main():
# Run tests # Run tests
# Can't pass arguments to unit tests, so set the backend to use in the environment # Can't pass arguments to unit tests, so set the backend to use in the environment
env = os.environ.copy() env = os.environ.copy()
with tempfile.TemporaryDirectory() as temp_dir: if not options.cross:
# Enable coverage on all subsequent processes. cmd = mesonlib.python_command + ['run_meson_command_tests.py', '-v']
if enable_coverage: if options.failfast:
Path(temp_dir, 'usercustomize.py').open('w').write( cmd += ['--failfast']
'import coverage\n' returncode += subprocess.call(cmd, env=env)
'coverage.process_startup()\n') if options.failfast and returncode != 0:
env['COVERAGE_PROCESS_START'] = '.coveragerc' return returncode
if 'PYTHONPATH' in env: if no_unittests:
env['PYTHONPATH'] = os.pathsep.join([temp_dir, env.get('PYTHONPATH')]) print('Skipping all unit tests.')
else: print(flush=True)
env['PYTHONPATH'] = temp_dir returncode = 0
if not options.cross: else:
cmd = mesonlib.python_command + ['run_meson_command_tests.py', '-v'] print(mlog.bold('Running unittests.'))
print(flush=True)
cmd = mesonlib.python_command + ['run_unittests.py', '--backend=' + backend.name, '-v']
if options.failfast: if options.failfast:
cmd += ['--failfast'] cmd += ['--failfast']
returncode += subprocess.call(cmd, env=env) returncode += subprocess.call(cmd, env=env)
if options.failfast and returncode != 0: if options.failfast and returncode != 0:
return returncode return returncode
if no_unittests: cmd = mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:]
print('Skipping all unit tests.') returncode += subprocess.call(cmd, env=env)
print(flush=True) else:
returncode = 0 cross_test_args = mesonlib.python_command + ['run_cross_test.py']
else: for cf in options.cross:
print(mlog.bold('Running unittests.')) print(mlog.bold(f'Running {cf} cross tests.'))
print(flush=True) print(flush=True)
cmd = mesonlib.python_command + ['run_unittests.py', '--backend=' + backend.name, '-v'] cmd = cross_test_args + ['cross/' + cf]
if options.failfast: if options.failfast:
cmd += ['--failfast'] cmd += ['--failfast']
returncode += subprocess.call(cmd, env=env) if options.cross_only:
if options.failfast and returncode != 0: cmd += ['--cross-only']
return returncode
cmd = mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:]
returncode += subprocess.call(cmd, env=env) returncode += subprocess.call(cmd, env=env)
else: if options.failfast and returncode != 0:
cross_test_args = mesonlib.python_command + ['run_cross_test.py'] return returncode
for cf in options.cross:
print(mlog.bold(f'Running {cf} cross tests.'))
print(flush=True)
cmd = cross_test_args + ['cross/' + cf]
if options.failfast:
cmd += ['--failfast']
if options.cross_only:
cmd += ['--cross-only']
returncode += subprocess.call(cmd, env=env)
if options.failfast and returncode != 0:
return returncode
return returncode return returncode
if __name__ == '__main__': if __name__ == '__main__':

@ -10302,6 +10302,13 @@ def main():
pytest_args += ['--color=yes'] pytest_args += ['--color=yes']
pytest_args += ['./run_unittests.py'] pytest_args += ['./run_unittests.py']
pytest_args += convert_args(sys.argv[1:]) pytest_args += convert_args(sys.argv[1:])
# Always disable pytest-cov because we use a custom setup
try:
import pytest_cov # noqa: F401
print('Disabling pytest-cov')
pytest_args += ['-p' 'no:cov']
except ImportError:
pass
return subprocess.run(python_command + ['-m', 'pytest'] + pytest_args).returncode return subprocess.run(python_command + ['-m', 'pytest'] + pytest_args).returncode
except ImportError: except ImportError:
print('pytest-xdist not found, using unittest instead') print('pytest-xdist not found, using unittest instead')

@ -0,0 +1,53 @@
#!/usr/bin/env python3
# Copyright 2021 The Meson development team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess
import coverage
import os
import sys
from pathlib import Path
root_path = Path(__file__).parent.parent.absolute()
# Python magic so we can import mesonlib
sys.path.append(root_path.as_posix())
from mesonbuild import mesonlib
def generate_coveragerc() -> Path:
i_file = (root_path / 'data' / '.coveragerc.in')
o_file = (root_path / '.coveragerc')
raw = i_file.read_text()
raw = raw.replace('@ROOT@', root_path.as_posix())
o_file.write_text(raw)
return o_file
def main() -> int:
# Remove old run data
out_dir = root_path / '.coverage'
mesonlib.windows_proof_rmtree(out_dir.as_posix())
out_dir.mkdir(parents=True, exist_ok=True)
# Setup coverage
python_path = (root_path / 'ci').as_posix()
os.environ['PYTHONPATH'] = os.pathsep.join([python_path, os.environ.get('PYTHONPATH', '')])
os.environ['COVERAGE_PROCESS_START'] = generate_coveragerc().as_posix()
coverage.process_startup()
# Run the actual command
cmd = mesonlib.python_command + sys.argv[1:]
return subprocess.run(cmd, env=os.environ.copy()).returncode
if __name__ == '__main__':
raise SystemExit(main())
Loading…
Cancel
Save