From a6e08e8fa7052c776989bf9410b87bf4d1f2ad49 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Mar 2022 19:00:31 -0500 Subject: [PATCH] use a more sane check instead of run_custom_lint Unfortunately, checking for strings without context is exceedingly prone to false positives, while missing anything that indirectly opens a file. Python 3.10 has a feature to warn about this though -- and it uses a runtime check which runs at the same time that the code fails to open files in the broken Windows locale. Set this up automatically when running the testsuite. Sadly, Python's builtin feature to change the warning level, e.g. by setting EncodingWarning to error at startup, is utterly broken if you want to limit it to only certain modules. This is tracked in order to be more efficiently ignored at https://bugs.python.org/issue34624 and https://github.com/python/cpython/pull/9358 It is also very trigger happy and passing stuff around via environment variable either messes with the testsuite, or with thirdparty programs which are implemented in python *such as lots of gnome*, or perhaps both. Instead, add runtime code to meson itself, to add a hidden "feature". In the application source code, running the 'warnings' module, you can actually get the expected behavior that $PYTHONWARNINGS doesn't have. So check for a magic testsuite variable every time meson starts up, and if it does, then go ahead and initialize a warnings filter that makes EncodingWarning fatal, but *only* when triggered via Meson and not arbitrary subprocess scripts. --- .github/workflows/lint.yml | 9 ----- mesonbuild/mesonmain.py | 5 +++ run_custom_lint.py | 76 -------------------------------------- run_mypy.py | 1 - run_tests.py | 8 ++++ 5 files changed, 13 insertions(+), 86 deletions(-) delete mode 100755 run_custom_lint.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ee4b1dbf3..4afbc848f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,15 +26,6 @@ jobs: - run: python -m pip install pylint - run: pylint mesonbuild - custom_lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - run: python ./run_custom_lint.py - mypy: runs-on: ubuntu-latest steps: diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index 7a4e42188..93cb8b0b7 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -220,6 +220,11 @@ def run(original_args, mainfile): print('Please update your environment') return 1 + if sys.version_info >= (3, 10) and os.environ.get('MESON_RUNNING_IN_PROJECT_TESTS'): + # workaround for https://bugs.python.org/issue34624 + import warnings + warnings.filterwarnings('error', category=EncodingWarning, module='mesonbuild') + # Meson gets confused if stdout can't output Unicode, if the # locale isn't Unicode, just force stdout to accept it. This tries # to emulate enough of PEP 540 to work elsewhere. diff --git a/run_custom_lint.py b/run_custom_lint.py deleted file mode 100755 index 89de9506e..000000000 --- a/run_custom_lint.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import Path -import typing as T - -root = Path(__file__).absolute().parent -mesonbuild = root / 'mesonbuild' - -whitelist = ['mesonbuild/', 'run_', 'ci/', 'tools/', 'docs/'] - -def check_missing_encoding(lines: T.List[str], path: str) -> int: - errors = 0 - functions = ['read_text', 'write_text', 'open'] - for num, line in enumerate(lines): - for func in functions: - l = line - - # Skip ignored lines - if '[ignore encoding]' in l: - continue - - # Do we have a match? - loc = l.find(func + '(') - if loc < 0: - continue - if loc > 0 and ord(l[loc-1].lower()) in [*range(ord('a'), ord('z')), *range(ord('0'), ord('9')), '_']: - continue - loc += len(func) + 1 - # Some preprocessign to make parsing easier - l = l[loc:] - l = l.replace(' ', '') - l = l.replace('\t', '') - l = l.replace('\n', '') - l = l.replace('\'', '"') - - # Parameter begin - args = '' - b_open = 1 - while l: - c = l[0] - l = l[1:] - if c == ')': - b_open -= 1 - if b_open == 0: - break - elif b_open == 1: - args += c - if c == '(': - b_open += 1 - - binary_modes = ['rb', 'br', 'r+b', 'wb', 'bw', 'ab', 'ba'] - is_binary = any([f'"{x}"' in args for x in binary_modes]) - if 'encoding=' not in args and not (func == 'open' and is_binary): - location = f'\x1b[33;1m[\x1b[0;1m{path}:{num+1}\x1b[33m]\x1b[0m' - #print(f'{location:<64}: \x1b[31;1mERROR:\x1b[0m Missing `encoding=` parameter in "{line.strip()}"') - print(f'{location:<72}: \x1b[31;1mERROR:\x1b[0m Missing `encoding=` parameter in `{func}` call') - errors += 1 - return errors - -def main() -> int: - print('Scanning mesonbuild...') - errors = 0 - for i in sorted(root.glob('**/*.py')): - raw = i.read_text(encoding='utf-8') - lines = raw.splitlines() - filename = i.relative_to(root).as_posix() - - if not any([filename.startswith(x) for x in whitelist]): - continue - - errors += check_missing_encoding(lines, filename) - print(f'Found {errors} errors while scanning mesonbuild') - return 0 if errors == 0 else 1 - -if __name__ == '__main__': - raise SystemExit(main()) diff --git a/run_mypy.py b/run_mypy.py index 2d7232bb2..d4fde4ff9 100755 --- a/run_mypy.py +++ b/run_mypy.py @@ -56,7 +56,6 @@ modules = [ 'mesonbuild/optinterpreter.py', 'mesonbuild/programs.py', - 'run_custom_lint.py', 'run_mypy.py', 'run_project_tests.py', 'run_single_test.py', diff --git a/run_tests.py b/run_tests.py index 6867ce62a..dc7a8f9ec 100755 --- a/run_tests.py +++ b/run_tests.py @@ -68,6 +68,14 @@ if NINJA_CMD is not None: else: raise RuntimeError('Could not find Ninja v1.7 or newer') +# Emulate running meson with -X utf8 by making sure all open() calls have a +# sane encoding. This should be a python default, but PEP 540 considered it not +# backwards compatible. Instead, much line noise in diffs to update this, and in +# python 3.10 we can also make it a warning when absent. +os.environ['PYTHONWARNDEFAULTENCODING'] = '1' +# work around https://bugs.python.org/issue34624 +os.environ['MESON_RUNNING_IN_PROJECT_TESTS'] = '1' + def guess_backend(backend_str: str, msbuild_exe: str) -> T.Tuple['Backend', T.List[str]]: # Auto-detect backend if unspecified backend_flags = []