Move classes used by scripts to their own module

Those classes are used by wrapper scripts and we should not have to
import the rest of mesonlib, build.py, and all their dependencies for
that.

This renames mesonlib/ directory to utils/ and add a mesonlib.py module
that imports everything from utils/ to not have to change `import
mesonlib` everywhere. It allows to import utils.core without importing
the rest of mesonlib.
pull/10869/head
Xavier Claessens 2 years ago committed by Eli Schwartz
parent a58ec322b3
commit 2dfd952eb9
  1. 2
      .flake8
  2. 2
      meson.py
  3. 24
      mesonbuild/backend/backends.py
  4. 67
      mesonbuild/build.py
  5. 3
      mesonbuild/dependencies/dub.py
  6. 6
      mesonbuild/interpreter/type_checking.py
  7. 13
      mesonbuild/mesonlib.py
  8. 20
      mesonbuild/mesonmain.py
  9. 6
      mesonbuild/scripts/meson_exe.py
  10. 11
      mesonbuild/scripts/test_loaded_modules.py
  11. 0
      mesonbuild/utils/__init__.py
  12. 160
      mesonbuild/utils/core.py
  13. 0
      mesonbuild/utils/platform.py
  14. 0
      mesonbuild/utils/posix.py
  15. 36
      mesonbuild/utils/universal.py
  16. 5
      mesonbuild/utils/vsenv.py
  17. 0
      mesonbuild/utils/win32.py
  18. 8
      run_mypy.py
  19. 32
      unittests/allplatformstests.py
  20. 2
      unittests/internaltests.py

@ -29,5 +29,5 @@ extend-ignore =
# A003: builtin class attribute
A003
per-file-ignores =
mesonbuild/mesonlib/__init__.py:F401,F403
mesonbuild/mesonlib.py:F401,F403
max-line-length = 120

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is an entry point for all commands, including scripts. Include the
# strict minimum python modules for performance reasons.
import sys
# Check python version before importing anything else, we might have an older

@ -34,7 +34,8 @@ from .. import mlog
from ..compilers import LANGUAGES_USING_LDFLAGS, detect
from ..mesonlib import (
File, MachineChoice, MesonException, OrderedSet,
classify_unity_sources, OptionKey, join_args
classify_unity_sources, OptionKey, join_args,
ExecutableSerialisation
)
if T.TYPE_CHECKING:
@ -185,27 +186,6 @@ class SubdirInstallData(InstallDataBase):
super().__init__(path, install_path, install_path_name, install_mode, subproject, tag, data_type)
self.exclude = exclude
@dataclass(eq=False)
class ExecutableSerialisation:
# XXX: should capture and feed default to False, instead of None?
cmd_args: T.List[str]
env: T.Optional[build.EnvironmentVariables] = None
exe_wrapper: T.Optional['programs.ExternalProgram'] = None
workdir: T.Optional[str] = None
extra_paths: T.Optional[T.List] = None
capture: T.Optional[bool] = None
feed: T.Optional[bool] = None
tag: T.Optional[str] = None
verbose: bool = False
def __post_init__(self) -> None:
if self.exe_wrapper is not None:
assert isinstance(self.exe_wrapper, programs.ExternalProgram)
self.pickled = False
self.skip_if_destdir = False
self.subproject = ''
@dataclass(eq=False)
class TestSerialisation:

@ -36,7 +36,7 @@ from .mesonlib import (
extract_as_list, typeslistify, stringlistify, classify_unity_sources,
get_filenames_templates_dict, substitute_values, has_path_sep,
OptionKey, PerMachineDefaultable, OptionOverrideProxy,
MesonBugException
MesonBugException, EnvironmentVariables
)
from .compilers import (
is_object, clink_langs, sort_clink, all_languages,
@ -502,71 +502,6 @@ class StructuredSources(HoldableObject):
return False
EnvInitValueType = T.Dict[str, T.Union[str, T.List[str]]]
class EnvironmentVariables(HoldableObject):
def __init__(self, values: T.Optional[EnvInitValueType] = None,
init_method: Literal['set', 'prepend', 'append'] = 'set', separator: str = os.pathsep) -> None:
self.envvars: T.List[T.Tuple[T.Callable[[T.Dict[str, str], str, T.List[str], str], str], str, T.List[str], str]] = []
# The set of all env vars we have operations for. Only used for self.has_name()
self.varnames: T.Set[str] = set()
if values:
init_func = getattr(self, init_method)
for name, value in values.items():
init_func(name, listify(value), separator)
def __repr__(self) -> str:
repr_str = "<{0}: {1}>"
return repr_str.format(self.__class__.__name__, self.envvars)
def hash(self, hasher: T.Any):
myenv = self.get_env({})
for key in sorted(myenv.keys()):
hasher.update(bytes(key, encoding='utf-8'))
hasher.update(b',')
hasher.update(bytes(myenv[key], encoding='utf-8'))
hasher.update(b';')
def has_name(self, name: str) -> bool:
return name in self.varnames
def get_names(self) -> T.Set[str]:
return self.varnames
def set(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
self.varnames.add(name)
self.envvars.append((self._set, name, values, separator))
def append(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
self.varnames.add(name)
self.envvars.append((self._append, name, values, separator))
def prepend(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
self.varnames.add(name)
self.envvars.append((self._prepend, name, values, separator))
@staticmethod
def _set(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
return separator.join(values)
@staticmethod
def _append(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
curr = env.get(name)
return separator.join(values if curr is None else [curr] + values)
@staticmethod
def _prepend(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
curr = env.get(name)
return separator.join(values if curr is None else values + [curr])
def get_env(self, full_env: T.MutableMapping[str, str]) -> T.Dict[str, str]:
env = full_env.copy()
for method, name, values, separator in self.envvars:
env[name] = method(env, name, values, separator)
return env
@dataclass(eq=False)
class Target(HoldableObject):

@ -14,8 +14,7 @@
from .base import ExternalDependency, DependencyException, DependencyTypeName
from .pkgconfig import PkgConfigDependency
from ..mesonlib import (Popen_safe, OptionKey)
from ..mesonlib.universal import join_args
from ..mesonlib import (Popen_safe, OptionKey, join_args)
from ..programs import ExternalProgram
from .. import mlog
import re

@ -8,13 +8,15 @@ import os
import typing as T
from .. import compilers
from ..build import (EnvironmentVariables, EnvInitValueType, CustomTarget, BuildTarget,
from ..build import (CustomTarget, BuildTarget,
CustomTargetIndex, ExtractedObjects, GeneratedList, IncludeDirs,
BothLibraries, SharedLibrary, StaticLibrary, Jar, Executable)
from ..coredata import UserFeatureOption
from ..dependencies import Dependency, InternalDependency
from ..interpreterbase.decorators import KwargInfo, ContainerTypeInfo
from ..mesonlib import File, FileMode, MachineChoice, listify, has_path_sep, OptionKey
from ..mesonlib import (
File, FileMode, MachineChoice, listify, has_path_sep, OptionKey,
EnvInitValueType, EnvironmentVariables)
from ..programs import ExternalProgram
# Helper definition for type checks that are `Optional[T]`

@ -14,19 +14,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: skip-file
"""Helper functions and classes."""
import os
from .universal import *
from .vsenv import setup_vsenv
from .utils.core import *
from .utils.vsenv import *
from .utils.universal import *
# Here we import either the posix implementations, the windows implementations,
# or a generic no-op implementation
if os.name == 'posix':
from .posix import *
from .utils.posix import *
elif os.name == 'nt':
from .win32 import *
from .utils.win32 import *
else:
from .platform import *
from .utils.platform import *

@ -18,18 +18,18 @@ from . import _pathlib
import sys
sys.modules['pathlib'] = _pathlib
# This file is an entry point for all commands, including scripts. Include the
# strict minimum python modules for performance reasons.
import os.path
import platform
import importlib
import traceback
import argparse
import shutil
from . import mesonlib
from .utils.core import MesonException, MesonBugException
from . import mlog
from .mesonlib import MesonException, MesonBugException
def errorhandler(e, command):
import traceback
if isinstance(e, MesonException):
mlog.exception(e)
logfile = mlog.shutdown()
@ -72,6 +72,7 @@ class CommandLineParser:
from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects, munstable_coredata, mcompile, mdevenv
from .scripts import env2mfile
from .wrap import wraptool
import shutil
self.term_width = shutil.get_terminal_size().columns
self.formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=int(self.term_width / 2), width=self.term_width)
@ -176,6 +177,7 @@ class CommandLineParser:
parser = self.parser
command = None
from . import mesonlib
args = mesonlib.expand_arguments(args)
options = parser.parse_args(args)
@ -228,6 +230,11 @@ def ensure_stdout_accepts_unicode():
if sys.stdout.encoding and not sys.stdout.encoding.upper().startswith('UTF-'):
sys.stdout.reconfigure(errors='surrogateescape')
def set_meson_command(mainfile):
# Set the meson command that will be used to run scripts and so on
from . import mesonlib
mesonlib.set_meson_command(mainfile)
def run(original_args, mainfile):
if sys.version_info >= (3, 10) and os.environ.get('MESON_RUNNING_IN_PROJECT_TESTS'):
# workaround for https://bugs.python.org/issue34624
@ -245,15 +252,13 @@ def run(original_args, mainfile):
mlog.error('Please install and use mingw-w64-x86_64-python3 and/or mingw-w64-x86_64-meson with Pacman')
return 2
# Set the meson command that will be used to run scripts and so on
mesonlib.set_meson_command(mainfile)
args = original_args[:]
# Special handling of internal commands called from backends, they don't
# need to go through argparse.
if len(args) >= 2 and args[0] == '--internal':
if args[1] == 'regenerate':
set_meson_command(mainfile)
from . import msetup
try:
return msetup.run(['--reconfigure'] + args[2:])
@ -262,6 +267,7 @@ def run(original_args, mainfile):
else:
return run_script_command(args[1], args[2:])
set_meson_command(mainfile)
return CommandLineParser().run(args)
def main():

@ -20,8 +20,7 @@ import subprocess
import typing as T
import locale
from .. import mesonlib
from ..backend.backends import ExecutableSerialisation
from ..utils.core import ExecutableSerialisation
def buildparser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?')
@ -46,7 +45,8 @@ def run_exe(exe: ExecutableSerialisation, extra_env: T.Optional[T.Dict[str, str]
if exe.extra_paths:
child_env['PATH'] = (os.pathsep.join(exe.extra_paths + ['']) +
child_env['PATH'])
if exe.exe_wrapper and mesonlib.substring_is_in_list('wine', exe.exe_wrapper.get_command()):
if exe.exe_wrapper and any('wine' in i for i in exe.exe_wrapper.get_command()):
from .. import mesonlib
child_env['WINEPATH'] = mesonlib.get_wine_shortpath(
exe.exe_wrapper.get_command(),
['Z:' + p for p in exe.extra_paths] + child_env.get('WINEPATH', '').split(';'),

@ -0,0 +1,11 @@
import sys
import json
import typing as T
from . import meson_exe
# This script is used by run_unittests.py to verify we don't load too many
# modules when executing a wrapped command.
def run(args: T.List[str]) -> int:
meson_exe.run(args)
print(json.dumps(list(sys.modules.keys())))
return 0

@ -0,0 +1,160 @@
# Copyright 2012-2022 The Meson development team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Contains the strict minimum to run scripts.
When the backend needs to call back into Meson during compilation for running
scripts or wrapping commands, it is important to load as little python modules
as possible for performance reasons.
"""
from __future__ import annotations
from dataclasses import dataclass
import os
import abc
import typing as T
if T.TYPE_CHECKING:
from typing_extensions import Literal
from ..mparser import BaseNode
from . import programs
__all__ = [
'MesonException',
'MesonBugException',
'HoldableObject',
'EnvInitValueType',
'EnvironmentVariables',
'ExecutableSerialisation',
]
class MesonException(Exception):
'''Exceptions thrown by Meson'''
def __init__(self, *args: object, file: T.Optional[str] = None,
lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
super().__init__(*args)
self.file = file
self.lineno = lineno
self.colno = colno
@classmethod
def from_node(cls, *args: object, node: BaseNode) -> MesonException:
"""Create a MesonException with location data from a BaseNode
:param node: A BaseNode to set location data from
:return: A Meson Exception instance
"""
return cls(*args, file=node.filename, lineno=node.lineno, colno=node.colno)
class MesonBugException(MesonException):
'''Exceptions thrown when there is a clear Meson bug that should be reported'''
def __init__(self, msg: str, file: T.Optional[str] = None,
lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
super().__init__(msg + '\n\n This is a Meson bug and should be reported!',
file=file, lineno=lineno, colno=colno)
class HoldableObject(metaclass=abc.ABCMeta):
''' Dummy base class for all objects that can be
held by an interpreter.baseobjects.ObjectHolder '''
EnvInitValueType = T.Dict[str, T.Union[str, T.List[str]]]
class EnvironmentVariables(HoldableObject):
def __init__(self, values: T.Optional[EnvInitValueType] = None,
init_method: Literal['set', 'prepend', 'append'] = 'set', separator: str = os.pathsep) -> None:
self.envvars: T.List[T.Tuple[T.Callable[[T.Dict[str, str], str, T.List[str], str], str], str, T.List[str], str]] = []
# The set of all env vars we have operations for. Only used for self.has_name()
self.varnames: T.Set[str] = set()
if values:
init_func = getattr(self, init_method)
for name, value in values.items():
v = value if isinstance(value, list) else [value]
init_func(name, v, separator)
def __repr__(self) -> str:
repr_str = "<{0}: {1}>"
return repr_str.format(self.__class__.__name__, self.envvars)
def hash(self, hasher: T.Any):
myenv = self.get_env({})
for key in sorted(myenv.keys()):
hasher.update(bytes(key, encoding='utf-8'))
hasher.update(b',')
hasher.update(bytes(myenv[key], encoding='utf-8'))
hasher.update(b';')
def has_name(self, name: str) -> bool:
return name in self.varnames
def get_names(self) -> T.Set[str]:
return self.varnames
def set(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
self.varnames.add(name)
self.envvars.append((self._set, name, values, separator))
def append(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
self.varnames.add(name)
self.envvars.append((self._append, name, values, separator))
def prepend(self, name: str, values: T.List[str], separator: str = os.pathsep) -> None:
self.varnames.add(name)
self.envvars.append((self._prepend, name, values, separator))
@staticmethod
def _set(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
return separator.join(values)
@staticmethod
def _append(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
curr = env.get(name)
return separator.join(values if curr is None else [curr] + values)
@staticmethod
def _prepend(env: T.Dict[str, str], name: str, values: T.List[str], separator: str) -> str:
curr = env.get(name)
return separator.join(values if curr is None else values + [curr])
def get_env(self, full_env: T.MutableMapping[str, str]) -> T.Dict[str, str]:
env = full_env.copy()
for method, name, values, separator in self.envvars:
env[name] = method(env, name, values, separator)
return env
@dataclass(eq=False)
class ExecutableSerialisation:
# XXX: should capture and feed default to False, instead of None?
cmd_args: T.List[str]
env: T.Optional[EnvironmentVariables] = None
exe_wrapper: T.Optional['programs.ExternalProgram'] = None
workdir: T.Optional[str] = None
extra_paths: T.Optional[T.List] = None
capture: T.Optional[bool] = None
feed: T.Optional[bool] = None
tag: T.Optional[str] = None
verbose: bool = False
def __post_init__(self) -> None:
self.pickled = False
self.skip_if_destdir = False
self.subproject = ''

@ -33,6 +33,7 @@ import copy
import pickle
from mesonbuild import mlog
from .core import MesonException, HoldableObject
if T.TYPE_CHECKING:
from typing_extensions import Literal
@ -41,7 +42,6 @@ if T.TYPE_CHECKING:
from ..build import ConfigurationData
from ..coredata import KeyedOptionDictType, UserOption
from ..compilers.compilers import Compiler
from ..mparser import BaseNode
FileOrString = T.Union['File', str]
@ -52,15 +52,12 @@ __all__ = [
'GIT',
'python_command',
'project_meson_versions',
'HoldableObject',
'SecondLevelHolder',
'File',
'FileMode',
'GitException',
'LibType',
'MachineChoice',
'MesonException',
'MesonBugException',
'EnvironmentException',
'FileOrString',
'GitException',
@ -164,33 +161,6 @@ else:
python_command = [sys.executable]
_meson_command: T.Optional['ImmutableListProtocol[str]'] = None
class MesonException(Exception):
'''Exceptions thrown by Meson'''
def __init__(self, *args: object, file: T.Optional[str] = None,
lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
super().__init__(*args)
self.file = file
self.lineno = lineno
self.colno = colno
@classmethod
def from_node(cls, *args: object, node: BaseNode) -> MesonException:
"""Create a MesonException with location data from a BaseNode
:param node: A BaseNode to set location data from
:return: A Meson Exception instance
"""
return cls(*args, file=node.filename, lineno=node.lineno, colno=node.colno)
class MesonBugException(MesonException):
'''Exceptions thrown when there is a clear Meson bug that should be reported'''
def __init__(self, msg: str, file: T.Optional[str] = None,
lineno: T.Optional[int] = None, colno: T.Optional[int] = None):
super().__init__(msg + '\n\n This is a Meson bug and should be reported!',
file=file, lineno=lineno, colno=colno)
class EnvironmentException(MesonException):
'''Exceptions thrown while processing and creating the build environment'''
@ -279,10 +249,6 @@ def check_direntry_issues(direntry_array: T.Union[T.Iterable[T.Union[str, bytes]
not pure ASCII. This may cause problems.
'''), file=sys.stderr)
class HoldableObject(metaclass=abc.ABCMeta):
''' Dummy base class for all objects that can be
held by an interpreter.baseobjects.ObjectHolder '''
class SecondLevelHolder(HoldableObject, metaclass=abc.ABCMeta):
''' A second level object holder. The primary purpose
of such objects is to hold multiple objects with one

@ -10,6 +10,11 @@ from .. import mlog
from .universal import MesonException, is_windows
__all__ = [
'setup_vsenv',
]
bat_template = '''@ECHO OFF
call "{}"

@ -33,8 +33,8 @@ modules = [
'mesonbuild/interpreter/type_checking.py',
'mesonbuild/mcompile.py',
'mesonbuild/mdevenv.py',
'mesonbuild/mesonlib/platform.py',
'mesonbuild/mesonlib/universal.py',
'mesonbuild/utils/platform.py',
'mesonbuild/utils/universal.py',
'mesonbuild/minit.py',
'mesonbuild/minstall.py',
'mesonbuild/mintro.py',
@ -69,9 +69,9 @@ modules = [
]
if os.name == 'posix':
modules.append('mesonbuild/mesonlib/posix.py')
modules.append('mesonbuild/utils/posix.py')
elif os.name == 'nt':
modules.append('mesonbuild/mesonlib/win32.py')
modules.append('mesonbuild/utils/win32.py')
def check_mypy() -> None:
try:

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from mesonbuild.mesonlib.universal import windows_proof_rm
import subprocess
import re
import json
@ -42,7 +41,8 @@ from mesonbuild.mesonlib import (
BuildDirLock, MachineChoice, is_windows, is_osx, is_cygwin, is_dragonflybsd,
is_sunos, windows_proof_rmtree, python_command, version_compare, split_args, quote_arg,
relpath, is_linux, git, search_version, do_conf_file, do_conf_str, default_prefix,
MesonException, EnvironmentException, OptionKey
MesonException, EnvironmentException, OptionKey, ExecutableSerialisation, EnvironmentVariables,
windows_proof_rm
)
from mesonbuild.compilers.mixins.clang import ClangCompiler
@ -4409,3 +4409,31 @@ class AllPlatformTests(BasePlatformTests):
self.setconf(["-Dopt=val"])
newmtime = os.path.getmtime(filename)
self.assertEqual(oldmtime, newmtime)
def test_scripts_loaded_modules(self):
'''
Simulate a wrapped command, as done for custom_target() that capture
output. The script will print all python modules loaded and we verify
that it contains only an acceptable subset. Loading too many modules
slows down the build when many custom targets get wrapped.
'''
es = ExecutableSerialisation(python_command + ['-c', 'exit(0)'], env=EnvironmentVariables())
p = Path(self.builddir, 'exe.dat')
with p.open('wb') as f:
pickle.dump(es, f)
cmd = self.meson_command + ['--internal', 'test_loaded_modules', '--unpickle', str(p)]
p = subprocess.run(cmd, stdout=subprocess.PIPE)
all_modules = json.loads(p.stdout.splitlines()[0])
meson_modules = [m for m in all_modules if 'meson' in m]
expected_meson_modules = [
'mesonbuild',
'mesonbuild._pathlib',
'mesonbuild.utils',
'mesonbuild.utils.core',
'mesonbuild.mesonmain',
'mesonbuild.mlog',
'mesonbuild.scripts',
'mesonbuild.scripts.meson_exe',
'mesonbuild.scripts.test_loaded_modules'
]
self.assertEqual(sorted(expected_meson_modules), sorted(meson_modules))

@ -13,7 +13,6 @@
# limitations under the License.
from configparser import ConfigParser
from mesonbuild.mesonlib.universal import OptionType
from pathlib import Path
from unittest import mock
import contextlib
@ -43,6 +42,7 @@ from mesonbuild.interpreterbase import typed_pos_args, InvalidArguments, typed_k
from mesonbuild.mesonlib import (
LibType, MachineChoice, PerMachine, Version, is_windows, is_osx,
is_cygwin, is_openbsd, search_version, MesonException, OptionKey,
OptionType
)
from mesonbuild.interpreter.type_checking import in_set_validator, NoneType
from mesonbuild.dependencies import PkgConfigDependency

Loading…
Cancel
Save