split mesonlib into a package

Currently mesonlib does some import tricks to figure out whether it
needs to use windows or posix specific functions. This is a little
hacky, but works fine. However, the way the typing stubs are implemented
for the msvcrt and fnctl modules will cause mypy to fail on the other
platform, since the functions are not implemented.

To aleviate this (and for slightly cleaner design), I've split mesonlib
into a pacakge with three modules. A universal module contains all of
the platform agnositc code, a win32 module contains window specific
code, a posix module contains the posix specific code, and a platform
module contains no-op implementations. Then the package's __init__ file
imports all of the universal functions and all of the functions from the
approriate platform module, or the no-op versions as fallbacks. This
makes mypy happy, and avoids `if`ing all over the code to switch between
the platform specific code.

mesonbuild/backend/ninjabackend.py

@@ -3026,14 +3026,14 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
 
     def generate_gcov_clean(self):
         gcno_elem = NinjaBuildElement(self.all_outputs, 'meson-clean-gcno', 'CUSTOM_COMMAND', 'PHONY')
-        gcno_elem.add_item('COMMAND', mesonlib.meson_command + ['--internal', 'delwithsuffix', '.', 'gcno'])
+        gcno_elem.add_item('COMMAND', mesonlib.get_meson_command() + ['--internal', 'delwithsuffix', '.', 'gcno'])
         gcno_elem.add_item('description', 'Deleting gcno files')
         self.add_build(gcno_elem)
         # Alias that runs the target defined above
         self.create_target_alias('meson-clean-gcno')
 
         gcda_elem = NinjaBuildElement(self.all_outputs, 'meson-clean-gcda', 'CUSTOM_COMMAND', 'PHONY')
-        gcda_elem.add_item('COMMAND', mesonlib.meson_command + ['--internal', 'delwithsuffix', '.', 'gcda'])
+        gcda_elem.add_item('COMMAND', mesonlib.get_meson_command() + ['--internal', 'delwithsuffix', '.', 'gcda'])
         gcda_elem.add_item('description', 'Deleting gcda files')
         self.add_build(gcda_elem)
         # Alias that runs the target defined above

mesonbuild/backend/xcodebackend.py

@@ -578,7 +578,7 @@ class XCodeBackend(backends.Backend):
         self.write_line(');')
         self.write_line('runOnlyForDeploymentPostprocessing = 0;')
         self.write_line('shellPath = /bin/sh;')
-        cmd = mesonlib.meson_command + ['test', test_data, '-C', self.environment.get_build_dir()]
+        cmd = mesonlib.get_meson_command() + ['test', test_data, '-C', self.environment.get_build_dir()]
         cmdstr = ' '.join(["'%s'" % i for i in cmd])
         self.write_line('shellScript = "%s";' % cmdstr)
         self.write_line('showEnvVarsInLog = 0;')

mesonbuild/cmake/interpreter.py

@@ -1322,7 +1322,7 @@ class CMakeInterpreter:
 
             # Generate the command list
             command = []  # type: T.List[T.Union[str, IdNode, IndexNode]]
-            command += mesonlib.meson_command
+            command += mesonlib.get_meson_command()
             command += ['--internal', 'cmake_run_ctgt']
             command += ['-o', '@OUTPUT@']
             if tgt.original_outputs:

mesonbuild/compilers/c.py

@@ -16,7 +16,8 @@ import os.path
 import typing as T
 
 from .. import coredata
-from ..mesonlib import MachineChoice, MesonException, mlog, version_compare, OptionKey
+from .. import mlog
+from ..mesonlib import MachineChoice, MesonException, version_compare, OptionKey
 from .c_function_attributes import C_FUNC_ATTRIBUTES
 from .mixins.clike import CLikeCompiler
 from .mixins.ccrx import CcrxCompiler

mesonbuild/environment.py

@@ -867,7 +867,7 @@ class Environment:
         # re-initialized with project options by the interpreter during
         # build file parsing.
         # meson_command is used by the regenchecker script, which runs meson
-        self.coredata = coredata.CoreData(options, self.scratch_dir, mesonlib.meson_command)
+        self.coredata = coredata.CoreData(options, self.scratch_dir, mesonlib.get_meson_command())
         self.first_invocation = True
 
     def is_cross_build(self, when_building_for: MachineChoice = MachineChoice.HOST) -> bool:
@@ -887,7 +887,7 @@ class Environment:
         return self.coredata
 
     def get_build_command(self, unbuffered=False):
-        cmd = mesonlib.meson_command[:]
+        cmd = mesonlib.get_meson_command().copy()
         if unbuffered and 'python' in os.path.basename(cmd[0]):
             cmd.insert(1, '-u')
         return cmd

mesonbuild/mdist.py

@@ -245,7 +245,7 @@ def run(options):
     b = build.load(options.wd)
     # This import must be load delayed, otherwise it will get the default
     # value of None.
-    from mesonbuild.mesonlib import meson_command
+    from mesonbuild.mesonlib import get_meson_command
     src_root = b.environment.source_dir
     bld_root = b.environment.build_dir
     priv_dir = os.path.join(bld_root, 'meson-private')
@@ -279,7 +279,7 @@ def run(options):
     rc = 0
     if not options.no_tests:
         # Check only one.
-        rc = check_dist(names[0], meson_command, extra_meson_args, bld_root, priv_dir)
+        rc = check_dist(names[0], get_meson_command(), extra_meson_args, bld_root, priv_dir)
     if rc == 0:
         for name in names:
             create_hash(name)

mesonbuild/mesonlib/__init__.py

@@ -0,0 +1,30 @@
+# SPDX-license-identifier: Apache-2.0
+# Copyright 2012-2021 The Meson development team
+# Copyright © 2021 Intel Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper functions and classes."""
+
+import os
+
+from .universal import *
+
+# Here we import either the posix implementations, the windows implementations,
+# or a generic no-op implementation
+if os.name == 'posix':
+    from .posix import *
+elif os.name == 'nt':
+    from .win32 import *
+else:
+    from .platform import *

mesonbuild/mesonlib/platform.py

@@ -0,0 +1,37 @@
+# SPDX-license-identifier: Apache-2.0
+# Copyright 2012-2021 The Meson development team
+# Copyright © 2021 Intel Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""base classes providing no-op functionality.."""
+
+import os
+import typing as T
+
+from .. import mlog
+
+__all__ = ['BuildDirLock']
+
+# This needs to be inheritted by the specific implementations to make type
+# checking happy
+class BuildDirLock:
+
+    def __init__(self, builddir: str) -> None:
+        self.lockfilename = os.path.join(builddir, 'meson-private/meson.lock')
+
+    def __enter__(self) -> None:
+        mlog.debug('Calling ther no-op version of BuildDirLock')
+
+    def __exit__(self, *args: T.Any) -> None:
+        pass

mesonbuild/mesonlib/posix.py

@@ -0,0 +1,39 @@
+# SPDX-license-identifier: Apache-2.0
+# Copyright 2012-2021 The Meson development team
+# Copyright © 2021 Intel Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Posix specific implementations of mesonlib functionality."""
+
+import fcntl
+import typing as T
+
+from .universal import MesonException
+from .platform import BuildDirLock as BuildDirLockBase
+
+__all__ = ['BuildDirLock']
+
+class BuildDirLock(BuildDirLockBase):
+
+    def __enter__(self) -> None:
+        self.lockfile = open(self.lockfilename, 'w')
+        try:
+            fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except (BlockingIOError, PermissionError):
+            self.lockfile.close()
+            raise MesonException('Some other Meson process is already using this build directory. Exiting.')
+
+    def __exit__(self, *args: T.Any) -> None:
+        fcntl.flock(self.lockfile, fcntl.LOCK_UN)
+        self.lockfile.close()

mesonbuild/mesonlib/universal.py

@@ -30,34 +30,117 @@ import textwrap
 from mesonbuild import mlog
 
 if T.TYPE_CHECKING:
-    from .build import ConfigurationData
+    from ..build import ConfigurationData
-    from .coredata import KeyedOptionDictType, UserOption
+    from ..coredata import KeyedOptionDictType, UserOption
-    from .compilers.compilers import CompilerType
+    from ..compilers.compilers import CompilerType
-    from .interpreterbase import ObjectHolder
+    from ..interpreterbase import ObjectHolder
 
-    FileOrString = T.Union['File', str]
+FileOrString = T.Union['File', str]
 
 _T = T.TypeVar('_T')
 _U = T.TypeVar('_U')
 
-have_fcntl = False
+__all__ = [
-have_msvcrt = False
+    'GIT',
+    'an_unpicklable_object',
+    'python_command',
+    'project_meson_versions',
+    'File',
+    'FileMode',
+    'GitException',
+    'LibType',
+    'MachineChoice',
+    'MesonException',
+    'EnvironmentException',
+    'FileOrString',
+    'GitException',
+    'OptionKey',
+    'dump_conf_header',
+    'OptionOverrideProxy',
+    'OptionProxy',
+    'OptionType',
+    'OrderedSet',
+    'PerMachine',
+    'PerMachineDefaultable',
+    'PerThreeMachine',
+    'PerThreeMachineDefaultable',
+    'ProgressBar',
+    'TemporaryDirectoryWinProof',
+    'Version',
+    'check_direntry_issues',
+    'classify_unity_sources',
+    'current_vs_supports_modules',
+    'darwin_get_object_archs',
+    'default_libdir',
+    'default_libexecdir',
+    'default_prefix',
+    'detect_subprojects',
+    'detect_vcs',
+    'do_conf_file',
+    'do_conf_str',
+    'do_define',
+    'do_replacement',
+    'exe_exists',
+    'expand_arguments',
+    'extract_as_list',
+    'get_compiler_for_source',
+    'get_filenames_templates_dict',
+    'get_library_dirs',
+    'get_variable_regex',
+    'get_wine_shortpath',
+    'git',
+    'has_path_sep',
+    'is_aix',
+    'is_android',
+    'is_ascii_string',
+    'is_cygwin',
+    'is_debianlike',
+    'is_dragonflybsd',
+    'is_freebsd',
+    'is_haiku',
+    'is_hurd',
+    'is_irix',
+    'is_linux',
+    'is_netbsd',
+    'is_openbsd',
+    'is_osx',
+    'is_qnx',
+    'is_sunos',
+    'is_windows',
+    'iter_regexin_iter',
+    'join_args',
+    'listify',
+    'partition',
+    'path_is_in_root',
+    'Popen_safe',
+    'quiet_git',
+    'quote_arg',
+    'relative_to_if_possible',
+    'relpath',
+    'replace_if_different',
+    'run_once',
+    'get_meson_command',
+    'set_meson_command',
+    'split_args',
+    'stringlistify',
+    'substitute_values',
+    'substring_is_in_list',
+    'typeslistify',
+    'unholder',
+    'verbose_git',
+    'version_compare',
+    'version_compare_condition_with_min',
+    'version_compare_many',
+    'windows_proof_rm',
+    'windows_proof_rmtree',
+]
+
+
 # TODO: this is such a hack, this really should be either in coredata or in the
 # interpreter
 # {subproject: project_meson_version}
 project_meson_versions = collections.defaultdict(str)  # type: T.DefaultDict[str, str]
 
-try:
-    import fcntl
-    have_fcntl = True
-except Exception:
-    pass
-
-try:
-    import msvcrt
-    have_msvcrt = True
-except Exception:
-    pass
 
 from glob import glob
 
@@ -66,7 +149,7 @@ if os.path.basename(sys.executable) == 'meson.exe':
     python_command = [sys.executable, 'runpython']
 else:
     python_command = [sys.executable]
-meson_command = None
+_meson_command = None
 
 class MesonException(Exception):
     '''Exceptions thrown by Meson'''
@@ -117,20 +200,24 @@ def verbose_git(cmd: T.List[str], workingdir: str, check: bool = False) -> bool:
 
 def set_meson_command(mainfile: str) -> None:
     global python_command
-    global meson_command
+    global _meson_command
     # On UNIX-like systems `meson` is a Python script
     # On Windows `meson` and `meson.exe` are wrapper exes
     if not mainfile.endswith('.py'):
-        meson_command = [mainfile]
+        _meson_command = [mainfile]
     elif os.path.isabs(mainfile) and mainfile.endswith('mesonmain.py'):
         # Can't actually run meson with an absolute path to mesonmain.py, it must be run as -m mesonbuild.mesonmain
-        meson_command = python_command + ['-m', 'mesonbuild.mesonmain']
+        _meson_command = python_command + ['-m', 'mesonbuild.mesonmain']
     else:
         # Either run uninstalled, or full path to meson-script.py
-        meson_command = python_command + [mainfile]
+        _meson_command = python_command + [mainfile]
     # We print this value for unit tests.
     if 'MESON_COMMAND_TESTS' in os.environ:
-        mlog.log('meson_command is {!r}'.format(meson_command))
+        mlog.log('meson_command is {!r}'.format(_meson_command))
+
+
+def get_meson_command() -> T.Optional[T.List[str]]:
+    return _meson_command
 
 
 def is_ascii_string(astring: T.Union[str, bytes]) -> bool:
@@ -1564,29 +1651,6 @@ class OrderedSet(T.MutableSet[_T]):
     def difference(self, set_: T.Union[T.Set[_T], 'OrderedSet[_T]']) -> 'OrderedSet[_T]':
         return type(self)(e for e in self if e not in set_)
 
-class BuildDirLock:
-
-    def __init__(self, builddir: str) -> None:
-        self.lockfilename = os.path.join(builddir, 'meson-private/meson.lock')
-
-    def __enter__(self) -> None:
-        self.lockfile = open(self.lockfilename, 'w')
-        try:
-            if have_fcntl:
-                fcntl.flock(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
-            elif have_msvcrt:
-                msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
-        except (BlockingIOError, PermissionError):
-            self.lockfile.close()
-            raise MesonException('Some other Meson process is already using this build directory. Exiting.')
-
-    def __exit__(self, *args: T.Any) -> None:
-        if have_fcntl:
-            fcntl.flock(self.lockfile, fcntl.LOCK_UN)
-        elif have_msvcrt:
-            msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
-        self.lockfile.close()
-
 def relpath(path: str, start: str) -> str:
     # On Windows a relative path can't be evaluated for paths on two different
     # drives (i.e. c:\foo and f:\bar).  The only thing left to do is to use the
@@ -1965,7 +2029,7 @@ class OptionKey:
             raw3 = raw2
             for_machine = MachineChoice.HOST
 
-        from .compilers import all_languages
+        from ..compilers import all_languages
         if any(raw3.startswith(f'{l}_') for l in all_languages):
             lang, opt = raw3.split('_', 1)
         else:

mesonbuild/mesonlib/win32.py

@@ -0,0 +1,39 @@
+# SPDX-license-identifier: Apache-2.0
+# Copyright 2012-2021 The Meson development team
+# Copyright © 2021 Intel Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Windows specific implementations of mesonlib functionality."""
+
+import msvcrt
+import typing as T
+
+from .universal import MesonException
+from .platform import BuildDirLock as BuildDirLockBase
+
+__all__ = ['BuildDirLock']
+
+class BuildDirLock(BuildDirLockBase):
+
+    def __enter__(self) -> None:
+        self.lockfile = open(self.lockfilename, 'w')
+        try:
+            msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
+        except (BlockingIOError, PermissionError):
+            self.lockfile.close()
+            raise MesonException('Some other Meson process is already using this build directory. Exiting.')
+
+    def __exit__(self, *args: T.Any) -> None:
+        msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
+        self.lockfile.close()

mesonbuild/minit.py

@@ -174,7 +174,7 @@ def run(options: 'argparse.Namespace') -> int:
             print('Build directory already exists, deleting it.')
             shutil.rmtree(options.builddir)
         print('Building...')
-        cmd = mesonlib.meson_command + [options.builddir]
+        cmd = mesonlib.get_meson_command() + [options.builddir]
         ret = subprocess.run(cmd)
         if ret.returncode:
             raise SystemExit

run_mypy.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
-import sys
-import subprocess
-import argparse
 from pathlib import Path
+import argparse
+import os
+import subprocess
+import sys
 import typing as T
 
 modules = [
@@ -24,7 +25,8 @@ modules = [
     'mesonbuild/interpreterbase.py',
     'mesonbuild/linkers.py',
     'mesonbuild/mcompile.py',
-    'mesonbuild/mesonlib.py',
+    'mesonbuild/mesonlib/platform.py',
+    'mesonbuild/mesonlib/universal.py',
     'mesonbuild/minit.py',
     'mesonbuild/minstall.py',
     'mesonbuild/mintro.py',
@@ -40,6 +42,11 @@ modules = [
     'tools'
 ]
 
+if os.name == 'posix':
+    modules.append('mesonbuild/mesonlib/posix.py')
+elif os.name == 'nt':
+    modules.append('mesonbuild/mesonlib/win32.py')
+
 def check_mypy() -> None:
     try:
         import mypy

setup.py

@@ -33,6 +33,7 @@ packages = ['mesonbuild',
             'mesonbuild.compilers',
             'mesonbuild.compilers.mixins',
             'mesonbuild.dependencies',
+            'mesonbuild.mesonlib',
             'mesonbuild.modules',
             'mesonbuild.scripts',
             'mesonbuild.templates',