Merge pull request #5265 from dcbaker/more-annotations

Fully annotate the envconfig and mlog modules

mesonbuild/envconfig.py

@@ -19,6 +19,8 @@ from . import mesonlib
 from .mesonlib import EnvironmentException, MachineChoice, PerMachine
 from . import mlog
 
+_T = typing.TypeVar('_T')
+
 
 # These classes contains all the data pulled from configuration files (native
 # and cross file currently), and also assists with the reading environment
@@ -69,7 +71,7 @@ CPU_FAMILES_64_BIT = [
 
 class MesonConfigFile:
     @classmethod
-    def from_config_parser(cls, parser: configparser.ConfigParser):
+    def from_config_parser(cls, parser: configparser.ConfigParser) -> typing.Dict[str, typing.Dict[str, typing.Dict[str, str]]]:
         out = {}
         # This is a bit hackish at the moment.
         for s in parser.sections():
@@ -106,55 +108,58 @@ class HasEnvVarFallback:
     that we deal with environment variables will become more structured, and
     this can be starting point.
     """
-    def __init__(self, fallback = True):
+    def __init__(self, fallback: bool = True):
         self.fallback = fallback
 
 class Properties(HasEnvVarFallback):
     def __init__(
             self,
             properties: typing.Optional[typing.Dict[str, typing.Union[str, typing.List[str]]]] = None,
-            fallback = True):
+            fallback: bool = True):
         super().__init__(fallback)
-        self.properties = properties or {}
+        self.properties = properties or {}  # type: typing.Dict[str, typing.Union[str, typing.List[str]]]
 
-    def has_stdlib(self, language):
+    def has_stdlib(self, language: str) -> bool:
         return language + '_stdlib' in self.properties
 
-    def get_stdlib(self, language):
+    # Some of get_stdlib, get_root, get_sys_root are wider than is actually
+    # true, but without heterogenious dict annotations it's not practical to
+    # narrow them
+    def get_stdlib(self, language: str) -> typing.Union[str, typing.List[str]]:
         return self.properties[language + '_stdlib']
 
-    def get_root(self):
+    def get_root(self) -> typing.Optional[typing.Union[str, typing.List[str]]]:
         return self.properties.get('root', None)
 
-    def get_sys_root(self):
+    def get_sys_root(self) -> typing.Optional[typing.Union[str, typing.List[str]]]:
         return self.properties.get('sys_root', None)
 
-    def __eq__(self, other):
+    def __eq__(self, other: typing.Any) -> typing.Union[bool, 'NotImplemented']:
         if isinstance(other, type(self)):
             return self.properties == other.properties
         return NotImplemented
 
     # TODO consider removing so Properties is less freeform
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> typing.Any:
         return self.properties[key]
 
     # TODO consider removing so Properties is less freeform
-    def __contains__(self, item):
+    def __contains__(self, item: typing.Any) -> bool:
         return item in self.properties
 
     # TODO consider removing, for same reasons as above
-    def get(self, key, default=None):
+    def get(self, key: str, default: typing.Any = None) -> typing.Any:
         return self.properties.get(key, default)
 
 class MachineInfo:
-    def __init__(self, system, cpu_family, cpu, endian):
+    def __init__(self, system: str, cpu_family: str, cpu: str, endian: str):
         self.system = system
         self.cpu_family = cpu_family
         self.cpu = cpu
         self.endian = endian
-        self.is_64_bit = cpu_family in CPU_FAMILES_64_BIT
+        self.is_64_bit = cpu_family in CPU_FAMILES_64_BIT  # type: bool
 
-    def __eq__(self, other):
+    def __eq__(self, other: typing.Any) -> typing.Union[bool, 'NotImplemented']:
         if self.__class__ is not other.__class__:
             return NotImplemented
         return \
@@ -163,16 +168,16 @@ class MachineInfo:
             self.cpu == other.cpu and \
             self.endian == other.endian
 
-    def __ne__(self, other):
+    def __ne__(self, other: typing.Any) -> typing.Union[bool, 'NotImplemented']:
         if self.__class__ is not other.__class__:
             return NotImplemented
         return not self.__eq__(other)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return '<MachineInfo: {} {} ({})>'.format(self.system, self.cpu_family, self.cpu)
 
-    @staticmethod
-    def from_literal(literal):
+    @classmethod
+    def from_literal(cls, literal: typing.Dict[str, str]) -> 'MachineInfo':
         minimum_literal = {'cpu', 'cpu_family', 'endian', 'system'}
         if set(literal) < minimum_literal:
             raise EnvironmentException(
@@ -187,49 +192,45 @@ class MachineInfo:
         if endian not in ('little', 'big'):
             mlog.warning('Unknown endian %s' % endian)
 
-        return MachineInfo(
-            literal['system'],
-            cpu_family,
-            literal['cpu'],
-            endian)
+        return cls(literal['system'], cpu_family, literal['cpu'], endian)
 
-    def is_windows(self):
+    def is_windows(self) -> bool:
         """
         Machine is windows?
         """
         return self.system == 'windows'
 
-    def is_cygwin(self):
+    def is_cygwin(self) -> bool:
         """
         Machine is cygwin?
         """
         return self.system == 'cygwin'
 
-    def is_linux(self):
+    def is_linux(self) -> bool:
         """
         Machine is linux?
         """
         return self.system == 'linux'
 
-    def is_darwin(self):
+    def is_darwin(self) -> bool:
         """
         Machine is Darwin (iOS/OS X)?
         """
         return self.system in ('darwin', 'ios')
 
-    def is_android(self):
+    def is_android(self) -> bool:
         """
         Machine is Android?
         """
         return self.system == 'android'
 
-    def is_haiku(self):
+    def is_haiku(self) -> bool:
         """
         Machine is Haiku?
         """
         return self.system == 'haiku'
 
-    def is_openbsd(self):
+    def is_openbsd(self) -> bool:
         """
         Machine is OpenBSD?
         """
@@ -239,29 +240,28 @@ class MachineInfo:
     # static libraries, and executables.
     # Versioning is added to these names in the backends as-needed.
 
-    def get_exe_suffix(self):
+    def get_exe_suffix(self) -> str:
         if self.is_windows() or self.is_cygwin():
             return 'exe'
         else:
             return ''
 
-    def get_object_suffix(self):
+    def get_object_suffix(self) -> str:
         if self.is_windows():
             return 'obj'
         else:
             return 'o'
 
-    def libdir_layout_is_win(self):
-        return self.is_windows() \
-            or self.is_cygwin()
+    def libdir_layout_is_win(self) -> bool:
+        return self.is_windows() or self.is_cygwin()
 
-class PerMachineDefaultable(PerMachine):
+class PerMachineDefaultable(PerMachine[_T]):
     """Extends `PerMachine` with the ability to default from `None`s.
     """
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__(None, None, None)
 
-    def default_missing(self):
+    def default_missing(self) -> None:
         """Default host to buid and target to host.
 
         This allows just specifying nothing in the native case, just host in the
@@ -273,7 +273,7 @@ class PerMachineDefaultable(PerMachine):
         if self.target is None:
             self.target = self.host
 
-    def miss_defaulting(self):
+    def miss_defaulting(self) -> None:
         """Unset definition duplicated from their previous to None
 
         This is the inverse of ''default_missing''. By removing defaulted
@@ -285,18 +285,17 @@ class PerMachineDefaultable(PerMachine):
         if self.host == self.build:
             self.host = None
 
-class MachineInfos(PerMachineDefaultable):
-    def matches_build_machine(self, machine: MachineChoice):
+class MachineInfos(PerMachineDefaultable[typing.Optional[MachineInfo]]):
+    def matches_build_machine(self, machine: MachineChoice) -> bool:
         return self.build == self[machine]
 
 class BinaryTable(HasEnvVarFallback):
     def __init__(
             self,
             binaries: typing.Optional[typing.Dict[str, typing.Union[str, typing.List[str]]]] = None,
-
-            fallback = True):
+            fallback: bool = True):
         super().__init__(fallback)
-        self.binaries = binaries or {}
+        self.binaries = binaries or {}  # type: typing.Dict[str, typing.Union[str, typing.List[str]]]
         for name, command in self.binaries.items():
             if not isinstance(command, (list, str)):
                 # TODO generalize message
@@ -325,29 +324,25 @@ class BinaryTable(HasEnvVarFallback):
         'cmake': 'CMAKE',
         'qmake': 'QMAKE',
         'pkgconfig': 'PKG_CONFIG',
-    }
+    }  # type: typing.Dict[str, str]
 
-    @classmethod
-    def detect_ccache(cls):
+    @staticmethod
+    def detect_ccache() -> typing.List[str]:
         try:
-            has_ccache = subprocess.call(['ccache', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        except OSError:
-            has_ccache = 1
-        if has_ccache == 0:
-            cmdlist = ['ccache']
-        else:
-            cmdlist = []
-        return cmdlist
+            subprocess.check_call(['ccache', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except (OSError, subprocess.CalledProcessError):
+            return []
+        return ['ccache']
 
     @classmethod
-    def _warn_about_lang_pointing_to_cross(cls, compiler_exe, evar):
+    def _warn_about_lang_pointing_to_cross(cls, compiler_exe: str, evar: str) -> None:
         evar_str = os.environ.get(evar, 'WHO_WOULD_CALL_THEIR_COMPILER_WITH_THIS_NAME')
         if evar_str == compiler_exe:
             mlog.warning('''Env var %s seems to point to the cross compiler.
 This is probably wrong, it should always point to the native compiler.''' % evar)
 
     @classmethod
-    def parse_entry(cls, entry):
+    def parse_entry(cls, entry: typing.Union[str, typing.List[str]]) -> typing.Tuple[typing.List[str], typing.List[str]]:
         compiler = mesonlib.stringlistify(entry)
         # Ensure ccache exists and remove it if it doesn't
         if compiler[0] == 'ccache':
@@ -358,8 +353,8 @@ This is probably wrong, it should always point to the native compiler.''' % evar
         # Return value has to be a list of compiler 'choices'
         return compiler, ccache
 
-    def lookup_entry(self, name):
-        """Lookup binary
+    def lookup_entry(self, name: str) -> typing.Optional[typing.List[str]]:
+        """Lookup binaryk
 
         Returns command with args as list if found, Returns `None` if nothing is
         found.
@@ -408,11 +403,12 @@ class Directories:
         self.sharedstatedir = sharedstatedir
         self.sysconfdir = sysconfdir
 
-    def __contains__(self, key: str) -> str:
+    def __contains__(self, key: str) -> bool:
         return hasattr(self, key)
 
-    def __getitem__(self, key: str) -> str:
-        return getattr(self, key)
+    def __getitem__(self, key: str) -> typing.Optional[str]:
+        # Mypy can't figure out what to do with getattr here, so we'll case for it
+        return typing.cast(typing.Optional[str], getattr(self, key))
 
     def __setitem__(self, key: str, value: typing.Optional[str]) -> None:
         setattr(self, key, value)

mesonbuild/mesonlib.py

@@ -22,9 +22,13 @@ import platform, subprocess, operator, os, shutil, re
 import collections
 from enum import Enum
 from functools import lru_cache
+import typing
 
 from mesonbuild import mlog
 
+_T = typing.TypeVar('_T')
+_U = typing.TypeVar('_U')
+
 have_fcntl = False
 have_msvcrt = False
 # {subproject: project_meson_version}
@@ -319,20 +323,22 @@ class MachineChoice(OrderedEnum):
     HOST = 1
     TARGET = 2
 
-class PerMachine:
-    def __init__(self, build, host, target):
+_T = typing.TypeVar('_T')
+
+class PerMachine(typing.Generic[_T]):
+    def __init__(self, build: typing.Optional[_T], host: typing.Optional[_T], target: typing.Optional[_T]):
         self.build = build
         self.host = host
         self.target = target
 
-    def __getitem__(self, machine: MachineChoice):
+    def __getitem__(self, machine: MachineChoice) -> typing.Optional[_T]:
         return {
             MachineChoice.BUILD:  self.build,
             MachineChoice.HOST:   self.host,
             MachineChoice.TARGET: self.target
         }[machine]
 
-    def __setitem__(self, machine: MachineChoice, val):
+    def __setitem__(self, machine: MachineChoice, val: typing.Optional[_T]) -> None:
         key = {
             MachineChoice.BUILD:  'build',
             MachineChoice.HOST:   'host',
@@ -916,14 +922,13 @@ def extract_as_list(dict_object, *keys, pop=False, **kwargs):
         result.append(listify(fetch(key, []), **kwargs))
     return result
 
-
-def typeslistify(item, types):
+def typeslistify(item: typing.Union[_T, typing.List[_T]], types: typing.Union[typing.Type[_T], typing.Tuple[typing.Type[_T]]]) -> typing.List[_T]:
     '''
     Ensure that type(@item) is one of @types or a
     list of items all of which are of type @types
     '''
     if isinstance(item, types):
-        item = [item]
+        item = typing.cast(typing.List[_T], [item])
     if not isinstance(item, list):
         raise MesonException('Item must be a list or one of {!r}'.format(types))
     for i in item:
@@ -931,7 +936,7 @@ def typeslistify(item, types):
             raise MesonException('List item must be one of {!r}'.format(types))
     return item
 
-def stringlistify(item):
+def stringlistify(item: typing.Union[str, typing.List[str]]) -> typing.List[str]:
     return typeslistify(item, str)
 
 def expand_arguments(args):
@@ -1202,7 +1207,14 @@ def detect_subprojects(spdir_name, current_dir='', result=None):
                 result[basename] = [trial]
     return result
 
-def get_error_location_string(fname, lineno):
+# This isn't strictly correct. What we really want here is something like:
+# class StringProtocol(typing_extensions.Protocol):
+#
+#      def __str__(self) -> str: ...
+#
+# This would more accurately embody what this funcitonc an handle, but we
+# don't have that yet, so instead we'll do some casting to work around it
+def get_error_location_string(fname: str, lineno: str) -> str:
     return '{}:{}:'.format(fname, lineno)
 
 def substring_is_in_list(substr, strlist):

mesonbuild/mlog.py

@@ -18,13 +18,15 @@ import sys
 import time
 import platform
 from contextlib import contextmanager
+import typing
 
 """This is (mostly) a standalone module used to write logging
 information about Meson runs. Some output goes to screen,
 some to logging dir and some goes to both."""
 
-def _windows_ansi():
-    from ctypes import windll, byref
+def _windows_ansi() -> bool:
+    # windll only exists on windows, so mypy will get mad
+    from ctypes import windll, byref  # type: ignore
     from ctypes.wintypes import DWORD
 
     kernel = windll.kernel32
@@ -35,48 +37,48 @@ def _windows_ansi():
     # ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0x4
     # If the call to enable VT processing fails (returns 0), we fallback to
     # original behavior
-    return kernel.SetConsoleMode(stdout, mode.value | 0x4) or os.environ.get('ANSICON')
+    return bool(kernel.SetConsoleMode(stdout, mode.value | 0x4) or os.environ.get('ANSICON'))
 
 if platform.system().lower() == 'windows':
-    colorize_console = os.isatty(sys.stdout.fileno()) and _windows_ansi()
+    colorize_console = os.isatty(sys.stdout.fileno()) and _windows_ansi()  # type: bool
 else:
     colorize_console = os.isatty(sys.stdout.fileno()) and os.environ.get('TERM') != 'dumb'
-log_dir = None
-log_file = None
-log_fname = 'meson-log.txt'
-log_depth = 0
-log_timestamp_start = None
-log_fatal_warnings = False
-log_disable_stdout = False
-log_errors_only = False
-
-def disable():
+log_dir = None               # type: typing.Optional[str]
+log_file = None              # type: typing.Optional[typing.TextIO]
+log_fname = 'meson-log.txt'  # type: str
+log_depth = 0                # type: int
+log_timestamp_start = None   # type: typing.Optional[float]
+log_fatal_warnings = False   # type: bool
+log_disable_stdout = False   # type: bool
+log_errors_only = False      # type: bool
+
+def disable() -> None:
     global log_disable_stdout
     log_disable_stdout = True
 
-def enable():
+def enable() -> None:
     global log_disable_stdout
     log_disable_stdout = False
 
-def set_quiet():
+def set_quiet() -> None:
     global log_errors_only
     log_errors_only = True
 
-def set_verbose():
+def set_verbose() -> None:
     global log_errors_only
     log_errors_only = False
 
-def initialize(logdir, fatal_warnings=False):
+def initialize(logdir: str, fatal_warnings: bool = False) -> None:
     global log_dir, log_file, log_fatal_warnings
     log_dir = logdir
     log_file = open(os.path.join(logdir, log_fname), 'w', encoding='utf8')
     log_fatal_warnings = fatal_warnings
 
-def set_timestamp_start(start):
+def set_timestamp_start(start: float) -> None:
     global log_timestamp_start
     log_timestamp_start = start
 
-def shutdown():
+def shutdown() -> typing.Optional[str]:
     global log_file
     if log_file is not None:
         path = log_file.name
@@ -89,12 +91,12 @@ def shutdown():
 class AnsiDecorator:
     plain_code = "\033[0m"
 
-    def __init__(self, text, code, quoted=False):
+    def __init__(self, text: str, code: str, quoted: bool = False):
         self.text = text
         self.code = code
         self.quoted = quoted
 
-    def get_text(self, with_codes):
+    def get_text(self, with_codes: bool) -> str:
         text = self.text
         if with_codes:
             text = self.code + self.text + AnsiDecorator.plain_code
@@ -102,26 +104,28 @@ class AnsiDecorator:
             text = '"{}"'.format(text)
         return text
 
-def bold(text, quoted=False):
+def bold(text: str, quoted: bool = False) -> AnsiDecorator:
     return AnsiDecorator(text, "\033[1m", quoted=quoted)
 
-def red(text):
+def red(text: str) -> AnsiDecorator:
     return AnsiDecorator(text, "\033[1;31m")
 
-def green(text):
+def green(text: str) -> AnsiDecorator:
     return AnsiDecorator(text, "\033[1;32m")
 
-def yellow(text):
+def yellow(text: str) -> AnsiDecorator:
     return AnsiDecorator(text, "\033[1;33m")
 
-def blue(text):
+def blue(text: str) -> AnsiDecorator:
     return AnsiDecorator(text, "\033[1;34m")
 
-def cyan(text):
+def cyan(text: str) -> AnsiDecorator:
     return AnsiDecorator(text, "\033[1;36m")
 
-def process_markup(args, keep):
-    arr = []
+# This really should be AnsiDecorator or anything that implements
+# __str__(), but that requires protocols from typing_extensions
+def process_markup(args: typing.Sequence[typing.Union[AnsiDecorator, str]], keep: bool) -> typing.List[str]:
+    arr = []  # type: typing.List[str]
     if log_timestamp_start is not None:
         arr = ['[{:.3f}]'.format(time.monotonic() - log_timestamp_start)]
     for arg in args:
@@ -135,7 +139,7 @@ def process_markup(args, keep):
             arr.append(str(arg))
     return arr
 
-def force_print(*args, **kwargs):
+def force_print(*args: str, **kwargs: typing.Any) -> None:
     global log_disable_stdout
     if log_disable_stdout:
         return
@@ -155,41 +159,51 @@ def force_print(*args, **kwargs):
         cleaned = raw.encode('ascii', 'replace').decode('ascii')
         print(cleaned, end='')
 
-def debug(*args, **kwargs):
+# We really want a heterogenous dict for this, but that's in typing_extensions
+def debug(*args: typing.Union[str, AnsiDecorator], **kwargs: typing.Any) -> None:
     arr = process_markup(args, False)
     if log_file is not None:
-        print(*arr, file=log_file, **kwargs) # Log file never gets ANSI codes.
+        print(*arr, file=log_file, **kwargs)
         log_file.flush()
 
-def log(*args, is_error=False, **kwargs):
+def log(*args: typing.Union[str, AnsiDecorator], is_error: bool = False,
+        **kwargs: typing.Any) -> None:
     global log_errors_only
     arr = process_markup(args, False)
     if log_file is not None:
-        print(*arr, file=log_file, **kwargs) # Log file never gets ANSI codes.
+        print(*arr, file=log_file, **kwargs)
         log_file.flush()
     if colorize_console:
         arr = process_markup(args, True)
     if not log_errors_only or is_error:
         force_print(*arr, **kwargs)
 
-def _log_error(severity, *args, **kwargs):
+def _log_error(severity: str, *rargs: typing.Union[str, AnsiDecorator], **kwargs: typing.Any) -> None:
     from .mesonlib import get_error_location_string
     from .environment import build_filename
     from .mesonlib import MesonException
+
+    # The tping requirements here are non-obvious. Lists are invariant,
+    # therefore List[A] and List[Union[A, B]] are not able to be joined
     if severity == 'warning':
-        args = (yellow('WARNING:'),) + args
+        label = [yellow('WARNING:')]  # type: typing.List[typing.Union[str, AnsiDecorator]]
     elif severity == 'error':
-        args = (red('ERROR:'),) + args
+        label = [red('ERROR:')]
     elif severity == 'deprecation':
-        args = (red('DEPRECATION:'),) + args
+        label = [red('DEPRECATION:')]
     else:
-        assert False, 'Invalid severity ' + severity
+        raise MesonException('Invalid severity ' + severity)
+    # rargs is a tuple, not a list
+    args = label + list(rargs)
 
     location = kwargs.pop('location', None)
     if location is not None:
         location_file = os.path.join(location.subdir, build_filename)
         location_str = get_error_location_string(location_file, location.lineno)
-        args = (location_str,) + args
+        # Unions are frankly awful, and we have to cast here to get mypy
+        # to understand that the list concatenation is safe
+        location_list = typing.cast(typing.List[typing.Union[str, AnsiDecorator]], [location_str])
+        args = location_list + args
 
     log(*args, **kwargs)
 
@@ -197,40 +211,42 @@ def _log_error(severity, *args, **kwargs):
     if log_fatal_warnings:
         raise MesonException("Fatal warnings enabled, aborting")
 
-def error(*args, **kwargs):
+def error(*args: typing.Union[str, AnsiDecorator], **kwargs: typing.Any) -> None:
     return _log_error('error', *args, **kwargs, is_error=True)
 
-def warning(*args, **kwargs):
+def warning(*args: typing.Union[str, AnsiDecorator], **kwargs: typing.Any) -> None:
     return _log_error('warning', *args, **kwargs, is_error=True)
 
-def deprecation(*args, **kwargs):
+def deprecation(*args: typing.Union[str, AnsiDecorator], **kwargs: typing.Any) -> None:
     return _log_error('deprecation', *args, **kwargs, is_error=True)
 
-def exception(e, prefix=red('ERROR:')):
+def exception(e: Exception, prefix: AnsiDecorator = red('ERROR:')) -> None:
     log()
-    args = []
+    args = []  # type: typing.List[typing.Union[AnsiDecorator, str]]
     if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(e, 'colno'):
-        args.append('%s:%d:%d:' % (e.file, e.lineno, e.colno))
+        # Mypy can't figure this out, and it's pretty easy to vidual inspect
+        # that this is correct, so we'll just ignore it.
+        args.append('%s:%d:%d:' % (e.file, e.lineno, e.colno))  # type: ignore
     if prefix:
         args.append(prefix)
-    args.append(e)
+    args.append(str(e))
     log(*args)
 
 # Format a list for logging purposes as a string. It separates
 # all but the last item with commas, and the last with 'and'.
-def format_list(list):
-    l = len(list)
+def format_list(list_: typing.List[str]) -> str:
+    l = len(list_)
     if l > 2:
-        return ' and '.join([', '.join(list[:-1]), list[-1]])
+        return ' and '.join([', '.join(list_[:-1]), list_[-1]])
     elif l == 2:
-        return ' and '.join(list)
+        return ' and '.join(list_)
     elif l == 1:
-        return list[0]
+        return list_[0]
     else:
         return ''
 
 @contextmanager
-def nested():
+def nested() -> typing.Generator[None, None, None]:
     global log_depth
     log_depth += 1
     try: