mintro: Introspection interpreter refactoring (#4733)

* Fixed spelling

* Merged the Buildoptions and Projectinfo interpreter

* Moved detect_compilers to Environment

* Added removed test case

* Split detect_compilers and moved even more code into Environment

* Moved set_default_options to coredata

* Small code simplification in mintro.run

* Move cmd_line_options back to `environment`

We don't actually wish to persist something this unstructured, so we
shouldn't make it a field on `coredata`. It would also be data
denormalization since the information we already store in coredata
depends on the CLI args.

mesonbuild/coredata.py

@@ -553,6 +553,45 @@ class CoreData:
 
         self.set_options(options, subproject)
 
+    def process_new_compilers(self, lang: str, comp, cross_comp, cmd_line_options):
+        from . import compilers
+        self.compilers[lang] = comp
+        # Native compiler always exist so always add its options.
+        new_options = comp.get_options()
+        if cross_comp is not None:
+            self.cross_compilers[lang] = cross_comp
+            new_options.update(cross_comp.get_options())
+
+        optprefix = lang + '_'
+        for k, o in new_options.items():
+            if not k.startswith(optprefix):
+                raise MesonException('Internal error, %s has incorrect prefix.' % k)
+            if k in cmd_line_options:
+                o.set_value(cmd_line_options[k])
+            self.compiler_options.setdefault(k, o)
+
+        # Unlike compiler and linker flags, preprocessor flags are not in
+        # compiler_options because they are not visible to user.
+        preproc_flags = comp.get_preproc_flags()
+        preproc_flags = shlex.split(preproc_flags)
+        self.external_preprocess_args.setdefault(lang, preproc_flags)
+
+        enabled_opts = []
+        for optname in comp.base_options:
+            if optname in self.base_options:
+                continue
+            oobj = compilers.base_options[optname]
+            if optname in cmd_line_options:
+                oobj.set_value(cmd_line_options[optname])
+                enabled_opts.append(optname)
+            self.base_options[optname] = oobj
+        self.emit_base_options_warnings(enabled_opts)
+
+    def emit_base_options_warnings(self, enabled_opts: list):
+        if 'b_bitcode' in enabled_opts:
+            mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as such as \'b_asneeded\' have been disabled.')
+            mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.')
+
 class CmdLineFileParser(configparser.ConfigParser):
     def __init__(self):
         # We don't want ':' as key delimiter, otherwise it would break when

mesonbuild/environment.py

@@ -33,6 +33,7 @@ from .compilers import (
     is_source,
 )
 from .compilers import (
+    Compiler,
     ArmCCompiler,
     ArmCPPCompiler,
     ArmclangCCompiler,
@@ -973,7 +974,7 @@ class Environment:
             return compilers.SwiftCompiler(exelist, version)
         raise EnvironmentException('Unknown compiler "' + ' '.join(exelist) + '"')
 
-    def detect_compilers(self, lang, need_cross_compiler):
+    def compilers_from_language(self, lang: str, need_cross_compiler: bool):
         comp = None
         cross_comp = None
         if lang == 'c':
@@ -1021,7 +1022,23 @@ class Environment:
             if need_cross_compiler:
                 raise EnvironmentException('Cross compilation with Swift is not working yet.')
                 # cross_comp = self.environment.detect_fortran_compiler(True)
+        else:
+            return None, None
+
+        return comp, cross_comp
+
+    def check_compilers(self, lang: str, comp: Compiler, cross_comp: Compiler):
+        if comp is None:
+            raise EnvironmentException('Tried to use unknown language "%s".' % lang)
+
+        comp.sanity_check(self.get_scratch_dir(), self)
+        if cross_comp:
+            cross_comp.sanity_check(self.get_scratch_dir(), self)
 
+    def detect_compilers(self, lang: str, need_cross_compiler: bool):
+        (comp, cross_comp) = self.compilers_from_language(lang, need_cross_compiler)
+        if comp is not None:
+            self.coredata.process_new_compilers(lang, comp, cross_comp, self.cmd_line_options)
         return comp, cross_comp
 
     def detect_static_linker(self, compiler):

mesonbuild/interpreter.py

@@ -2657,36 +2657,6 @@ external dependencies (including libraries) must go to "dependencies".''')
         self.validate_arguments(args, 0, [])
         raise Exception()
 
-    def detect_compilers(self, lang, need_cross_compiler):
-        comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler)
-        if comp is None:
-            raise InvalidCode('Tried to use unknown language "%s".' % lang)
-
-        comp.sanity_check(self.environment.get_scratch_dir(), self.environment)
-        self.coredata.compilers[lang] = comp
-        # Native compiler always exist so always add its options.
-        new_options = comp.get_options()
-        if cross_comp is not None:
-            cross_comp.sanity_check(self.environment.get_scratch_dir(), self.environment)
-            self.coredata.cross_compilers[lang] = cross_comp
-            new_options.update(cross_comp.get_options())
-
-        optprefix = lang + '_'
-        for k, o in new_options.items():
-            if not k.startswith(optprefix):
-                raise InterpreterException('Internal error, %s has incorrect prefix.' % k)
-            if k in self.environment.cmd_line_options:
-                o.set_value(self.environment.cmd_line_options[k])
-            self.coredata.compiler_options.setdefault(k, o)
-
-        # Unlike compiler and linker flags, preprocessor flags are not in
-        # compiler_options because they are not visible to user.
-        preproc_flags = comp.get_preproc_flags()
-        preproc_flags = shlex.split(preproc_flags)
-        self.coredata.external_preprocess_args.setdefault(lang, preproc_flags)
-
-        return comp, cross_comp
-
     def add_languages(self, args, required):
         success = True
         need_cross_compiler = self.environment.is_cross_build()
@@ -2697,7 +2667,8 @@ external dependencies (including libraries) must go to "dependencies".''')
                 cross_comp = self.coredata.cross_compilers.get(lang, None)
             else:
                 try:
-                    (comp, cross_comp) = self.detect_compilers(lang, need_cross_compiler)
+                    (comp, cross_comp) = self.environment.detect_compilers(lang, need_cross_compiler)
+                    self.environment.check_compilers(lang, comp, cross_comp)
                 except Exception:
                     if not required:
                         mlog.log('Compiler for language', mlog.bold(lang), 'not found.')
@@ -2717,26 +2688,8 @@ external dependencies (including libraries) must go to "dependencies".''')
                 mlog.log('Cross', cross_comp.get_display_language(), 'compiler:',
                          mlog.bold(' '.join(cross_comp.get_exelist())), version_string)
                 self.build.ensure_static_cross_linker(comp)
-            self.add_base_options(comp)
         return success
 
-    def emit_base_options_warnings(self, enabled_opts):
-        if 'b_bitcode' in enabled_opts:
-            mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as such as \'b_asneeded\' have been disabled.')
-            mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.')
-
-    def add_base_options(self, compiler):
-        enabled_opts = []
-        for optname in compiler.base_options:
-            if optname in self.coredata.base_options:
-                continue
-            oobj = compilers.base_options[optname]
-            if optname in self.environment.cmd_line_options:
-                oobj.set_value(self.environment.cmd_line_options[optname])
-                enabled_opts.append(optname)
-            self.coredata. base_options[optname] = oobj
-        self.emit_base_options_warnings(enabled_opts)
-
     def program_from_file_for(self, for_machine, prognames, silent):
         bins = self.environment.binaries[for_machine]
         for p in prognames:

mesonbuild/mintro.py

@@ -151,20 +151,20 @@ def list_targets(builddata: build.Build, installdata, backend: backends.Backend)
         tlist.append(t)
     return tlist
 
-class BuildoptionsOptionHelper:
+class IntrospectionHelper:
     # mimic an argparse namespace
     def __init__(self, cross_file):
         self.cross_file = cross_file
         self.native_file = None
         self.cmd_line_options = {}
 
-class BuildoptionsInterperter(astinterpreter.AstInterpreter):
+class IntrospectionInterpreter(astinterpreter.AstInterpreter):
     # Interpreter to detect the options without a build directory
     # Most of the code is stolen from interperter.Interpreter
     def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None):
         super().__init__(source_root, subdir)
 
-        options = BuildoptionsOptionHelper(cross_file)
+        options = IntrospectionHelper(cross_file)
         self.cross_file = cross_file
         if env is None:
             self.environment = environment.Environment(source_root, None, options)
@@ -176,37 +176,18 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter):
         self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt')
         self.backend = backend
         self.default_options = {'backend': self.backend}
+        self.project_data = {}
 
         self.funcs.update({
             'project': self.func_project,
             'add_languages': self.func_add_languages
         })
 
-    def detect_compilers(self, lang, need_cross_compiler):
-        comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler)
-        if comp is None:
-            return None, None
-
-        self.coredata.compilers[lang] = comp
-        # Native compiler always exist so always add its options.
-        new_options = comp.get_options()
-        if cross_comp is not None:
-            self.coredata.cross_compilers[lang] = cross_comp
-            new_options.update(cross_comp.get_options())
-
-        optprefix = lang + '_'
-        for k, o in new_options.items():
-            if not k.startswith(optprefix):
-                raise RuntimeError('Internal error, %s has incorrect prefix.' % k)
-            if k in self.environment.cmd_line_options:
-                o.set_value(self.environment.cmd_line_options[k])
-            self.coredata.compiler_options.setdefault(k, o)
-
-        return comp, cross_comp
-
     def flatten_args(self, args):
         # Resolve mparser.ArrayNode if needed
         flattend_args = []
+        if isinstance(args, mparser.ArrayNode):
+            args = [x.value for x in args.args.arguments]
         for i in args:
             if isinstance(i, mparser.ArrayNode):
                 flattend_args += [x.value for x in i.args.arguments]
@@ -216,35 +197,25 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter):
                 pass
         return flattend_args
 
-    def add_languages(self, args):
-        need_cross_compiler = self.environment.is_cross_build()
-        for lang in sorted(args, key=compilers.sort_clink):
-            lang = lang.lower()
-            if lang not in self.coredata.compilers:
-                (comp, _) = self.detect_compilers(lang, need_cross_compiler)
-                if comp is None:
-                    return
-                for optname in comp.base_options:
-                    if optname in self.coredata.base_options:
-                        continue
-                    oobj = compilers.base_options[optname]
-                    self.coredata.base_options[optname] = oobj
-
     def func_project(self, node, args, kwargs):
         if len(args) < 1:
             raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.')
 
+        proj_name = args[0]
+        proj_vers = kwargs.get('version', 'undefined')
         proj_langs = self.flatten_args(args[1:])
+        if isinstance(proj_vers, mparser.ElementaryNode):
+            proj_vers = proj_vers.value
+        if not isinstance(proj_vers, str):
+            proj_vers = 'undefined'
+        self.project_data = {'descriptive_name': proj_name, 'version': proj_vers}
 
         if os.path.exists(self.option_file):
             oi = optinterpreter.OptionInterpreter(self.subproject)
             oi.process(self.option_file)
             self.coredata.merge_user_options(oi.options)
 
-        def_opts = kwargs.get('default_options', [])
-        if isinstance(def_opts, mparser.ArrayNode):
-            def_opts = [x.value for x in def_opts.args.arguments]
-
+        def_opts = self.flatten_args(kwargs.get('default_options', []))
         self.project_default_options = mesonlib.stringlistify(def_opts)
         self.project_default_options = cdata.create_options_dict(self.project_default_options)
         self.default_options.update(self.project_default_options)
@@ -255,6 +226,7 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter):
             if isinstance(spdirname, str):
                 self.subproject_dir = spdirname
         if not self.is_subproject():
+            self.project_data['subprojects'] = []
             subprojects_dir = os.path.join(self.source_root, self.subproject_dir)
             if os.path.isdir(subprojects_dir):
                 for i in os.listdir(subprojects_dir):
@@ -265,19 +237,26 @@ class BuildoptionsInterperter(astinterpreter.AstInterpreter):
         options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')}
 
         self.coredata.set_options(options)
-        self.add_languages(proj_langs)
+        self.func_add_languages(None, proj_langs, None)
 
     def do_subproject(self, dirname):
         subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
         subpr = os.path.join(subproject_dir_abs, dirname)
         try:
-            subi = BuildoptionsInterperter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment)
+            subi = IntrospectionInterpreter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment)
             subi.analyze()
+            subi.project_data['name'] = dirname
+            self.project_data['subprojects'] += [subi.project_data]
         except:
             return
 
     def func_add_languages(self, node, args, kwargs):
-        return self.add_languages(self.flatten_args(args))
+        args = self.flatten_args(args)
+        need_cross_compiler = self.environment.is_cross_build()
+        for lang in sorted(args, key=compilers.sort_clink):
+            lang = lang.lower()
+            if lang not in self.coredata.compilers:
+                self.environment.detect_compilers(lang, need_cross_compiler)
 
     def is_subproject(self):
         return self.subproject != ''
@@ -292,7 +271,7 @@ def list_buildoptions_from_source(sourcedir, backend, indent):
     # Make sure that log entries in other parts of meson don't interfere with the JSON output
     mlog.disable()
     backend = backends.get_backend_from_name(backend, None)
-    intr = BuildoptionsInterperter(sourcedir, '', backend.name)
+    intr = IntrospectionInterpreter(sourcedir, '', backend.name)
     intr.analyze()
     # Reenable logging just in case
     mlog.enable()
@@ -438,60 +417,22 @@ def list_projinfo(builddata: build.Build):
     result['subprojects'] = subprojects
     return result
 
-class ProjectInfoInterperter(astinterpreter.AstInterpreter):
-    def __init__(self, source_root, subdir):
-        super().__init__(source_root, subdir)
-        self.funcs.update({'project': self.func_project})
-        self.project_name = None
-        self.project_version = None
-
-    def func_project(self, node, args, kwargs):
-        if len(args) < 1:
-            raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.')
-        self.project_name = args[0]
-        self.project_version = kwargs.get('version', 'undefined')
-        if isinstance(self.project_version, mparser.ElementaryNode):
-            self.project_version = self.project_version.value
-
-    def set_variable(self, varname, variable):
-        pass
-
-    def analyze(self):
-        self.load_root_meson_file()
-        self.sanity_check_ast()
-        self.parse_project()
-        self.run()
-
 def list_projinfo_from_source(sourcedir, indent):
     files = find_buildsystem_files_list(sourcedir)
+    files = [os.path.normpath(x) for x in files]
 
-    result = {'buildsystem_files': []}
-    subprojects = {}
-
-    for f in files:
-        f = f.replace('\\', '/')
-        if f == 'meson.build':
-            interpreter = ProjectInfoInterperter(sourcedir, '')
-            interpreter.analyze()
-            version = None
-            if interpreter.project_version is str:
-                version = interpreter.project_version
-            result.update({'version': version, 'descriptive_name': interpreter.project_name})
-            result['buildsystem_files'].append(f)
-        elif f.startswith('subprojects/'):
-            subproject_id = f.split('/')[1]
-            subproject = subprojects.setdefault(subproject_id, {'buildsystem_files': []})
-            subproject['buildsystem_files'].append(f)
-            if f.count('/') == 2 and f.endswith('meson.build'):
-                interpreter = ProjectInfoInterperter(os.path.join(sourcedir, 'subprojects', subproject_id), '')
-                interpreter.analyze()
-                subproject.update({'name': subproject_id, 'version': interpreter.project_version, 'descriptive_name': interpreter.project_name})
-        else:
-            result['buildsystem_files'].append(f)
+    mlog.disable()
+    intr = IntrospectionInterpreter(sourcedir, '', 'ninja')
+    intr.analyze()
+    mlog.enable()
 
-    subprojects = [obj for name, obj in subprojects.items()]
-    result['subprojects'] = subprojects
-    print(json.dumps(result, indent=indent))
+    for i in intr.project_data['subprojects']:
+        basedir = os.path.join(intr.subproject_dir, i['name'])
+        i['buildsystem_files'] = [x for x in files if x.startswith(basedir)]
+        files = [x for x in files if not x.startswith(basedir)]
+
+    intr.project_data['buildsystem_files'] = files
+    print(json.dumps(intr.project_data, indent=indent))
 
 def run(options):
     datadir = 'meson-private'
@@ -532,13 +473,8 @@ def run(options):
             return 1
 
     results = []
-    toextract = []
     intro_types = get_meson_introspection_types()
 
-    for i in intro_types.keys():
-        if options.all or getattr(options, i, False):
-            toextract += [i]
-
     # Handle the one option that does not have its own JSON file (meybe deprecate / remove this?)
     if options.target_files is not None:
         targets_file = os.path.join(infodir, 'intro-targets.json')
@@ -547,7 +483,9 @@ def run(options):
         results += [('target_files', list_target_files(options.target_files, targets, source_dir))]
 
     # Extract introspection information from JSON
-    for i in toextract:
+    for i in intro_types.keys():
+        if not options.all and not getattr(options, i, False):
+            continue
         curr = os.path.join(infodir, 'intro-{}.json'.format(i))
         if not os.path.isfile(curr):
             print('Introspection file {} does not exist.'.format(curr))

run_unittests.py

@@ -3044,21 +3044,21 @@ recommended as it is not supported on some platforms''')
         testfile = os.path.join(self.common_test_dir, '36 run program', 'meson.build')
         res = self.introspect_directory(testfile, '--projectinfo')
         self.assertEqual(set(res['buildsystem_files']), set(['meson.build']))
-        self.assertEqual(res['version'], None)
+        self.assertEqual(res['version'], 'undefined')
         self.assertEqual(res['descriptive_name'], 'run command')
         self.assertEqual(res['subprojects'], [])
 
         testfile = os.path.join(self.common_test_dir, '44 options', 'meson.build')
         res = self.introspect_directory(testfile, '--projectinfo')
         self.assertEqual(set(res['buildsystem_files']), set(['meson_options.txt', 'meson.build']))
-        self.assertEqual(res['version'], None)
+        self.assertEqual(res['version'], 'undefined')
         self.assertEqual(res['descriptive_name'], 'options')
         self.assertEqual(res['subprojects'], [])
 
         testfile = os.path.join(self.common_test_dir, '47 subproject options', 'meson.build')
         res = self.introspect_directory(testfile, '--projectinfo')
         self.assertEqual(set(res['buildsystem_files']), set(['meson_options.txt', 'meson.build']))
-        self.assertEqual(res['version'], None)
+        self.assertEqual(res['version'], 'undefined')
         self.assertEqual(res['descriptive_name'], 'suboptions')
         self.assertEqual(len(res['subprojects']), 1)
         subproject_files = set(f.replace('\\', '/') for f in res['subprojects'][0]['buildsystem_files'])

test cases/unit/51 introspect buildoptions/subprojects/projectBad/meson.build

@@ -0,0 +1,9 @@
+pfggggaergaeg(sdgrgjgn)aga
+
+rgqeh
+th
+thtr
+e
+tb
+tbqebt
+tbqebttrtt

test cases/unit/51 introspect buildoptions/subprojects/projectBad/meson_options.txt

@@ -0,0 +1 @@
+option('should_not_appear', type: 'integer', min: 0, value: 125)