Merge pull request #4564 from mensinda/introBuildOpts

mintro: Introspect --buildoptions without a build directory

docs/markdown/IDE-integration.md

@@ -70,6 +70,17 @@ The possible values for `section` are:
 
 To set the options, use the `meson configure` command.
 
+Since Meson 0.50.0 it is also possible to get the default buildoptions
+without a build directory by providing the root `meson.build` instead of a
+build directory to `meson introspect --buildoptions`.
+
+Running `--buildoptions` without a build directory produces the same output as running
+it with a freshly configured build directory.
+
+However, this behavior is not guaranteed if subprojects are present. Due to internal
+limitations all subprojects are processed even if they are never used in a real meson run.
+Because of this options for the subprojects can differ.
+
 ## Tests
 
 Compilation and unit tests are done as usual by running the `ninja` and `ninja test` commands. A JSON formatted result log can be found in `workspace/project/builddir/meson-logs/testlog.json`.

docs/markdown/snippets/introspect_buildoptions_no_bd.md

@@ -0,0 +1,11 @@
+## `introspect --buildoptions` can now be used without configured build directory
+
+It is now possible to run `meson introspect --buildoptions /path/to/meson.build`
+without a configured build directory.
+
+Running `--buildoptions` without a build directory produces the same output as running
+it with a freshly configured build directory.
+
+However, this behavior is not guaranteed if subprojects are present. Due to internal
+limitations all subprojects are processed even if they are never used in a real meson run.
+Because of this options for the subprojects can differ.

mesonbuild/backend/backends.py

@@ -124,10 +124,35 @@ class OptionOverrideProxy:
             return OptionProxy(base_opt.name, base_opt.validate_value(self.overrides[option_name]))
         return base_opt
 
+def get_backend_from_name(backend, build):
+    if backend == 'ninja':
+        from . import ninjabackend
+        return ninjabackend.NinjaBackend(build)
+    elif backend == 'vs':
+        from . import vs2010backend
+        return vs2010backend.autodetect_vs_version(build)
+    elif backend == 'vs2010':
+        from . import vs2010backend
+        return vs2010backend.Vs2010Backend(build)
+    elif backend == 'vs2015':
+        from . import vs2015backend
+        return vs2015backend.Vs2015Backend(build)
+    elif backend == 'vs2017':
+        from . import vs2017backend
+        return vs2017backend.Vs2017Backend(build)
+    elif backend == 'xcode':
+        from . import xcodebackend
+        return xcodebackend.XCodeBackend(build)
+    return None
+
 # This class contains the basic functionality that is needed by all backends.
 # Feel free to move stuff in and out of it as you see fit.
 class Backend:
     def __init__(self, build):
+        # Make it possible to construct a dummy backend
+        # This is used for introspection without a build directory
+        if build is None:
+            return
         self.build = build
         self.environment = build.environment
         self.processed_targets = {}

mesonbuild/coredata.py

@@ -551,6 +551,36 @@ class CoreData:
             sub = 'In subproject {}: '.format(subproject) if subproject else ''
             mlog.warning('{}Unknown options: "{}"'.format(sub, unknown_options))
 
+    def set_default_options(self, default_options, subproject, cmd_line_options):
+        # Set default options as if they were passed to the command line.
+        # Subprojects can only define default for user options.
+        from . import optinterpreter
+        for k, v in default_options.items():
+            if subproject:
+                if optinterpreter.is_invalid_name(k):
+                    continue
+                k = subproject + ':' + k
+            cmd_line_options.setdefault(k, v)
+
+        # Create a subset of cmd_line_options, keeping only options for this
+        # subproject. Also take builtin options if it's the main project.
+        # Language and backend specific options will be set later when adding
+        # languages and setting the backend (builtin options must be set first
+        # to know which backend we'll use).
+        options = {}
+        for k, v in cmd_line_options.items():
+            if subproject:
+                if not k.startswith(subproject + ':'):
+                    continue
+            elif k not in get_builtin_options():
+                if ':' in k:
+                    continue
+                if optinterpreter.is_invalid_name(k):
+                    continue
+            options[k] = v
+
+        self.set_options(options, subproject)
+
 class CmdLineFileParser(configparser.ConfigParser):
     def __init__(self):
         # We don't want ':' as key delimiter, otherwise it would break when

mesonbuild/environment.py

@@ -310,6 +310,10 @@ class Environment:
     def __init__(self, source_dir, build_dir, options):
         self.source_dir = source_dir
         self.build_dir = build_dir
+
+        # Do not try to create build directories when build_dir is none.
+        # This reduced mode is used by the --buildoptions introspector
+        if build_dir is not None:
             self.scratch_dir = os.path.join(build_dir, Environment.private_dir)
             self.log_dir = os.path.join(build_dir, Environment.log_dir)
             os.makedirs(self.scratch_dir, exist_ok=True)
@@ -329,6 +333,9 @@ class Environment:
                     self.create_new_coredata(options)
                 else:
                     raise e
+        else:
+            # Just create a fresh coredata in this case
+            self.create_new_coredata(options)
         self.exe_wrapper = None
 
         self.machines = MachineInfos()
@@ -954,6 +961,57 @@ class Environment:
             return compilers.SwiftCompiler(exelist, version)
         raise EnvironmentException('Unknown compiler "' + ' '.join(exelist) + '"')
 
+    def detect_compilers(self, lang, need_cross_compiler):
+        comp = None
+        cross_comp = None
+        if lang == 'c':
+            comp = self.detect_c_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_c_compiler(True)
+        elif lang == 'cpp':
+            comp = self.detect_cpp_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_cpp_compiler(True)
+        elif lang == 'objc':
+            comp = self.detect_objc_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_objc_compiler(True)
+        elif lang == 'objcpp':
+            comp = self.detect_objcpp_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_objcpp_compiler(True)
+        elif lang == 'java':
+            comp = self.detect_java_compiler()
+            if need_cross_compiler:
+                cross_comp = comp  # Java is platform independent.
+        elif lang == 'cs':
+            comp = self.detect_cs_compiler()
+            if need_cross_compiler:
+                cross_comp = comp  # C# is platform independent.
+        elif lang == 'vala':
+            comp = self.detect_vala_compiler()
+            if need_cross_compiler:
+                cross_comp = comp  # Vala compiles to platform-independent C
+        elif lang == 'd':
+            comp = self.detect_d_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_d_compiler(True)
+        elif lang == 'rust':
+            comp = self.detect_rust_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_rust_compiler(True)
+        elif lang == 'fortran':
+            comp = self.detect_fortran_compiler(False)
+            if need_cross_compiler:
+                cross_comp = self.detect_fortran_compiler(True)
+        elif lang == 'swift':
+            comp = self.detect_swift_compiler()
+            if need_cross_compiler:
+                raise EnvironmentException('Cross compilation with Swift is not working yet.')
+                # cross_comp = self.environment.detect_fortran_compiler(True)
+
+        return comp, cross_comp
+
     def detect_static_linker(self, compiler):
         if compiler.is_cross:
             linker = self.cross_info.config['binaries']['ar']

mesonbuild/interpreter.py

@@ -2504,62 +2504,20 @@ external dependencies (including libraries) must go to "dependencies".''')
             cdata.set_method([k, v], {})
         return cdata
 
-    def set_options(self, default_options):
-        # Set default options as if they were passed to the command line.
-        # Subprojects can only define default for user options.
-        for k, v in default_options.items():
-            if self.subproject:
-                if optinterpreter.is_invalid_name(k):
-                    continue
-                k = self.subproject + ':' + k
-            self.environment.cmd_line_options.setdefault(k, v)
-
-        # Create a subset of cmd_line_options, keeping only options for this
-        # subproject. Also take builtin options if it's the main project.
-        # Language and backend specific options will be set later when adding
-        # languages and setting the backend (builtin options must be set first
-        # to know which backend we'll use).
-        options = {}
-        for k, v in self.environment.cmd_line_options.items():
-            if self.subproject:
-                if not k.startswith(self.subproject + ':'):
-                    continue
-            elif k not in coredata.get_builtin_options():
-                if ':' in k:
-                    continue
-                if optinterpreter.is_invalid_name(k):
-                    continue
-            options[k] = v
-
-        self.coredata.set_options(options, self.subproject)
-
     def set_backend(self):
         # The backend is already set when parsing subprojects
         if self.backend is not None:
             return
         backend = self.coredata.get_builtin_option('backend')
-        if backend == 'ninja':
+        from .backend import backends
-            from .backend import ninjabackend
+        self.backend = backends.get_backend_from_name(backend, self.build)
-            self.backend = ninjabackend.NinjaBackend(self.build)
+
-        elif backend == 'vs':
+        if self.backend is None:
-            from .backend import vs2010backend
-            self.backend = vs2010backend.autodetect_vs_version(self.build)
-            self.coredata.set_builtin_option('backend', self.backend.name)
-            mlog.log('Auto detected Visual Studio backend:', mlog.bold(self.backend.name))
-        elif backend == 'vs2010':
-            from .backend import vs2010backend
-            self.backend = vs2010backend.Vs2010Backend(self.build)
-        elif backend == 'vs2015':
-            from .backend import vs2015backend
-            self.backend = vs2015backend.Vs2015Backend(self.build)
-        elif backend == 'vs2017':
-            from .backend import vs2017backend
-            self.backend = vs2017backend.Vs2017Backend(self.build)
-        elif backend == 'xcode':
-            from .backend import xcodebackend
-            self.backend = xcodebackend.XCodeBackend(self.build)
-        else:
             raise InterpreterException('Unknown backend "%s".' % backend)
+        if backend != self.backend.name:
+            if self.backend.name.startswith('vs'):
+                mlog.log('Auto detected Visual Studio backend:', mlog.bold(self.backend.name))
+            self.coredata.set_builtin_option('backend', self.backend.name)
 
         # Only init backend options on first invocation otherwise it would
         # override values previously set from command line.
@@ -2595,7 +2553,7 @@ external dependencies (including libraries) must go to "dependencies".''')
             default_options.update(self.default_project_options)
         else:
             default_options = {}
-        self.set_options(default_options)
+        self.coredata.set_default_options(default_options, self.subproject, self.environment.cmd_line_options)
         self.set_backend()
 
         if not self.is_subproject():
@@ -2696,54 +2654,10 @@ external dependencies (including libraries) must go to "dependencies".''')
         raise Exception()
 
     def detect_compilers(self, lang, need_cross_compiler):
-        cross_comp = None
+        comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler)
-        if lang == 'c':
+        if comp is None:
-            comp = self.environment.detect_c_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_c_compiler(True)
-        elif lang == 'cpp':
-            comp = self.environment.detect_cpp_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_cpp_compiler(True)
-        elif lang == 'objc':
-            comp = self.environment.detect_objc_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_objc_compiler(True)
-        elif lang == 'objcpp':
-            comp = self.environment.detect_objcpp_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_objcpp_compiler(True)
-        elif lang == 'java':
-            comp = self.environment.detect_java_compiler()
-            if need_cross_compiler:
-                cross_comp = comp  # Java is platform independent.
-        elif lang == 'cs':
-            comp = self.environment.detect_cs_compiler()
-            if need_cross_compiler:
-                cross_comp = comp  # C# is platform independent.
-        elif lang == 'vala':
-            comp = self.environment.detect_vala_compiler()
-            if need_cross_compiler:
-                cross_comp = comp  # Vala compiles to platform-independent C
-        elif lang == 'd':
-            comp = self.environment.detect_d_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_d_compiler(True)
-        elif lang == 'rust':
-            comp = self.environment.detect_rust_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_rust_compiler(True)
-        elif lang == 'fortran':
-            comp = self.environment.detect_fortran_compiler(False)
-            if need_cross_compiler:
-                cross_comp = self.environment.detect_fortran_compiler(True)
-        elif lang == 'swift':
-            comp = self.environment.detect_swift_compiler()
-            if need_cross_compiler:
-                raise InterpreterException('Cross compilation with Swift is not working yet.')
-                # cross_comp = self.environment.detect_fortran_compiler(True)
-        else:
             raise InvalidCode('Tried to use unknown language "%s".' % lang)
+
         comp.sanity_check(self.environment.get_scratch_dir(), self.environment)
         self.coredata.compilers[lang] = comp
         # Native compiler always exist so always add its options.

mesonbuild/mintro.py

@@ -21,11 +21,15 @@ project files and don't need this info."""
 
 import json
 from . import build, mtest, coredata as cdata
+from . import environment
 from . import mesonlib
 from . import astinterpreter
 from . import mparser
+from . import mlog
+from . import compilers
+from . import optinterpreter
 from .interpreterbase import InvalidArguments
-from .backend import ninjabackend
+from .backend import ninjabackend, backends
 import sys, os
 import pathlib
 
@@ -48,6 +52,8 @@ def add_arguments(parser):
                         help='List external dependencies.')
     parser.add_argument('--projectinfo', action='store_true', dest='projectinfo', default=False,
                         help='Information about projects.')
+    parser.add_argument('--backend', choices=cdata.backendlist, dest='backend', default='ninja',
+                        help='The backend to use for the --buildoptions introspection.')
     parser.add_argument('builddir', nargs='?', default='.', help='The build directory')
 
 def determine_installed_path(target, installdata):
@@ -129,7 +135,154 @@ def list_target_files(target_name, coredata, builddata):
         out.append(i)
     print(json.dumps(out))
 
-def list_buildoptions(coredata, builddata):
+class BuildoptionsOptionHelper:
+    # mimic an argparse namespace
+    def __init__(self, cross_file):
+        self.cross_file = cross_file
+        self.native_file = None
+        self.cmd_line_options = {}
+
+class BuildoptionsInterperter(astinterpreter.AstInterpreter):
+    # Interpreter to detect the options without a build directory
+    # Most of the code is stolen from interperter.Interpreter
+    def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None):
+        super().__init__(source_root, subdir)
+
+        options = BuildoptionsOptionHelper(cross_file)
+        self.cross_file = cross_file
+        if env is None:
+            self.environment = environment.Environment(source_root, None, options)
+        else:
+            self.environment = env
+        self.subproject = subproject
+        self.subproject_dir = subproject_dir
+        self.coredata = self.environment.get_coredata()
+        self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt')
+        self.backend = backend
+        self.default_options = {'backend': self.backend}
+
+        self.funcs.update({
+            'project': self.func_project,
+            'add_languages': self.func_add_languages
+        })
+
+    def detect_compilers(self, lang, need_cross_compiler):
+        comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler)
+        if comp is None:
+            return None, None
+
+        self.coredata.compilers[lang] = comp
+        # Native compiler always exist so always add its options.
+        new_options = comp.get_options()
+        if cross_comp is not None:
+            self.coredata.cross_compilers[lang] = cross_comp
+            new_options.update(cross_comp.get_options())
+
+        optprefix = lang + '_'
+        for k, o in new_options.items():
+            if not k.startswith(optprefix):
+                raise RuntimeError('Internal error, %s has incorrect prefix.' % k)
+            if k in self.environment.cmd_line_options:
+                o.set_value(self.environment.cmd_line_options[k])
+            self.coredata.compiler_options.setdefault(k, o)
+
+        return comp, cross_comp
+
+    def flatten_args(self, args):
+        # Resolve mparser.ArrayNode if needed
+        flattend_args = []
+        for i in args:
+            if isinstance(i, mparser.ArrayNode):
+                flattend_args += [x.value for x in i.args.arguments]
+            elif isinstance(i, str):
+                flattend_args += [i]
+            else:
+                pass
+        return flattend_args
+
+    def add_languages(self, args):
+        need_cross_compiler = self.environment.is_cross_build() and self.environment.cross_info.need_cross_compiler()
+        for lang in sorted(args, key=compilers.sort_clink):
+            lang = lang.lower()
+            if lang not in self.coredata.compilers:
+                (comp, _) = self.detect_compilers(lang, need_cross_compiler)
+                if comp is None:
+                    return
+                for optname in comp.base_options:
+                    if optname in self.coredata.base_options:
+                        continue
+                    oobj = compilers.base_options[optname]
+                    self.coredata.base_options[optname] = oobj
+
+    def func_project(self, node, args, kwargs):
+        if len(args) < 1:
+            raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.')
+
+        proj_langs = self.flatten_args(args[1:])
+
+        if os.path.exists(self.option_file):
+            oi = optinterpreter.OptionInterpreter(self.subproject)
+            oi.process(self.option_file)
+            self.coredata.merge_user_options(oi.options)
+
+        def_opts = kwargs.get('default_options', [])
+        if isinstance(def_opts, mparser.ArrayNode):
+            def_opts = [x.value for x in def_opts.args.arguments]
+
+        self.project_default_options = mesonlib.stringlistify(def_opts)
+        self.project_default_options = cdata.create_options_dict(self.project_default_options)
+        self.default_options.update(self.project_default_options)
+        self.coredata.set_default_options(self.default_options, self.subproject, self.environment.cmd_line_options)
+
+        if not self.is_subproject() and 'subproject_dir' in kwargs:
+            spdirname = kwargs['subproject_dir']
+            if isinstance(spdirname, str):
+                self.subproject_dir = spdirname
+        if not self.is_subproject():
+            subprojects_dir = os.path.join(self.source_root, self.subproject_dir)
+            if os.path.isdir(subprojects_dir):
+                for i in os.listdir(subprojects_dir):
+                    if os.path.isdir(os.path.join(subprojects_dir, i)):
+                        self.do_subproject(i)
+
+        self.coredata.init_backend_options(self.backend)
+        options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')}
+
+        self.coredata.set_options(options)
+        self.add_languages(proj_langs)
+
+    def do_subproject(self, dirname):
+        subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
+        subpr = os.path.join(subproject_dir_abs, dirname)
+        try:
+            subi = BuildoptionsInterperter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment)
+            subi.analyze()
+        except:
+            return
+
+    def func_add_languages(self, node, args, kwargs):
+        return self.add_languages(self.flatten_args(args))
+
+    def is_subproject(self):
+        return self.subproject != ''
+
+    def analyze(self):
+        self.load_root_meson_file()
+        self.sanity_check_ast()
+        self.parse_project()
+        self.run()
+
+def list_buildoptions_from_source(sourcedir, backend):
+    # Make sure that log entries in other parts of meson don't interfere with the JSON output
+    mlog.disable()
+    backend = backends.get_backend_from_name(backend, None)
+    intr = BuildoptionsInterperter(sourcedir, '', backend.name)
+    intr.analyze()
+    # Reenable logging just in case
+    mlog.enable()
+    list_buildoptions(intr.coredata)
+
+def list_buildoptions(coredata):
     optlist = []
 
     dir_option_names = ['bindir',
@@ -301,10 +454,13 @@ def run(options):
     if options.builddir is not None:
         datadir = os.path.join(options.builddir, datadir)
     if options.builddir.endswith('/meson.build') or options.builddir.endswith('\\meson.build') or options.builddir == 'meson.build':
-        if options.projectinfo:
         sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
+        if options.projectinfo:
             list_projinfo_from_source(sourcedir)
             return 0
+        if options.buildoptions:
+            list_buildoptions_from_source(sourcedir, options.backend)
+            return 0
     if not os.path.isdir(datadir):
         print('Current directory is not a build dir. Please specify it or '
               'change the working directory to it.')
@@ -330,7 +486,7 @@ def run(options):
     elif options.buildsystem_files:
         list_buildsystem_files(builddata)
     elif options.buildoptions:
-        list_buildoptions(coredata, builddata)
+        list_buildoptions(coredata)
     elif options.tests:
         list_tests(testdata)
     elif options.benchmarks:

mesonbuild/mlog.py

@@ -47,6 +47,15 @@ log_fname = 'meson-log.txt'
 log_depth = 0
 log_timestamp_start = None
 log_fatal_warnings = False
+log_disable_stdout = False
+
+def disable():
+    global log_disable_stdout
+    log_disable_stdout = True
+
+def enable():
+    global log_disable_stdout
+    log_disable_stdout = False
 
 def initialize(logdir, fatal_warnings=False):
     global log_dir, log_file, log_fatal_warnings
@@ -118,6 +127,9 @@ def process_markup(args, keep):
     return arr
 
 def force_print(*args, **kwargs):
+    global log_disable_stdout
+    if log_disable_stdout:
+        return
     iostr = io.StringIO()
     kwargs['file'] = iostr
     print(*args, **kwargs)

run_unittests.py

@@ -1199,7 +1199,12 @@ class BasePlatformTests(unittest.TestCase):
             args = [args]
         out = subprocess.check_output(self.mintro_command + args + [directory],
                                       universal_newlines=True)
-        return json.loads(out)
+        try:
+            obj = json.loads(out)
+        except Exception as e:
+            print(out)
+            raise e
+        return obj
 
     def assertPathEqual(self, path1, path2):
         '''
@@ -3089,6 +3094,16 @@ recommended as it is not supported on some platforms''')
         self.assertEqual(Path(testfile).read_text(),
                          Path(goodfile).read_text())
 
+    def test_introspect_buildoptions_without_configured_build(self):
+        testdir = os.path.join(self.unit_test_dir, '51 introspect buildoptions')
+        testfile = os.path.join(testdir, 'meson.build')
+        res_nb = self.introspect_directory(testfile, ['--buildoptions'] + self.meson_args)
+        self.init(testdir, default_args=False)
+        res_wb = self.introspect('--buildoptions')
+        self.maxDiff = None
+        self.assertListEqual(res_nb, res_wb)
+
+
 class FailureTests(BasePlatformTests):
     '''
     Tests that test failure conditions. Build files here should be dynamically

test cases/unit/51 introspect buildoptions/c_compiler.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+
+print('c')

test cases/unit/51 introspect buildoptions/meson.build

@@ -0,0 +1,11 @@
+project('introspect buildargs', ['c'], default_options: ['c_std=c11', 'cpp_std=c++14', 'buildtype=release'])
+
+subA = subproject('projectA')
+
+r = run_command(find_program('c_compiler.py'))
+if r.returncode() != 0
+  error('FAILED')
+endif
+
+add_languages(r.stdout().strip(), required: true)
+add_languages('afgggergearvearghergervergreaergaergasv', required: false)

test cases/unit/51 introspect buildoptions/meson_options.txt

@@ -0,0 +1,2 @@
+option('max_register_count', type: 'integer', min: 0, value: 125)
+option('use_external_fmt',   type: 'boolean', value: false)

test cases/unit/51 introspect buildoptions/subprojects/projectA/meson.build

@@ -0,0 +1,3 @@
+project('introspect subproject A', 'c', default_options: ['cpp_std=c++11', 'buildtype=debug'])
+
+add_languages('cpp')

test cases/unit/51 introspect buildoptions/subprojects/projectA/meson_options.txt

@@ -0,0 +1 @@
+option('subproj_var', type: 'boolean', value: false)

test cases/unit/51 introspect buildoptions/subprojects/projectBad/meson.build

@@ -0,0 +1,9 @@
+pfggggaergaeg(sdgrgjgn)aga
+
+rgqeh
+th
+thtr
+e
+tb
+tbqebt
+tbqebttrtt

test cases/unit/51 introspect buildoptions/subprojects/projectBad/meson_options.txt

@@ -0,0 +1 @@
+option('should_not_appear', type: 'integer', min: 0, value: 125)