Merge pull request #4564 from mensinda/introBuildOpts

mintro: Introspect --buildoptions without a build directory
pull/4699/head
Jussi Pakkanen 6 years ago committed by GitHub
commit 8c9fcb1fea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 11
      docs/markdown/IDE-integration.md
  2. 11
      docs/markdown/snippets/introspect_buildoptions_no_bd.md
  3. 25
      mesonbuild/backend/backends.py
  4. 30
      mesonbuild/coredata.py
  5. 94
      mesonbuild/environment.py
  6. 110
      mesonbuild/interpreter.py
  7. 164
      mesonbuild/mintro.py
  8. 12
      mesonbuild/mlog.py
  9. 17
      run_unittests.py
  10. 3
      test cases/unit/51 introspect buildoptions/c_compiler.py
  11. 11
      test cases/unit/51 introspect buildoptions/meson.build
  12. 2
      test cases/unit/51 introspect buildoptions/meson_options.txt
  13. 0
      test cases/unit/51 introspect buildoptions/subprojects/evilFile.txt
  14. 3
      test cases/unit/51 introspect buildoptions/subprojects/projectA/meson.build
  15. 1
      test cases/unit/51 introspect buildoptions/subprojects/projectA/meson_options.txt
  16. 9
      test cases/unit/51 introspect buildoptions/subprojects/projectBad/meson.build
  17. 1
      test cases/unit/51 introspect buildoptions/subprojects/projectBad/meson_options.txt

@ -70,6 +70,17 @@ The possible values for `section` are:
To set the options, use the `meson configure` command.
Since Meson 0.50.0 it is also possible to get the default buildoptions
without a build directory by providing the root `meson.build` instead of a
build directory to `meson introspect --buildoptions`.
Running `--buildoptions` without a build directory produces the same output as running
it with a freshly configured build directory.
However, this behavior is not guaranteed if subprojects are present. Due to internal
limitations all subprojects are processed even if they are never used in a real meson run.
Because of this options for the subprojects can differ.
## Tests
Compilation and unit tests are done as usual by running the `ninja` and `ninja test` commands. A JSON formatted result log can be found in `workspace/project/builddir/meson-logs/testlog.json`.

@ -0,0 +1,11 @@
## `introspect --buildoptions` can now be used without configured build directory
It is now possible to run `meson introspect --buildoptions /path/to/meson.build`
without a configured build directory.
Running `--buildoptions` without a build directory produces the same output as running
it with a freshly configured build directory.
However, this behavior is not guaranteed if subprojects are present. Due to internal
limitations all subprojects are processed even if they are never used in a real meson run.
Because of this options for the subprojects can differ.

@ -124,10 +124,35 @@ class OptionOverrideProxy:
return OptionProxy(base_opt.name, base_opt.validate_value(self.overrides[option_name]))
return base_opt
def get_backend_from_name(backend, build):
if backend == 'ninja':
from . import ninjabackend
return ninjabackend.NinjaBackend(build)
elif backend == 'vs':
from . import vs2010backend
return vs2010backend.autodetect_vs_version(build)
elif backend == 'vs2010':
from . import vs2010backend
return vs2010backend.Vs2010Backend(build)
elif backend == 'vs2015':
from . import vs2015backend
return vs2015backend.Vs2015Backend(build)
elif backend == 'vs2017':
from . import vs2017backend
return vs2017backend.Vs2017Backend(build)
elif backend == 'xcode':
from . import xcodebackend
return xcodebackend.XCodeBackend(build)
return None
# This class contains the basic functionality that is needed by all backends.
# Feel free to move stuff in and out of it as you see fit.
class Backend:
def __init__(self, build):
# Make it possible to construct a dummy backend
# This is used for introspection without a build directory
if build is None:
return
self.build = build
self.environment = build.environment
self.processed_targets = {}

@ -551,6 +551,36 @@ class CoreData:
sub = 'In subproject {}: '.format(subproject) if subproject else ''
mlog.warning('{}Unknown options: "{}"'.format(sub, unknown_options))
def set_default_options(self, default_options, subproject, cmd_line_options):
# Set default options as if they were passed to the command line.
# Subprojects can only define default for user options.
from . import optinterpreter
for k, v in default_options.items():
if subproject:
if optinterpreter.is_invalid_name(k):
continue
k = subproject + ':' + k
cmd_line_options.setdefault(k, v)
# Create a subset of cmd_line_options, keeping only options for this
# subproject. Also take builtin options if it's the main project.
# Language and backend specific options will be set later when adding
# languages and setting the backend (builtin options must be set first
# to know which backend we'll use).
options = {}
for k, v in cmd_line_options.items():
if subproject:
if not k.startswith(subproject + ':'):
continue
elif k not in get_builtin_options():
if ':' in k:
continue
if optinterpreter.is_invalid_name(k):
continue
options[k] = v
self.set_options(options, subproject)
class CmdLineFileParser(configparser.ConfigParser):
def __init__(self):
# We don't want ':' as key delimiter, otherwise it would break when

@ -310,25 +310,32 @@ class Environment:
def __init__(self, source_dir, build_dir, options):
self.source_dir = source_dir
self.build_dir = build_dir
self.scratch_dir = os.path.join(build_dir, Environment.private_dir)
self.log_dir = os.path.join(build_dir, Environment.log_dir)
os.makedirs(self.scratch_dir, exist_ok=True)
os.makedirs(self.log_dir, exist_ok=True)
try:
self.coredata = coredata.load(self.get_build_dir())
self.first_invocation = False
except FileNotFoundError:
self.create_new_coredata(options)
except MesonException as e:
# If we stored previous command line options, we can recover from
# a broken/outdated coredata.
if os.path.isfile(coredata.get_cmd_line_file(self.build_dir)):
mlog.warning('Regenerating configuration from scratch.')
mlog.log('Reason:', mlog.red(str(e)))
coredata.read_cmd_line_file(self.build_dir, options)
# Do not try to create build directories when build_dir is none.
# This reduced mode is used by the --buildoptions introspector
if build_dir is not None:
self.scratch_dir = os.path.join(build_dir, Environment.private_dir)
self.log_dir = os.path.join(build_dir, Environment.log_dir)
os.makedirs(self.scratch_dir, exist_ok=True)
os.makedirs(self.log_dir, exist_ok=True)
try:
self.coredata = coredata.load(self.get_build_dir())
self.first_invocation = False
except FileNotFoundError:
self.create_new_coredata(options)
else:
raise e
except MesonException as e:
# If we stored previous command line options, we can recover from
# a broken/outdated coredata.
if os.path.isfile(coredata.get_cmd_line_file(self.build_dir)):
mlog.warning('Regenerating configuration from scratch.')
mlog.log('Reason:', mlog.red(str(e)))
coredata.read_cmd_line_file(self.build_dir, options)
self.create_new_coredata(options)
else:
raise e
else:
# Just create a fresh coredata in this case
self.create_new_coredata(options)
self.exe_wrapper = None
self.machines = MachineInfos()
@ -954,6 +961,57 @@ class Environment:
return compilers.SwiftCompiler(exelist, version)
raise EnvironmentException('Unknown compiler "' + ' '.join(exelist) + '"')
def detect_compilers(self, lang, need_cross_compiler):
comp = None
cross_comp = None
if lang == 'c':
comp = self.detect_c_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_c_compiler(True)
elif lang == 'cpp':
comp = self.detect_cpp_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_cpp_compiler(True)
elif lang == 'objc':
comp = self.detect_objc_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_objc_compiler(True)
elif lang == 'objcpp':
comp = self.detect_objcpp_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_objcpp_compiler(True)
elif lang == 'java':
comp = self.detect_java_compiler()
if need_cross_compiler:
cross_comp = comp # Java is platform independent.
elif lang == 'cs':
comp = self.detect_cs_compiler()
if need_cross_compiler:
cross_comp = comp # C# is platform independent.
elif lang == 'vala':
comp = self.detect_vala_compiler()
if need_cross_compiler:
cross_comp = comp # Vala compiles to platform-independent C
elif lang == 'd':
comp = self.detect_d_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_d_compiler(True)
elif lang == 'rust':
comp = self.detect_rust_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_rust_compiler(True)
elif lang == 'fortran':
comp = self.detect_fortran_compiler(False)
if need_cross_compiler:
cross_comp = self.detect_fortran_compiler(True)
elif lang == 'swift':
comp = self.detect_swift_compiler()
if need_cross_compiler:
raise EnvironmentException('Cross compilation with Swift is not working yet.')
# cross_comp = self.environment.detect_fortran_compiler(True)
return comp, cross_comp
def detect_static_linker(self, compiler):
if compiler.is_cross:
linker = self.cross_info.config['binaries']['ar']

@ -2504,62 +2504,20 @@ external dependencies (including libraries) must go to "dependencies".''')
cdata.set_method([k, v], {})
return cdata
def set_options(self, default_options):
# Set default options as if they were passed to the command line.
# Subprojects can only define default for user options.
for k, v in default_options.items():
if self.subproject:
if optinterpreter.is_invalid_name(k):
continue
k = self.subproject + ':' + k
self.environment.cmd_line_options.setdefault(k, v)
# Create a subset of cmd_line_options, keeping only options for this
# subproject. Also take builtin options if it's the main project.
# Language and backend specific options will be set later when adding
# languages and setting the backend (builtin options must be set first
# to know which backend we'll use).
options = {}
for k, v in self.environment.cmd_line_options.items():
if self.subproject:
if not k.startswith(self.subproject + ':'):
continue
elif k not in coredata.get_builtin_options():
if ':' in k:
continue
if optinterpreter.is_invalid_name(k):
continue
options[k] = v
self.coredata.set_options(options, self.subproject)
def set_backend(self):
# The backend is already set when parsing subprojects
if self.backend is not None:
return
backend = self.coredata.get_builtin_option('backend')
if backend == 'ninja':
from .backend import ninjabackend
self.backend = ninjabackend.NinjaBackend(self.build)
elif backend == 'vs':
from .backend import vs2010backend
self.backend = vs2010backend.autodetect_vs_version(self.build)
self.coredata.set_builtin_option('backend', self.backend.name)
mlog.log('Auto detected Visual Studio backend:', mlog.bold(self.backend.name))
elif backend == 'vs2010':
from .backend import vs2010backend
self.backend = vs2010backend.Vs2010Backend(self.build)
elif backend == 'vs2015':
from .backend import vs2015backend
self.backend = vs2015backend.Vs2015Backend(self.build)
elif backend == 'vs2017':
from .backend import vs2017backend
self.backend = vs2017backend.Vs2017Backend(self.build)
elif backend == 'xcode':
from .backend import xcodebackend
self.backend = xcodebackend.XCodeBackend(self.build)
else:
from .backend import backends
self.backend = backends.get_backend_from_name(backend, self.build)
if self.backend is None:
raise InterpreterException('Unknown backend "%s".' % backend)
if backend != self.backend.name:
if self.backend.name.startswith('vs'):
mlog.log('Auto detected Visual Studio backend:', mlog.bold(self.backend.name))
self.coredata.set_builtin_option('backend', self.backend.name)
# Only init backend options on first invocation otherwise it would
# override values previously set from command line.
@ -2595,7 +2553,7 @@ external dependencies (including libraries) must go to "dependencies".''')
default_options.update(self.default_project_options)
else:
default_options = {}
self.set_options(default_options)
self.coredata.set_default_options(default_options, self.subproject, self.environment.cmd_line_options)
self.set_backend()
if not self.is_subproject():
@ -2696,54 +2654,10 @@ external dependencies (including libraries) must go to "dependencies".''')
raise Exception()
def detect_compilers(self, lang, need_cross_compiler):
cross_comp = None
if lang == 'c':
comp = self.environment.detect_c_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_c_compiler(True)
elif lang == 'cpp':
comp = self.environment.detect_cpp_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_cpp_compiler(True)
elif lang == 'objc':
comp = self.environment.detect_objc_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_objc_compiler(True)
elif lang == 'objcpp':
comp = self.environment.detect_objcpp_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_objcpp_compiler(True)
elif lang == 'java':
comp = self.environment.detect_java_compiler()
if need_cross_compiler:
cross_comp = comp # Java is platform independent.
elif lang == 'cs':
comp = self.environment.detect_cs_compiler()
if need_cross_compiler:
cross_comp = comp # C# is platform independent.
elif lang == 'vala':
comp = self.environment.detect_vala_compiler()
if need_cross_compiler:
cross_comp = comp # Vala compiles to platform-independent C
elif lang == 'd':
comp = self.environment.detect_d_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_d_compiler(True)
elif lang == 'rust':
comp = self.environment.detect_rust_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_rust_compiler(True)
elif lang == 'fortran':
comp = self.environment.detect_fortran_compiler(False)
if need_cross_compiler:
cross_comp = self.environment.detect_fortran_compiler(True)
elif lang == 'swift':
comp = self.environment.detect_swift_compiler()
if need_cross_compiler:
raise InterpreterException('Cross compilation with Swift is not working yet.')
# cross_comp = self.environment.detect_fortran_compiler(True)
else:
comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler)
if comp is None:
raise InvalidCode('Tried to use unknown language "%s".' % lang)
comp.sanity_check(self.environment.get_scratch_dir(), self.environment)
self.coredata.compilers[lang] = comp
# Native compiler always exist so always add its options.

@ -21,11 +21,15 @@ project files and don't need this info."""
import json
from . import build, mtest, coredata as cdata
from . import environment
from . import mesonlib
from . import astinterpreter
from . import mparser
from . import mlog
from . import compilers
from . import optinterpreter
from .interpreterbase import InvalidArguments
from .backend import ninjabackend
from .backend import ninjabackend, backends
import sys, os
import pathlib
@ -48,6 +52,8 @@ def add_arguments(parser):
help='List external dependencies.')
parser.add_argument('--projectinfo', action='store_true', dest='projectinfo', default=False,
help='Information about projects.')
parser.add_argument('--backend', choices=cdata.backendlist, dest='backend', default='ninja',
help='The backend to use for the --buildoptions introspection.')
parser.add_argument('builddir', nargs='?', default='.', help='The build directory')
def determine_installed_path(target, installdata):
@ -129,7 +135,154 @@ def list_target_files(target_name, coredata, builddata):
out.append(i)
print(json.dumps(out))
def list_buildoptions(coredata, builddata):
class BuildoptionsOptionHelper:
# mimic an argparse namespace
def __init__(self, cross_file):
self.cross_file = cross_file
self.native_file = None
self.cmd_line_options = {}
class BuildoptionsInterperter(astinterpreter.AstInterpreter):
# Interpreter to detect the options without a build directory
# Most of the code is stolen from interperter.Interpreter
def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None):
super().__init__(source_root, subdir)
options = BuildoptionsOptionHelper(cross_file)
self.cross_file = cross_file
if env is None:
self.environment = environment.Environment(source_root, None, options)
else:
self.environment = env
self.subproject = subproject
self.subproject_dir = subproject_dir
self.coredata = self.environment.get_coredata()
self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt')
self.backend = backend
self.default_options = {'backend': self.backend}
self.funcs.update({
'project': self.func_project,
'add_languages': self.func_add_languages
})
def detect_compilers(self, lang, need_cross_compiler):
comp, cross_comp = self.environment.detect_compilers(lang, need_cross_compiler)
if comp is None:
return None, None
self.coredata.compilers[lang] = comp
# Native compiler always exist so always add its options.
new_options = comp.get_options()
if cross_comp is not None:
self.coredata.cross_compilers[lang] = cross_comp
new_options.update(cross_comp.get_options())
optprefix = lang + '_'
for k, o in new_options.items():
if not k.startswith(optprefix):
raise RuntimeError('Internal error, %s has incorrect prefix.' % k)
if k in self.environment.cmd_line_options:
o.set_value(self.environment.cmd_line_options[k])
self.coredata.compiler_options.setdefault(k, o)
return comp, cross_comp
def flatten_args(self, args):
# Resolve mparser.ArrayNode if needed
flattend_args = []
for i in args:
if isinstance(i, mparser.ArrayNode):
flattend_args += [x.value for x in i.args.arguments]
elif isinstance(i, str):
flattend_args += [i]
else:
pass
return flattend_args
def add_languages(self, args):
need_cross_compiler = self.environment.is_cross_build() and self.environment.cross_info.need_cross_compiler()
for lang in sorted(args, key=compilers.sort_clink):
lang = lang.lower()
if lang not in self.coredata.compilers:
(comp, _) = self.detect_compilers(lang, need_cross_compiler)
if comp is None:
return
for optname in comp.base_options:
if optname in self.coredata.base_options:
continue
oobj = compilers.base_options[optname]
self.coredata.base_options[optname] = oobj
def func_project(self, node, args, kwargs):
if len(args) < 1:
raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.')
proj_langs = self.flatten_args(args[1:])
if os.path.exists(self.option_file):
oi = optinterpreter.OptionInterpreter(self.subproject)
oi.process(self.option_file)
self.coredata.merge_user_options(oi.options)
def_opts = kwargs.get('default_options', [])
if isinstance(def_opts, mparser.ArrayNode):
def_opts = [x.value for x in def_opts.args.arguments]
self.project_default_options = mesonlib.stringlistify(def_opts)
self.project_default_options = cdata.create_options_dict(self.project_default_options)
self.default_options.update(self.project_default_options)
self.coredata.set_default_options(self.default_options, self.subproject, self.environment.cmd_line_options)
if not self.is_subproject() and 'subproject_dir' in kwargs:
spdirname = kwargs['subproject_dir']
if isinstance(spdirname, str):
self.subproject_dir = spdirname
if not self.is_subproject():
subprojects_dir = os.path.join(self.source_root, self.subproject_dir)
if os.path.isdir(subprojects_dir):
for i in os.listdir(subprojects_dir):
if os.path.isdir(os.path.join(subprojects_dir, i)):
self.do_subproject(i)
self.coredata.init_backend_options(self.backend)
options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')}
self.coredata.set_options(options)
self.add_languages(proj_langs)
def do_subproject(self, dirname):
subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
subpr = os.path.join(subproject_dir_abs, dirname)
try:
subi = BuildoptionsInterperter(subpr, '', self.backend, cross_file=self.cross_file, subproject=dirname, subproject_dir=self.subproject_dir, env=self.environment)
subi.analyze()
except:
return
def func_add_languages(self, node, args, kwargs):
return self.add_languages(self.flatten_args(args))
def is_subproject(self):
return self.subproject != ''
def analyze(self):
self.load_root_meson_file()
self.sanity_check_ast()
self.parse_project()
self.run()
def list_buildoptions_from_source(sourcedir, backend):
# Make sure that log entries in other parts of meson don't interfere with the JSON output
mlog.disable()
backend = backends.get_backend_from_name(backend, None)
intr = BuildoptionsInterperter(sourcedir, '', backend.name)
intr.analyze()
# Reenable logging just in case
mlog.enable()
list_buildoptions(intr.coredata)
def list_buildoptions(coredata):
optlist = []
dir_option_names = ['bindir',
@ -301,10 +454,13 @@ def run(options):
if options.builddir is not None:
datadir = os.path.join(options.builddir, datadir)
if options.builddir.endswith('/meson.build') or options.builddir.endswith('\\meson.build') or options.builddir == 'meson.build':
sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
if options.projectinfo:
sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
list_projinfo_from_source(sourcedir)
return 0
if options.buildoptions:
list_buildoptions_from_source(sourcedir, options.backend)
return 0
if not os.path.isdir(datadir):
print('Current directory is not a build dir. Please specify it or '
'change the working directory to it.')
@ -330,7 +486,7 @@ def run(options):
elif options.buildsystem_files:
list_buildsystem_files(builddata)
elif options.buildoptions:
list_buildoptions(coredata, builddata)
list_buildoptions(coredata)
elif options.tests:
list_tests(testdata)
elif options.benchmarks:

@ -47,6 +47,15 @@ log_fname = 'meson-log.txt'
log_depth = 0
log_timestamp_start = None
log_fatal_warnings = False
log_disable_stdout = False
def disable():
global log_disable_stdout
log_disable_stdout = True
def enable():
global log_disable_stdout
log_disable_stdout = False
def initialize(logdir, fatal_warnings=False):
global log_dir, log_file, log_fatal_warnings
@ -118,6 +127,9 @@ def process_markup(args, keep):
return arr
def force_print(*args, **kwargs):
global log_disable_stdout
if log_disable_stdout:
return
iostr = io.StringIO()
kwargs['file'] = iostr
print(*args, **kwargs)

@ -1199,7 +1199,12 @@ class BasePlatformTests(unittest.TestCase):
args = [args]
out = subprocess.check_output(self.mintro_command + args + [directory],
universal_newlines=True)
return json.loads(out)
try:
obj = json.loads(out)
except Exception as e:
print(out)
raise e
return obj
def assertPathEqual(self, path1, path2):
'''
@ -3089,6 +3094,16 @@ recommended as it is not supported on some platforms''')
self.assertEqual(Path(testfile).read_text(),
Path(goodfile).read_text())
def test_introspect_buildoptions_without_configured_build(self):
testdir = os.path.join(self.unit_test_dir, '51 introspect buildoptions')
testfile = os.path.join(testdir, 'meson.build')
res_nb = self.introspect_directory(testfile, ['--buildoptions'] + self.meson_args)
self.init(testdir, default_args=False)
res_wb = self.introspect('--buildoptions')
self.maxDiff = None
self.assertListEqual(res_nb, res_wb)
class FailureTests(BasePlatformTests):
'''
Tests that test failure conditions. Build files here should be dynamically

@ -0,0 +1,3 @@
#!/usr/bin/env python3
print('c')

@ -0,0 +1,11 @@
project('introspect buildargs', ['c'], default_options: ['c_std=c11', 'cpp_std=c++14', 'buildtype=release'])
subA = subproject('projectA')
r = run_command(find_program('c_compiler.py'))
if r.returncode() != 0
error('FAILED')
endif
add_languages(r.stdout().strip(), required: true)
add_languages('afgggergearvearghergervergreaergaergasv', required: false)

@ -0,0 +1,2 @@
option('max_register_count', type: 'integer', min: 0, value: 125)
option('use_external_fmt', type: 'boolean', value: false)

@ -0,0 +1,3 @@
project('introspect subproject A', 'c', default_options: ['cpp_std=c++11', 'buildtype=debug'])
add_languages('cpp')

@ -0,0 +1 @@
option('subproj_var', type: 'boolean', value: false)

@ -0,0 +1,9 @@
pfggggaergaeg(sdgrgjgn)aga
rgqeh
th
thtr
e
tb
tbqebt
tbqebttrtt

@ -0,0 +1 @@
option('should_not_appear', type: 'integer', min: 0, value: 125)
Loading…
Cancel
Save