Merge pull request #2498 from mesonbuild/runpython

Add possibility to run Python scripts with current interpreter
pull/2653/head
Jussi Pakkanen 7 years ago committed by GitHub
commit d882b6fbd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 9
      docs/markdown/snippets/builtin-python.md
  2. 4
      mesonbuild/backend/ninjabackend.py
  3. 11
      mesonbuild/backend/vs2010backend.py
  4. 2
      mesonbuild/backend/xcodebackend.py
  5. 2
      mesonbuild/dependencies/base.py
  6. 12
      mesonbuild/environment.py
  7. 9
      mesonbuild/mesonlib.py
  8. 94
      mesonbuild/mesonmain.py
  9. 2
      mesonbuild/modules/python3.py
  10. 15
      mesonbuild/scripts/regen_checker.py
  11. 4
      msi/createmsi.py
  12. 27
      run_project_tests.py
  13. 27
      run_tests.py
  14. 29
      run_unittests.py

@ -0,0 +1,9 @@
# Embedded Python in Windows MSI packages
Meson now ships an internal version of Python in the MSI installer packages.
This means that it can run Python scripts that are part of your build
transparently. That is, if you do the following:
myprog = find_program('myscript.py')
Then Meson will run the script with its internal Python version if necessary.

@ -2524,7 +2524,7 @@ rule FORTRAN_DEP_HACK
gcno_elem = NinjaBuildElement(self.all_outputs, 'meson-clean-gcno', 'CUSTOM_COMMAND', 'PHONY')
script_root = self.environment.get_script_dir()
clean_script = os.path.join(script_root, 'delwithsuffix.py')
gcno_elem.add_item('COMMAND', [sys.executable, clean_script, '.', 'gcno'])
gcno_elem.add_item('COMMAND', mesonlib.python_command + [clean_script, '.', 'gcno'])
gcno_elem.add_item('description', 'Deleting gcno files.')
gcno_elem.write(outfile)
# Alias that runs the target defined above
@ -2533,7 +2533,7 @@ rule FORTRAN_DEP_HACK
gcda_elem = NinjaBuildElement(self.all_outputs, 'meson-clean-gcda', 'CUSTOM_COMMAND', 'PHONY')
script_root = self.environment.get_script_dir()
clean_script = os.path.join(script_root, 'delwithsuffix.py')
gcda_elem.add_item('COMMAND', [sys.executable, clean_script, '.', 'gcda'])
gcda_elem.add_item('COMMAND', mesonlib.python_command + [clean_script, '.', 'gcda'])
gcda_elem.add_item('description', 'Deleting gcda files.')
gcda_elem.write(outfile)
# Alias that runs the target defined above

@ -23,7 +23,7 @@ from .. import dependencies
from .. import mlog
from .. import compilers
from ..compilers import CompilerArgs
from ..mesonlib import MesonException, File
from ..mesonlib import MesonException, File, python_command
from ..environment import Environment
def autodetect_vs_version(build):
@ -396,10 +396,11 @@ class Vs2010Backend(backends.Backend):
action = ET.SubElement(root, 'ItemDefinitionGroup')
customstep = ET.SubElement(action, 'PostBuildEvent')
cmd_raw = [target.command] + target.args
cmd = [sys.executable, os.path.join(self.environment.get_script_dir(), 'commandrunner.py'),
self.environment.get_build_dir(),
self.environment.get_source_dir(),
self.get_target_dir(target)] + self.environment.get_build_command()
cmd = python_command + \
[os.path.join(self.environment.get_script_dir(), 'commandrunner.py'),
self.environment.get_build_dir(),
self.environment.get_source_dir(),
self.get_target_dir(target)] + self.environment.get_build_command()
for i in cmd_raw:
if isinstance(i, build.BuildTarget):
cmd.append(os.path.join(self.environment.get_build_dir(), self.get_target_filename(i)))

@ -567,7 +567,7 @@ class XCodeBackend(backends.Backend):
self.write_line('shellPath = /bin/sh;')
script_root = self.environment.get_script_dir()
test_script = os.path.join(script_root, 'meson_test.py')
cmd = [sys.executable, test_script, test_data, '--wd', self.environment.get_build_dir()]
cmd = mesonlib.python_command + [test_script, test_data, '--wd', self.environment.get_build_dir()]
cmdstr = ' '.join(["'%s'" % i for i in cmd])
self.write_line('shellScript = "%s";' % cmdstr)
self.write_line('showEnvVarsInLog = 0;')

@ -434,7 +434,7 @@ class ExternalProgram:
commands = commands[1:]
# Windows does not ship python3.exe, but we know the path to it
if len(commands) > 0 and commands[0] == 'python3':
commands[0] = sys.executable
commands = mesonlib.python_command + commands[1:]
return commands + [script]
except Exception:
pass

@ -325,14 +325,10 @@ class Environment:
return self.coredata
def get_build_command(self, unbuffered=False):
# If running an executable created with cx_freeze,
# Python might not be installed so don't prefix
# the command with it.
if sys.executable.endswith('meson.exe'):
return [sys.executable]
if unbuffered:
[sys.executable, '-u', self.meson_script_launcher]
return [sys.executable, self.meson_script_launcher]
cmd = mesonlib.meson_command[:]
if unbuffered and 'python' in cmd[0]:
cmd.insert(1, '-u')
return cmd
def is_header(self, fname):
return is_header(fname)

@ -22,6 +22,15 @@ import collections
from glob import glob
if os.path.basename(sys.executable) == 'meson.exe':
# In Windows and using the MSI installed executable.
meson_command = [sys.executable]
python_command = [sys.executable, 'runpython']
else:
meson_command = [sys.executable, os.path.join(os.path.dirname(__file__), '..', 'meson.py')]
python_command = [sys.executable]
# Put this in objects that should not get dumped to pickle files
# by accident.
import threading

@ -23,12 +23,9 @@ from . import mlog, coredata
from .mesonlib import MesonException
from .wrap import WrapMode, wraptool
parser = argparse.ArgumentParser(prog='meson')
default_warning = '1'
def add_builtin_argument(name, **kwargs):
def add_builtin_argument(p, name, **kwargs):
k = kwargs.get('dest', name.replace('-', '_'))
c = coredata.get_builtin_option_choices(k)
b = True if kwargs.get('action', None) in ['store_true', 'store_false'] else False
@ -42,31 +39,45 @@ def add_builtin_argument(name, **kwargs):
kwargs['default'] = default
else:
kwargs['default'] = argparse.SUPPRESS
parser.add_argument('--' + name, help=h, **kwargs)
p.add_argument('--' + name, help=h, **kwargs)
add_builtin_argument('prefix')
add_builtin_argument('libdir')
add_builtin_argument('libexecdir')
add_builtin_argument('bindir')
add_builtin_argument('sbindir')
add_builtin_argument('includedir')
add_builtin_argument('datadir')
add_builtin_argument('mandir')
add_builtin_argument('infodir')
add_builtin_argument('localedir')
add_builtin_argument('sysconfdir')
add_builtin_argument('localstatedir')
add_builtin_argument('sharedstatedir')
add_builtin_argument('backend')
add_builtin_argument('buildtype')
add_builtin_argument('strip', action='store_true')
add_builtin_argument('unity')
add_builtin_argument('werror', action='store_true')
add_builtin_argument('layout')
add_builtin_argument('default-library')
add_builtin_argument('warnlevel', dest='warning_level')
add_builtin_argument('stdsplit', action='store_false')
add_builtin_argument('errorlogs', action='store_false')
def create_parser():
p = argparse.ArgumentParser(prog='meson')
add_builtin_argument(p, 'prefix')
add_builtin_argument(p, 'libdir')
add_builtin_argument(p, 'libexecdir')
add_builtin_argument(p, 'bindir')
add_builtin_argument(p, 'sbindir')
add_builtin_argument(p, 'includedir')
add_builtin_argument(p, 'datadir')
add_builtin_argument(p, 'mandir')
add_builtin_argument(p, 'infodir')
add_builtin_argument(p, 'localedir')
add_builtin_argument(p, 'sysconfdir')
add_builtin_argument(p, 'localstatedir')
add_builtin_argument(p, 'sharedstatedir')
add_builtin_argument(p, 'backend')
add_builtin_argument(p, 'buildtype')
add_builtin_argument(p, 'strip', action='store_true')
add_builtin_argument(p, 'unity')
add_builtin_argument(p, 'werror', action='store_true')
add_builtin_argument(p, 'layout')
add_builtin_argument(p, 'default-library')
add_builtin_argument(p, 'warnlevel', dest='warning_level')
add_builtin_argument(p, 'stdsplit', action='store_false')
add_builtin_argument(p, 'errorlogs', action='store_false')
p.add_argument('--cross-file', default=None,
help='File describing cross compilation environment.')
p.add_argument('-D', action='append', dest='projectoptions', default=[], metavar="option",
help='Set the value of an option, can be used several times to set multiple options.')
p.add_argument('-v', '--version', action='version',
version=coredata.version)
# See the mesonlib.WrapMode enum for documentation
p.add_argument('--wrap-mode', default=WrapMode.default,
type=wrapmodetype, choices=WrapMode,
help='Special wrap mode to use')
p.add_argument('directories', nargs='*')
return p
def wrapmodetype(string):
try:
@ -76,18 +87,6 @@ def wrapmodetype(string):
msg = 'invalid argument {!r}, use one of {}'.format(string, msg)
raise argparse.ArgumentTypeError(msg)
parser.add_argument('--cross-file', default=None,
help='File describing cross compilation environment.')
parser.add_argument('-D', action='append', dest='projectoptions', default=[], metavar="option",
help='Set the value of an option, can be used several times to set multiple options.')
parser.add_argument('-v', '--version', action='version',
version=coredata.version)
# See the mesonlib.WrapMode enum for documentation
parser.add_argument('--wrap-mode', default=WrapMode.default,
type=wrapmodetype, choices=WrapMode,
help='Special wrap mode to use')
parser.add_argument('directories', nargs='*')
class MesonApp:
def __init__(self, dir1, dir2, script_launcher, handshake, options, original_cmd_line_args):
@ -155,7 +154,7 @@ class MesonApp:
def _generate(self, env):
mlog.debug('Build started at', datetime.datetime.now().isoformat())
mlog.debug('Python binary:', sys.executable)
mlog.debug('Main binary:', sys.executable)
mlog.debug('Python system:', platform.system())
mlog.log(mlog.bold('The Meson build system'))
self.check_pkgconfig_envvar(env)
@ -278,12 +277,13 @@ def run_script_command(args):
raise MesonException('Unknown internal command {}.'.format(cmdname))
return cmdfunc(cmdargs)
def run(args, mainfile=None):
def run(original_args, mainfile=None):
if sys.version_info < (3, 4):
print('Meson works correctly only with python 3.4+.')
print('You have python %s.' % sys.version)
print('Please update your environment')
return 1
args = original_args[:]
if len(args) > 0:
# First check if we want to run a subcommand.
cmd_name = args[0]
@ -303,6 +303,12 @@ def run(args, mainfile=None):
return mconf.run(remaining_args)
elif cmd_name == 'wrap':
return wraptool.run(remaining_args)
elif cmd_name == 'runpython':
import runpy
script_file = remaining_args[0]
sys.argv[1:] = remaining_args[1:]
runpy.run_path(script_file, run_name='__main__')
sys.exit(0)
# No special command? Do the basic setup/reconf.
if len(args) >= 2 and args[0] == '--internal':
@ -319,6 +325,8 @@ def run(args, mainfile=None):
else:
handshake = False
parser = create_parser()
args = mesonlib.expand_arguments(args)
options = parser.parse_args(args)
args = options.directories
@ -342,7 +350,7 @@ def run(args, mainfile=None):
try:
if mainfile is None:
raise AssertionError('I iz broken. Sorry.')
app = MesonApp(dir1, dir2, mainfile, handshake, options, sys.argv)
app = MesonApp(dir1, dir2, mainfile, handshake, options, original_args)
except Exception as e:
# Log directory does not exist, so just print
# to stdout.

@ -52,7 +52,7 @@ class Python3Module(ExtensionModule):
@noKwargs
def find_python(self, state, args, kwargs):
py3 = dependencies.ExternalProgram('python3', sys.executable, silent=True)
py3 = dependencies.ExternalProgram('python3', mesonlib.python_command, silent=True)
return ModuleReturnValue(py3, [py3])
@noKwargs

@ -14,6 +14,7 @@
import sys, os
import pickle, subprocess
from mesonbuild.mesonlib import meson_command
# This could also be used for XCode.
@ -32,15 +33,11 @@ def need_regen(regeninfo, regen_timestamp):
return False
def regen(regeninfo, mesonscript, backend):
if sys.executable.lower().endswith('meson.exe'):
cmd_exe = [sys.executable]
else:
cmd_exe = [sys.executable, mesonscript]
cmd = cmd_exe + ['--internal',
'regenerate',
regeninfo.build_dir,
regeninfo.source_dir,
'--backend=' + backend]
cmd = meson_command + ['--internal',
'regenerate',
regeninfo.build_dir,
regeninfo.source_dir,
'--backend=' + backend]
subprocess.check_call(cmd)
def run(args):

@ -82,9 +82,7 @@ class PackageGenerator:
modules = [os.path.splitext(os.path.split(x)[1])[0] for x in glob(os.path.join('mesonbuild/modules/*'))]
modules = ['mesonbuild.modules.' + x for x in modules if not x.startswith('_')]
modulestr = ','.join(modules)
python = 'c:\\Python\python.exe'
if sys.executable:
python = sys.executable
python = shutil.which('python')
cxfreeze = os.path.join(os.path.dirname(python), "Scripts", "cxfreeze")
if not os.path.isfile(cxfreeze):
print("ERROR: This script requires cx_freeze module")

@ -33,7 +33,7 @@ import time
import multiprocessing
import concurrent.futures as conc
import re
from run_unittests import get_fake_options, run_configure_inprocess
from run_unittests import get_fake_options, run_configure
from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
from run_tests import ensure_backend_detects_changes
@ -322,14 +322,31 @@ def run_test(skipped, testdir, extra_args, compiler, backend, flags, commands, s
finally:
mlog.shutdown() # Close the log file because otherwise Windows wets itself.
def pass_prefix_to_test(dirname):
if '40 prefix' in dirname:
return False
return True
def pass_libdir_to_test(dirname):
if '8 install' in dirname:
return False
if '39 libdir' in dirname:
return False
return True
def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backend, flags, commands, should_fail):
compile_commands, clean_commands, install_commands, uninstall_commands = commands
test_args = parse_test_args(testdir)
gen_start = time.time()
# Configure in-process
gen_command = [meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir]\
+ flags + test_args + extra_args
(returncode, stdo, stde) = run_configure_inprocess(gen_command)
if pass_prefix_to_test(testdir):
gen_args = ['--prefix', '/usr']
else:
gen_args = []
if pass_libdir_to_test(testdir):
gen_args += ['--libdir', 'lib']
gen_args += [testdir, test_build_dir] + flags + test_args + extra_args
(returncode, stdo, stde) = run_configure(meson_command, gen_args)
try:
logfile = os.path.join(test_build_dir, 'meson-logs/meson-log.txt')
with open(logfile, errors='ignore') as f:
@ -603,7 +620,7 @@ def check_meson_commands_work():
testdir = 'test cases/common/1 trivial'
with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
print('Checking that configuring works...')
gen_cmd = [sys.executable, meson_command, testdir, build_dir] + backend_flags
gen_cmd = mesonlib.meson_command + [testdir, build_dir] + backend_flags
pc, o, e = Popen_safe(gen_cmd)
if pc.returncode != 0:
raise RuntimeError('Failed to configure {!r}:\n{}\n{}'.format(testdir, e, o))

@ -31,6 +31,12 @@ from glob import glob
Backend = Enum('Backend', 'ninja vs xcode')
if 'MESON_EXE' in os.environ:
import shlex
meson_exe = shlex.split(os.environ['MESON_EXE'])
else:
meson_exe = None
if mesonlib.is_windows() or mesonlib.is_cygwin():
exe_suffix = '.exe'
else:
@ -127,18 +133,28 @@ def get_fake_options(prefix):
def should_run_linux_cross_tests():
return shutil.which('arm-linux-gnueabihf-gcc-7') and not platform.machine().lower().startswith('arm')
def run_configure_inprocess(commandlist):
def run_configure_inprocess(meson_command, commandlist):
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
old_stderr = sys.stderr
sys.stderr = mystderr = StringIO()
try:
returncode = mesonmain.run(commandlist[1:], commandlist[0])
returncode = mesonmain.run(commandlist, meson_command)
finally:
sys.stdout = old_stdout
sys.stderr = old_stderr
return returncode, mystdout.getvalue(), mystderr.getvalue()
def run_configure_external(full_command):
pc, o, e = mesonlib.Popen_safe(full_command)
return pc.returncode, o, e
def run_configure(meson_command, commandlist):
global meson_exe
if meson_exe:
return run_configure_external(meson_exe + commandlist)
return run_configure_inprocess(meson_command, commandlist)
class FakeEnvironment(object):
def __init__(self):
self.cross_info = None
@ -207,11 +223,12 @@ if __name__ == '__main__':
'coverage.process_startup()\n')
env['COVERAGE_PROCESS_START'] = '.coveragerc'
env['PYTHONPATH'] = os.pathsep.join([td] + env.get('PYTHONPATH', []))
returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'], env=env)
returncode += subprocess.call(mesonlib.python_command + ['run_unittests.py', '-v'], env=env)
# Ubuntu packages do not have a binary without -6 suffix.
if should_run_linux_cross_tests():
print(mlog.bold('Running cross compilation tests.').get_text(mlog.colorize_console))
print()
returncode += subprocess.call([sys.executable, 'run_cross_test.py', 'cross/ubuntu-armhf.txt'], env=env)
returncode += subprocess.call([sys.executable, 'run_project_tests.py'] + sys.argv[1:], env=env)
returncode += subprocess.call(mesonlib.python_command + ['run_cross_test.py', 'cross/ubuntu-armhf.txt'],
env=env)
returncode += subprocess.call(mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:], env=env)
sys.exit(returncode)

@ -34,13 +34,14 @@ import mesonbuild.mesonlib
import mesonbuild.coredata
from mesonbuild.interpreter import ObjectHolder
from mesonbuild.mesonlib import is_linux, is_windows, is_osx, is_cygwin, windows_proof_rmtree
from mesonbuild.mesonlib import python_command, meson_command
from mesonbuild.environment import Environment
from mesonbuild.dependencies import DependencyException
from mesonbuild.dependencies import PkgConfigDependency, ExternalProgram
from run_tests import exe_suffix, get_fake_options, FakeEnvironment
from run_tests import get_builddir_target_args, get_backend_commands, Backend
from run_tests import ensure_backend_detects_changes, run_configure_inprocess
from run_tests import ensure_backend_detects_changes, run_configure, meson_exe
from run_tests import should_run_linux_cross_tests
@ -460,11 +461,12 @@ class BasePlatformTests(unittest.TestCase):
# Get the backend
# FIXME: Extract this from argv?
self.backend = getattr(Backend, os.environ.get('MESON_UNIT_TEST_BACKEND', 'ninja'))
self.meson_args = [os.path.join(src_root, 'meson.py'), '--backend=' + self.backend.name]
self.meson_command = [sys.executable] + self.meson_args
self.mconf_command = [sys.executable, os.path.join(src_root, 'meson.py'), 'configure']
self.mintro_command = [sys.executable, os.path.join(src_root, 'meson.py'), 'introspect']
self.mtest_command = [sys.executable, os.path.join(src_root, 'meson.py'), 'test', '-C', self.builddir]
self.meson_mainfile = os.path.join(src_root, 'meson.py')
self.meson_args = ['--backend=' + self.backend.name]
self.meson_command = meson_command + self.meson_args
self.mconf_command = meson_command + ['configure']
self.mintro_command = meson_command + ['introspect']
self.mtest_command = meson_command + ['test', '-C', self.builddir]
# Backend-specific build commands
self.build_command, self.clean_command, self.test_command, self.install_command, \
self.uninstall_command = get_backend_commands(self.backend)
@ -527,7 +529,7 @@ class BasePlatformTests(unittest.TestCase):
self.privatedir = os.path.join(self.builddir, 'meson-private')
if inprocess:
try:
out = run_configure_inprocess(self.meson_args + args + extra_args)[1]
out = run_configure(self.meson_mainfile, self.meson_args + args + extra_args)[1]
except:
self._print_meson_log()
raise
@ -1121,14 +1123,14 @@ class AllPlatformTests(BasePlatformTests):
# exelist + some argument. This is meant to test that setting
# something like `ccache gcc -pipe` or `distcc ccache gcc` works.
wrapper = os.path.join(testdir, 'compiler wrapper.py')
wrappercc = [sys.executable, wrapper] + cc.get_exelist() + ['-DSOME_ARG']
wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG']
wrappercc_s = ''
for w in wrappercc:
wrappercc_s += shlex.quote(w) + ' '
os.environ[evar] = wrappercc_s
wcc = getattr(env, 'detect_{}_compiler'.format(lang))(False)
# Check static linker too
wrapperlinker = [sys.executable, wrapper] + linker.get_exelist() + linker.get_always_args()
wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args()
wrapperlinker_s = ''
for w in wrapperlinker:
wrapperlinker_s += shlex.quote(w) + ' '
@ -1586,6 +1588,9 @@ class FailureTests(BasePlatformTests):
Assert that running meson configure on the specified @contents raises
a error message matching regex @match.
'''
if meson_exe is not None:
# Because the exception happens in a different process.
raise unittest.SkipTest('Can not test assert raise tests with an external Meson command.')
if langs is None:
langs = []
with open(self.mbuild, 'w') as f:
@ -1717,12 +1722,12 @@ class WindowsTests(BasePlatformTests):
os.environ['PATH'] += os.pathsep + testdir
prog = ExternalProgram('test-script-ext')
self.assertTrue(prog.found(), msg='test-script-ext not found in PATH')
self.assertPathEqual(prog.get_command()[0], sys.executable)
self.assertPathEqual(prog.get_command()[0], python_command[0])
self.assertPathBasenameEqual(prog.get_path(), 'test-script-ext.py')
# Finding a script in PATH with extension works and adds the interpreter
prog = ExternalProgram('test-script-ext.py')
self.assertTrue(prog.found(), msg='test-script-ext.py not found in PATH')
self.assertPathEqual(prog.get_command()[0], sys.executable)
self.assertPathEqual(prog.get_command()[0], python_command[0])
self.assertPathBasenameEqual(prog.get_path(), 'test-script-ext.py')
def test_ignore_libs(self):
@ -2246,7 +2251,7 @@ class RewriterTests(unittest.TestCase):
super().setUp()
src_root = os.path.dirname(__file__)
self.testroot = os.path.realpath(tempfile.mkdtemp())
self.rewrite_command = [sys.executable, os.path.join(src_root, 'mesonrewriter.py')]
self.rewrite_command = python_command + [os.path.join(src_root, 'mesonrewriter.py')]
self.tmpdir = os.path.realpath(tempfile.mkdtemp())
self.workdir = os.path.join(self.tmpdir, 'foo')
self.test_dir = os.path.join(src_root, 'test cases/rewrite')

Loading…
Cancel
Save