Merge pull request #1228 from mesonbuild/runcross

Fix cross test and run them if a cross compiler is available.

.gitignore

@@ -10,6 +10,8 @@ __pycache__
 
 /meson-test-run.txt
 /meson-test-run.xml
+/meson-cross-test-run.txt
+/meson-cross-test-run.xml
 
 .DS_Store
 *~

cross/ubuntu-armhf.txt

@@ -1,8 +1,8 @@
 [binaries]
 # we could set exe_wrapper = qemu-arm-static but to test the case
 # when cross compiled binaries can't be run we don't do that
-c = '/usr/bin/arm-linux-gnueabihf-gcc'
-cpp = '/usr/bin/arm-linux-gnueabihf-g++'
+c = '/usr/bin/arm-linux-gnueabihf-gcc-6'
+cpp = '/usr/bin/arm-linux-gnueabihf-g++-6'
 ar = '/usr/arm-linux-gnueabihf/bin/ar'
 strip = '/usr/arm-linux-gnueabihf/bin/strip'
 pkgconfig = '/usr/bin/arm-linux-gnueabihf-pkg-config'

mesontest.py

@@ -132,7 +132,11 @@ def write_json_log(jsonlogfile, test_name, result):
               'duration' : result.duration,
               'returncode' : result.returncode,
               'command' : result.cmd,
-              'env' : result.env}
+              }
+    if isinstance(result.env, dict):
+        jresult['env'] = result.env
+    else:
+        jresult['env'] = result.env.get_env(os.environ)
     if result.stde:
         jresult['stderr'] = result.stde
     jsonlogfile.write(json.dumps(jresult) + '\n')
@@ -198,7 +202,7 @@ class TestHarness:
             duration = 0.0
             stdo = 'Not run because can not execute cross compiled binaries.'
             stde = None
-            returncode = -1
+            returncode = GNU_SKIP_RETURNCODE
         else:
             cmd = wrap + cmd + test.cmd_args
             starttime = time.time()

run_cross_test.py

@@ -22,68 +22,27 @@ Not part of the main test suite because of two reasons:
 
 Eventually migrate to something fancier.'''
 
-import os, subprocess, shutil, sys
-import mesonbuild.environment as environment
+import sys, os
 
-from run_tests import gather_tests
+from run_project_tests import gather_tests, run_tests, StopException, setup_commands
+from run_project_tests import failing_logs
 
-test_build_dir = 'work area'
-install_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], 'install dir')
-meson_command = './meson.py'
-
-extra_flags = ['--cross-file', sys.argv[1]]
-ninja_command = environment.detect_ninja()
-if ninja_command is None:
-    raise RuntimeError('Could not find Ninja v1.6 or newer')
-compile_commands = [ninja_command]
-test_commands = [ninja_command, 'test']
-install_commands = [ninja_command, 'install']
-
-def run_test(testdir, should_succeed=True):
-    shutil.rmtree(test_build_dir)
-    shutil.rmtree(install_dir)
-    os.mkdir(test_build_dir)
-    os.mkdir(install_dir)
-    print('Running test: ' + testdir)
-    gen_command = [sys.executable, meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir] + extra_flags
-    p = subprocess.Popen(gen_command)
-    p.wait()
-    if not should_succeed:
-        if p.returncode != 0:
-            return
-        raise RuntimeError('Test that should fail succeeded.')
-    if p.returncode != 0:
-        raise RuntimeError('Generating the build system failed.')
-    pc = subprocess.Popen(compile_commands, cwd=test_build_dir)
-    pc.wait()
-    if pc.returncode != 0:
-        raise RuntimeError('Compiling source code failed.')
-    pt = subprocess.Popen(test_commands, cwd=test_build_dir)
-    pt.wait()
-    if pt.returncode != 0:
-        raise RuntimeError('Running unit tests failed.')
-    install_env = os.environ.copy()
-    install_env['DESTDIR'] = install_dir
-    pi = subprocess.Popen(install_commands, cwd=test_build_dir, env=install_env)
-    pi.wait()
-    if pi.returncode != 0:
-        raise RuntimeError('Running install failed.')
-
-def run_tests():
-    commontests = gather_tests('test cases/common')
-    try:
-        os.mkdir(test_build_dir)
-    except OSError:
-        pass
+def runtests(cross_file):
+    commontests = [('common', gather_tests('test cases/common'), False)]
     try:
-        os.mkdir(install_dir)
-    except OSError:
+        (passing_tests, failing_tests, skipped_tests) = run_tests(commontests, 'meson-cross-test-run', ['--cross', cross_file])
+    except StopException:
         pass
-    print('\nRunning cross compilation tests.\n')
-    [run_test(t) for t in commontests]
+    print('\nTotal passed cross tests:', passing_tests)
+    print('Total failed cross tests:', failing_tests)
+    print('Total skipped cross tests:', skipped_tests)
+    if failing_tests > 0 and ('TRAVIS' in os.environ or 'APPVEYOR' in os.environ):
+        print('\nMesonlogs of failing tests\n')
+        for l in failing_logs:
+            print(l, '\n')
+    sys.exit(failing_tests)
 
 if __name__ == '__main__':
-    script_dir = os.path.split(__file__)[0]
-    if script_dir != '':
-        os.chdir(script_dir)
-    run_tests()
+    setup_commands('ninja')
+    cross_file = sys.argv[1]
+    runtests(cross_file)

run_project_tests.py

@@ -63,9 +63,6 @@ class AutoDeletedDir():
                     raise
                 time.sleep(0.1 * (2**i))
 
-passing_tests = 0
-failing_tests = 0
-skipped_tests = 0
 failing_logs = []
 print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ
 do_debug = not {'MESON_PRINT_TEST_OUTPUT', 'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ)
@@ -370,14 +367,18 @@ def detect_tests_to_run():
     all_tests.append(('python3', gather_tests('test cases/python3'), False if using_backend('ninja') and shutil.which('python3') else True))
     return all_tests
 
-def run_tests(extra_args):
-    global install_commands, passing_tests, failing_tests, stop, executor, futures
-    all_tests = detect_tests_to_run()
-    logfile = open('meson-test-run.txt', 'w', encoding="utf_8")
+def run_tests(all_tests, log_name_base, extra_args):
+    global stop, executor, futures
+    txtname = log_name_base + '.txt'
+    xmlname = log_name_base + '.xml'
+    logfile = open(txtname, 'w', encoding="utf_8")
     junit_root = ET.Element('testsuites')
     conf_time = 0
     build_time = 0
     test_time = 0
+    passing_tests = 0
+    failing_tests = 0
+    skipped_tests = 0
 
     try:
         # This fails in some CI environments for unknown reasons.
@@ -412,7 +413,6 @@ def run_tests(extra_args):
                 current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname,
                                                                          'classname' : name})
                 ET.SubElement(current_test, 'skipped', {})
-                global skipped_tests
                 skipped_tests += 1
             else:
                 without_install = "" if len(install_commands) > 0 else " (without install)"
@@ -442,7 +442,8 @@ def run_tests(extra_args):
     print("\nTotal configuration time: %.2fs" % conf_time)
     print("Total build time: %.2fs" % build_time)
     print("Total test time: %.2fs" % test_time)
-    ET.ElementTree(element=junit_root).write('meson-test-run.xml', xml_declaration=True, encoding='UTF-8')
+    ET.ElementTree(element=junit_root).write(xmlname, xml_declaration=True, encoding='UTF-8')
+    return (passing_tests, failing_tests, skipped_tests)
 
 def check_file(fname):
     linenum = 1
@@ -539,7 +540,8 @@ if __name__ == '__main__':
     check_format()
     pbfiles = generate_prebuilt()
     try:
-        run_tests(options.extra_args)
+        all_tests = detect_tests_to_run()
+        (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.extra_args)
     except StopException:
         pass
     for f in pbfiles:

run_tests.py

@@ -23,5 +23,8 @@ if __name__ == '__main__':
     if mesonlib.is_linux():
         print('Running unittests.\n')
         returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'])
+    if shutil.which('arm-linux-gnueabihf-gcc-6'): # Ubuntu packages do not have a binary without -6 suffix.
+        print('Running cross compilation tests.\n')
+        returncode += subprocess.call([sys.executable, 'run_cross_test.py', 'cross/ubuntu-armhf.txt'])
     returncode += subprocess.call([sys.executable, 'run_project_tests.py'] + sys.argv[1:])
     sys.exit(returncode)

test cases/common/103 manygen/meson.build

@@ -1,8 +1,14 @@
 project('manygen', 'c')
 
-subdir('subdir')
+if meson.is_cross_build()
+  # FIXME error out with skip message once cross test runner
+  # recognizes it.
+  message('Not running this test during cross build.')
+else
+  subdir('subdir')
 
-exe = executable('depuser', 'depuser.c',
-  generated)
+  exe = executable('depuser', 'depuser.c',
+    generated)
 
-test('depuser test', exe)
+  test('depuser test', exe)
+endif

test cases/common/111 has header symbol/meson.build

@@ -24,9 +24,12 @@ assert (cpp.has_header_symbol('iostream', 'std::iostream'), 'iostream not found
 assert (cpp.has_header_symbol('vector', 'std::vector'), 'vector not found in vector.h')
 assert (not cpp.has_header_symbol('limits.h', 'std::iostream'), 'iostream should not be defined in limits.h')
 
-boost = dependency('boost', required : false)
-if boost.found()
-  assert (cpp.has_header_symbol('boost/math/quaternion.hpp', 'boost::math::quaternion', dependencies : boost), 'quaternion not found')
-else
-  assert (not cpp.has_header_symbol('boost/math/quaternion.hpp', 'boost::math::quaternion', dependencies : boost), 'quaternion found?!')
+# Cross compilation and boost do not mix.
+if not meson.is_cross_build()
+  boost = dependency('boost', required : false)
+  if boost.found()
+    assert (cpp.has_header_symbol('boost/math/quaternion.hpp', 'boost::math::quaternion', dependencies : boost), 'quaternion not found')
+  else
+    assert (not cpp.has_header_symbol('boost/math/quaternion.hpp', 'boost::math::quaternion',  dependencies : boost), 'quaternion found?!')
+  endif
 endif

test cases/common/97 selfbuilt custom/meson.build

@@ -3,7 +3,7 @@ project('selfbuilt custom', 'cpp')
 # Build an exe and use it in a custom target
 # whose output is used to build a different exe.
 
-tool = executable('tool', 'tool.cpp')
+tool = executable('tool', 'tool.cpp', native : true)
 
 hfile = custom_target('datah',
   output : 'data.h',