Merge pull request #1346 from msink/test-mingw

appveyor: run tests against MSYS2-MinGW

.appveyor.yml

@@ -20,6 +20,10 @@ environment:
       compiler: msvc2015
       backend: vs2015
 
+    - arch: x86
+      compiler: msys2-mingw
+      backend: ninja
+
     - arch: x64
       compiler: msvc2017
       backend: ninja
@@ -30,6 +34,10 @@ environment:
       backend: vs2017
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
 
+    - arch: x64
+      compiler: msys2-mingw
+      backend: ninja
+
 platform:
   - x64
 
@@ -46,6 +54,7 @@ install:
   - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% )
+  - cmd: if %compiler%==msys2-mingw (if %arch%==x86 (set "PATH=C:\msys64\mingw32\bin;%PATH%") else (set "PATH=C:\msys64\mingw64\bin;%PATH%"))
 
 build_script:
   - cmd: echo No build step.

mesonbuild/backend/backends.py

@@ -601,8 +601,13 @@ class Backend:
 
     def eval_custom_target_command(self, target, absolute_outputs=False):
         # We want the outputs to be absolute only when using the VS backend
+        # XXX: Maybe allow the vs backend to use relative paths too?
+        source_root = self.build_to_src
+        build_root = '.'
         outdir = self.get_target_dir(target)
         if absolute_outputs:
+            source_root = self.environment.get_source_dir()
+            build_root = self.environment.get_source_dir()
             outdir = os.path.join(self.environment.get_build_dir(), outdir)
         outputs = []
         for i in target.output:
@@ -628,6 +633,10 @@ class Backend:
             elif not isinstance(i, str):
                 err_msg = 'Argument {0} is of unknown type {1}'
                 raise RuntimeError(err_msg.format(str(i), str(type(i))))
+            elif '@SOURCE_ROOT@' in i:
+                i = i.replace('@SOURCE_ROOT@', source_root)
+            elif '@BUILD_ROOT@' in i:
+                i = i.replace('@BUILD_ROOT@', build_root)
             elif '@DEPFILE@' in i:
                 if target.depfile is None:
                     msg = 'Custom target {!r} has @DEPFILE@ but no depfile ' \

mesonbuild/backend/ninjabackend.py

@@ -1602,6 +1602,8 @@ rule FORTRAN_DEP_HACK
             relout = self.get_target_private_dir(target)
             args = [x.replace("@SOURCE_DIR@", self.build_to_src).replace("@BUILD_DIR@", relout)
                     for x in args]
+            args = [x.replace("@SOURCE_ROOT@", self.build_to_src).replace("@BUILD_ROOT@", '.')
+                    for x in args]
             cmdlist = exe_arr + self.replace_extra_args(args, genlist)
             elem = NinjaBuildElement(self.all_outputs, outfiles, rulename, infilename)
             if generator.depfile is not None:

mesonbuild/backend/vs2010backend.py

@@ -151,7 +151,11 @@ class Vs2010Backend(backends.Backend):
                     args = [x.replace("@INPUT@", infilename).replace('@OUTPUT@', sole_output)
                             for x in base_args]
                     args = self.replace_outputs(args, target_private_dir, outfiles_rel)
-                    args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir()).replace("@BUILD_DIR@", target_private_dir)
+                    args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir())
+                             .replace("@BUILD_DIR@", target_private_dir)
+                            for x in args]
+                    args = [x.replace("@SOURCE_ROOT@", self.environment.get_source_dir())
+                             .replace("@BUILD_ROOT@", self.environment.get_build_dir())
                             for x in args]
                     cmd = exe_arr + self.replace_extra_args(args, genlist)
                     cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename)

mesonbuild/modules/__init__.py

@@ -24,7 +24,11 @@ def find_program(program_name, target_name):
     return program
 
 
-def get_include_args(environment, include_dirs, prefix='-I'):
+def get_include_args(include_dirs, prefix='-I'):
+    '''
+    Expand include arguments to refer to the source and build dirs
+    by using @SOURCE_ROOT@ and @BUILD_ROOT@ for later substitution
+    '''
     if not include_dirs:
         return []
 
@@ -43,8 +47,8 @@ def get_include_args(environment, include_dirs, prefix='-I'):
         basedir = dirs.get_curdir()
         for d in dirs.get_incdirs():
             expdir = os.path.join(basedir, d)
-            srctreedir = os.path.join(environment.get_source_dir(), expdir)
+            srctreedir = os.path.join('@SOURCE_ROOT@', expdir)
-            buildtreedir = os.path.join(environment.get_build_dir(), expdir)
+            buildtreedir = os.path.join('@BUILD_ROOT@', expdir)
             dirs_str += ['%s%s' % (prefix, buildtreedir),
                          '%s%s' % (prefix, srctreedir)]
         for d in dirs.get_extra_build_dirs():

mesonbuild/modules/gnome.py

@@ -309,7 +309,7 @@ class GnomeModule(ExtensionModule):
             if hasattr(dep, 'held_object'):
                 dep = dep.held_object
             if isinstance(dep, InternalDependency):
-                cflags.update(get_include_args(state.environment, dep.include_directories))
+                cflags.update(get_include_args(dep.include_directories))
                 for lib in dep.libraries:
                     ldflags.update(self._get_link_args(state, lib.held_object, depends, include_rpath))
                     libdepflags = self._get_dependencies_flags(lib.held_object.get_external_deps(), state, depends, include_rpath,
@@ -398,7 +398,7 @@ class GnomeModule(ExtensionModule):
         scan_command += extra_args
         scan_command += ['-I' + os.path.join(state.environment.get_source_dir(), state.subdir),
                          '-I' + os.path.join(state.environment.get_build_dir(), state.subdir)]
-        scan_command += get_include_args(state.environment, girtarget.get_include_dirs())
+        scan_command += get_include_args(girtarget.get_include_dirs())
 
         if 'link_with' in kwargs:
             link_with = kwargs.pop('link_with')
@@ -525,9 +525,8 @@ class GnomeModule(ExtensionModule):
             if not isinstance(incd.held_object, (str, build.IncludeDirs)):
                 raise MesonException(
                     'Gir include dirs should be include_directories().')
-        scan_command += get_include_args(state.environment, inc_dirs)
+        scan_command += get_include_args(inc_dirs)
-        scan_command += get_include_args(state.environment, gir_inc_dirs + inc_dirs,
+        scan_command += get_include_args(gir_inc_dirs + inc_dirs, prefix='--add-include-path=')
-                                         prefix='--add-include-path=')
 
         if isinstance(girtarget, build.Executable):
             scan_command += ['--program', girtarget]
@@ -546,8 +545,7 @@ class GnomeModule(ExtensionModule):
 
         typelib_output = '%s-%s.typelib' % (ns, nsversion)
         typelib_cmd = [gicompiler, scan_target, '--output', '@OUTPUT@']
-        typelib_cmd += get_include_args(state.environment, gir_inc_dirs,
+        typelib_cmd += get_include_args(gir_inc_dirs, prefix='--includedir=')
-                                        prefix='--includedir=')
         for incdir in typelib_includes:
             typelib_cmd += ["--includedir=" + incdir]
 
@@ -716,7 +714,7 @@ class GnomeModule(ExtensionModule):
             if not isinstance(incd.held_object, (str, build.IncludeDirs)):
                 raise MesonException(
                     'Gir include dirs should be include_directories().')
-        cflags.update(get_include_args(state.environment, inc_dirs))
+        cflags.update(get_include_args(inc_dirs))
         if cflags:
             args += ['--cflags=%s' % ' '.join(cflags)]
         if ldflags:

mesonbuild/modules/windows.py

@@ -14,6 +14,7 @@
 
 import os
 
+from .. import mlog
 from .. import mesonlib, dependencies, build
 from ..mesonlib import MesonException
 from . import get_include_args
@@ -38,13 +39,18 @@ class WindowsModule(ExtensionModule):
         for incd in inc_dirs:
             if not isinstance(incd.held_object, (str, build.IncludeDirs)):
                 raise MesonException('Resource include dirs should be include_directories().')
-        extra_args += get_include_args(state.environment, inc_dirs)
+        extra_args += get_include_args(inc_dirs)
 
         if comp.id == 'msvc':
             rescomp = dependencies.ExternalProgram('rc', silent=True)
             res_args = extra_args + ['/nologo', '/fo@OUTPUT@', '@INPUT@']
             suffix = 'res'
         else:
+            m = 'Argument {!r} has a space which may not work with windres due to ' \
+                'a MinGW bug: https://sourceware.org/bugzilla/show_bug.cgi?id=4933'
+            for arg in extra_args:
+                if ' ' in arg:
+                    mlog.warning(m.format(arg))
             # Pick-up env var WINDRES if set. This is often used for specifying
             # an arch-specific windres.
             rescomp_name = os.environ.get('WINDRES', 'windres')

run_project_tests.py

@@ -34,6 +34,7 @@ import concurrent.futures as conc
 
 from mesonbuild.coredata import backendlist
 
+
 class BuildStep(Enum):
     configure = 1
     build = 2
@@ -41,6 +42,7 @@ class BuildStep(Enum):
     install = 4
     clean = 5
 
+
 class TestResult:
     def __init__(self, msg, step, stdo, stde, mlog, conftime=0, buildtime=0, testtime=0):
         self.msg = msg
@@ -52,6 +54,54 @@ class TestResult:
         self.buildtime = buildtime
         self.testtime = testtime
 
+class DummyFuture(conc.Future):
+    '''
+    Dummy Future implementation that executes the provided function when you
+    ask for the result. Used on platforms where sem_open() is not available:
+    MSYS2, OpenBSD, etc: https://bugs.python.org/issue3770
+    '''
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def set_function(self, fn, *args, **kwargs):
+        self.fn = fn
+        self.fn_args = args
+        self.fn_kwargs = kwargs
+
+    def result(self, **kwargs):
+        try:
+            result = self.fn(*self.fn_args, **self.fn_kwargs)
+        except BaseException as e:
+            self.set_exception(e)
+        else:
+            self.set_result(result)
+        return super().result(**kwargs)
+
+
+class DummyExecutor(conc.Executor):
+    '''
+    Dummy single-thread 'concurrent' executor for use on platforms where
+    sem_open is not available: https://bugs.python.org/issue3770
+    '''
+
+    def __init__(self):
+        from threading import Lock
+        self._shutdown = False
+        self._shutdownLock = Lock()
+
+    def submit(self, fn, *args, **kwargs):
+        with self._shutdownLock:
+            if self._shutdown:
+                raise RuntimeError('Cannot schedule new futures after shutdown')
+            f = DummyFuture()
+            f.set_function(fn, *args, **kwargs)
+            return f
+
+    def shutdown(self, wait=True):
+        with self._shutdownLock:
+            self._shutdown = True
+
+
 class AutoDeletedDir:
     def __init__(self, d):
         self.dir = d
@@ -194,7 +244,7 @@ def validate_install(srcdir, installdir):
         # Windows-specific tests check for the existence of installed PDB
         # files, but common tests do not, for obvious reasons. Ignore any
         # extra PDB files found.
-        if fname not in expected and not fname.endswith('.pdb'):
+        if fname not in expected and not fname.endswith('.pdb') and compiler == 'cl':
             ret_msg += 'Extra file {0} found.\n'.format(fname)
     return ret_msg
 
@@ -421,7 +471,11 @@ def run_tests(all_tests, log_name_base, extra_args):
         print('Could not determine number of CPUs due to the following reason:' + str(e))
         print('Defaulting to using only one process')
         num_workers = 1
+    try:
         executor = conc.ProcessPoolExecutor(max_workers=num_workers)
+    except ImportError:
+        print('Platform doesn\'t ProcessPoolExecutor, falling back to single-threaded testing\n')
+        executor = DummyExecutor()
 
     for name, test_cases, skipped in all_tests:
         current_suite = ET.SubElement(junit_root, 'testsuite', {'name': name, 'tests': str(len(test_cases))})
@@ -441,6 +495,7 @@ def run_tests(all_tests, log_name_base, extra_args):
             result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, should_fail)
             futures.append((testname, t, result))
         for (testname, t, result) in futures:
+            sys.stdout.flush()
             result = result.result()
             if result is None or 'MESON_SKIP_TEST' in result.stdo:
                 print('Skipping:', t)
@@ -532,6 +587,7 @@ def generate_pb_static(compiler, object_suffix, static_suffix):
     return stlibfile
 
 def generate_prebuilt():
+    global compiler
     static_suffix = 'a'
     if shutil.which('cl'):
         compiler = 'cl'

run_unittests.py

@@ -554,6 +554,8 @@ class AllPlatformTests(BasePlatformTests):
                           get_fake_options(self.prefix), [])
         cc = env.detect_c_compiler(False)
         static_linker = env.detect_static_linker(cc)
+        if is_windows():
+            raise unittest.SkipTest('https://github.com/mesonbuild/meson/issues/1526')
         if not isinstance(static_linker, mesonbuild.compilers.ArLinker):
             raise unittest.SkipTest('static linker is not `ar`')
         # Configure

test cases/common/127 cpp and asm/meson.build

@@ -1,5 +1,6 @@
 project('c++ and assembly test', 'cpp')
 
+cpp = meson.get_compiler('cpp')
 cpu = host_machine.cpu_family()
 
 supported_cpus = ['arm', 'x86', 'x86_64']
@@ -8,6 +9,10 @@ if not supported_cpus.contains(cpu)
   error('MESON_SKIP_TEST unsupported cpu:' + cpu)
 endif
 
+if cpp.symbols_have_underscore_prefix()
+  add_project_arguments('-DMESON_TEST__UNDERSCORE_SYMBOL', language : 'cpp')
+endif
+
 sources = ['trivial.cc']
 # If the compiler cannot compile assembly, don't use it
 if meson.get_compiler('cpp').get_id() != 'msvc'

test cases/common/127 cpp and asm/symbol-underscore.h

@@ -1,4 +1,4 @@
-#if defined(__WIN32__) || defined(__APPLE__)
+#if defined(MESON_TEST__UNDERSCORE_SYMBOL)
 # define SYMBOL_NAME(name) _##name
 #else
 # define SYMBOL_NAME(name) name

test cases/common/135 generated assembly/square-arm.S.in

@@ -2,7 +2,8 @@
 
 .text
 .globl SYMBOL_NAME(square_unsigned)
-#ifndef __APPLE__
+/* Only supported on Linux with GAS */
+# ifdef __linux__
 .type	square_unsigned,%function
 #endif
 

test cases/common/135 generated assembly/square-x86.S.in

@@ -21,7 +21,8 @@ END
 
 .text
 .globl SYMBOL_NAME(square_unsigned)
-# ifndef __APPLE__
+/* Only supported on Linux with GAS */
+# ifdef __linux__
 .type  square_unsigned,@function
 # endif
 

test cases/common/135 generated assembly/square-x86_64.S.in

@@ -1,7 +1,5 @@
 #include "symbol-underscore.h"
 
-#include "symbol-underscore.h"
-
 #ifdef _MSC_VER /* MSVC on Windows */
 
 PUBLIC SYMBOL_NAME(square_unsigned)
@@ -20,7 +18,8 @@ END
 
 .text
 .globl SYMBOL_NAME(square_unsigned)
-# ifndef __APPLE__
+/* Only supported on Linux with GAS */
+# ifdef __linux__
 .type  square_unsigned,@function
 # endif
 

test cases/windows/5 resources/inc/meson.build

@@ -0,0 +1 @@
+inc = include_directories('resource')

test cases/windows/5 resources/meson.build

@@ -1,9 +1,66 @@
 project('winmain', 'c')
 
-win = import('windows')
+# MinGW windres has a bug due to which it doesn't parse args with space properly:
-res = win.compile_resources('myres.rc',
+# https://github.com/mesonbuild/meson/pull/1346
-  include_directories : include_directories('inc')
+# https://sourceware.org/bugzilla/show_bug.cgi?id=4933
-)
+if meson.get_compiler('c').get_id() == 'gcc'
+  # Construct build_to_src and skip this test if it has spaces
+  # because then the -I flag to windres will also have spaces
+  # and we know the test will fail
+  src_parts = meson.source_root().split('/')
+  build_parts = meson.build_root().split('/')
+
+  # Get the common path (which might just be '/' or 'C:/')
+  common = []
+  done = false
+  count = 0
+  if src_parts.length() > build_parts.length()
+    parts = build_parts
+    other = src_parts
+  else
+    parts = src_parts
+    other = build_parts
+  endif
+  foreach part : parts
+    if not done and part == other.get(count)
+      common += [part]
+    else
+      done = true
+    endif
+    count += 1
+  endforeach
+
+  # Create path components to go down from the build root to the common path
+  count = 0
+  rel = build_parts
+  foreach build : build_parts
+    if count < build_parts.length() - common.length()
+      rel += ['..']
+    endif
+    count += 1
+  endforeach
+
+  # Create path components to go up from the common path to the build root
+  count = 0
+  foreach src : src_parts
+    if count >= common.length()
+      rel += [src]
+    endif
+    count += 1
+  endforeach
+
+  build_to_src = '/'.join(rel)
+
+  if build_to_src.contains(' ')
+    message('build_to_src is: ' + build_to_src)
+    error('MESON_SKIP_TEST build_to_src has spaces')
+  endif
+  # Welcome to the end of this conditional.
+  # We hope you never have to implement something like this.
+endif
+
+subdir('inc')
+subdir('res')
 
 exe = executable('prog', 'prog.c',
   res,

test cases/windows/5 resources/res/meson.build

@@ -0,0 +1,4 @@
+win = import('windows')
+
+res = win.compile_resources('myres.rc',
+  include_directories : inc)