Rewrite custom_target template string substitution

Factor it out into a function in mesonlib.py. This will allow us to
reuse it for generators and for configure_file(). The latter doesn't
implement this at all right now.

Also includes unit tests.

mesonbuild/backend/backends.py

@@ -603,19 +603,15 @@ class Backend:
         return srcs
 
     def eval_custom_target_command(self, target, absolute_outputs=False):
-        # We only want the outputs to be absolute when using the VS backend
+        # We want the outputs to be absolute only when using the VS backend
-        if not absolute_outputs:
-            ofilenames = [os.path.join(self.get_target_dir(target), i) for i in target.output]
-        else:
-            ofilenames = [os.path.join(self.environment.get_build_dir(), self.get_target_dir(target), i)
-                          for i in target.output]
-        srcs = self.get_custom_target_sources(target)
         outdir = self.get_target_dir(target)
-        # Many external programs fail on empty arguments.
+        if absolute_outputs:
-        if outdir == '':
-            outdir = '.'
-        if target.absolute_paths:
             outdir = os.path.join(self.environment.get_build_dir(), outdir)
+        outputs = []
+        for i in target.output:
+            outputs.append(os.path.join(outdir, i))
+        inputs = self.get_custom_target_sources(target)
+        # Evaluate the command list
         cmd = []
         for i in target.command:
             if isinstance(i, build.Executable):
@@ -631,37 +627,10 @@ class Backend:
                 if target.absolute_paths:
                     i = os.path.join(self.environment.get_build_dir(), i)
             # FIXME: str types are blindly added ignoring 'target.absolute_paths'
+            # because we can't know if they refer to a file or just a string
             elif not isinstance(i, str):
                 err_msg = 'Argument {0} is of unknown type {1}'
                 raise RuntimeError(err_msg.format(str(i), str(type(i))))
-            for (j, src) in enumerate(srcs):
-                i = i.replace('@INPUT%d@' % j, src)
-            for (j, res) in enumerate(ofilenames):
-                i = i.replace('@OUTPUT%d@' % j, res)
-            if '@INPUT@' in i:
-                msg = 'Custom target {} has @INPUT@ in the command, but'.format(target.name)
-                if len(srcs) == 0:
-                    raise MesonException(msg + ' no input files')
-                if i == '@INPUT@':
-                    cmd += srcs
-                    continue
-                else:
-                    if len(srcs) > 1:
-                        raise MesonException(msg + ' more than one input file')
-                    i = i.replace('@INPUT@', srcs[0])
-            elif '@OUTPUT@' in i:
-                msg = 'Custom target {} has @OUTPUT@ in the command, but'.format(target.name)
-                if len(ofilenames) == 0:
-                    raise MesonException(msg + ' no output files')
-                if i == '@OUTPUT@':
-                    cmd += ofilenames
-                    continue
-                else:
-                    if len(ofilenames) > 1:
-                        raise MesonException(msg + ' more than one output file')
-                    i = i.replace('@OUTPUT@', ofilenames[0])
-            elif '@OUTDIR@' in i:
-                i = i.replace('@OUTDIR@', outdir)
             elif '@DEPFILE@' in i:
                 if target.depfile is None:
                     msg = 'Custom target {!r} has @DEPFILE@ but no depfile ' \
@@ -680,10 +649,11 @@ class Backend:
                     lead_dir = ''
                 else:
                     lead_dir = self.environment.get_build_dir()
-                i = i.replace(source,
+                i = i.replace(source, os.path.join(lead_dir, outdir))
-                              os.path.join(lead_dir,
-                                           outdir))
             cmd.append(i)
+        # Substitute the rest of the template strings
+        values = mesonlib.get_filenames_templates_dict(inputs, outputs)
+        cmd = mesonlib.substitute_values(cmd, values)
         # This should not be necessary but removing it breaks
         # building GStreamer on Windows. The underlying issue
         # is problems with quoting backslashes on Windows
@@ -703,7 +673,7 @@ class Backend:
         #
         # https://github.com/mesonbuild/meson/pull/737
         cmd = [i.replace('\\', '/') for i in cmd]
-        return srcs, ofilenames, cmd
+        return inputs, outputs, cmd
 
     def run_postconf_scripts(self):
         env = {'MESON_SOURCE_ROOT': self.environment.get_source_dir(),

mesonbuild/build.py

@@ -1530,3 +1530,22 @@ class TestSetup:
         self.gdb = gdb
         self.timeout_multiplier = timeout_multiplier
         self.env = env
+
+def get_sources_output_names(sources):
+    '''
+    For the specified list of @sources which can be strings, Files, or targets,
+    get all the output basenames.
+    '''
+    names = []
+    for s in sources:
+        if hasattr(s, 'held_object'):
+            s = s.held_object
+        if isinstance(s, str):
+            names.append(s)
+        elif isinstance(s, (BuildTarget, CustomTarget, GeneratedList)):
+            names += s.get_outputs()
+        elif isinstance(s, File):
+            names.append(s.fname)
+        else:
+            raise AssertionError('Unknown source type: {!r}'.format(s))
+    return names

mesonbuild/mesonlib.py

@@ -521,3 +521,154 @@ def commonpath(paths):
         new = os.path.join(*new)
         common = pathlib.PurePath(new)
     return str(common)
+
+def iter_regexin_iter(regexiter, initer):
+    '''
+    Takes each regular expression in @regexiter and tries to search for it in
+    every item in @initer. If there is a match, returns that match.
+    Else returns False.
+    '''
+    for regex in regexiter:
+        for ii in initer:
+            if not isinstance(ii, str):
+                continue
+            match = re.search(regex, ii)
+            if match:
+                return match.group()
+    return False
+
+def _substitute_values_check_errors(command, values):
+    # Error checking
+    inregex = ('@INPUT([0-9]+)?@', '@PLAINNAME@', '@BASENAME@')
+    outregex = ('@OUTPUT([0-9]+)?@', '@OUTDIR@')
+    if '@INPUT@' not in values:
+        # Error out if any input-derived templates are present in the command
+        match = iter_regexin_iter(inregex, command)
+        if match:
+            m = 'Command cannot have {!r}, since no input files were specified'
+            raise MesonException(m.format(match))
+    else:
+        if len(values['@INPUT@']) > 1:
+            # Error out if @PLAINNAME@ or @BASENAME@ is present in the command
+            match = iter_regexin_iter(inregex[1:], command)
+            if match:
+                raise MesonException('Command cannot have {!r} when there is '
+                                     'more than one input file'.format(match))
+        # Error out if an invalid @INPUTnn@ template was specified
+        for each in command:
+            if not isinstance(each, str):
+                continue
+            match = re.search(inregex[0], each)
+            if match and match.group() not in values:
+                m = 'Command cannot have {!r} since there are only {!r} inputs'
+                raise MesonException(m.format(match.group(), len(values['@INPUT@'])))
+    if '@OUTPUT@' not in values:
+        # Error out if any output-derived templates are present in the command
+        match = iter_regexin_iter(outregex, command)
+        if match:
+            m = 'Command cannot have {!r} since there are no outputs'
+            raise MesonException(m.format(match))
+    else:
+        # Error out if an invalid @OUTPUTnn@ template was specified
+        for each in command:
+            if not isinstance(each, str):
+                continue
+            match = re.search(outregex[0], each)
+            if match and match.group() not in values:
+                m = 'Command cannot have {!r} since there are only {!r} outputs'
+                raise MesonException(m.format(match.group(), len(values['@OUTPUT@'])))
+
+def substitute_values(command, values):
+    '''
+    Substitute the template strings in the @values dict into the list of
+    strings @command and return a new list. For a full list of the templates,
+    see get_filenames_templates_dict()
+
+    If multiple inputs/outputs are given in the @values dictionary, we
+    substitute @INPUT@ and @OUTPUT@ only if they are the entire string, not
+    just a part of it, and in that case we substitute *all* of them.
+    '''
+    # Error checking
+    _substitute_values_check_errors(command, values)
+    # Substitution
+    outcmd = []
+    for vv in command:
+        if not isinstance(vv, str):
+            outcmd.append(vv)
+        elif '@INPUT@' in vv:
+            inputs = values['@INPUT@']
+            if vv == '@INPUT@':
+                outcmd += inputs
+            elif len(inputs) == 1:
+                outcmd.append(vv.replace('@INPUT@', inputs[0]))
+            else:
+                raise MesonException("Command has '@INPUT@' as part of a "
+                                     "string and more than one input file")
+        elif '@OUTPUT@' in vv:
+            outputs = values['@OUTPUT@']
+            if vv == '@OUTPUT@':
+                outcmd += outputs
+            elif len(outputs) == 1:
+                outcmd.append(vv.replace('@OUTPUT@', outputs[0]))
+            else:
+                raise MesonException("Command has '@OUTPUT@' as part of a "
+                                     "string and more than one output file")
+        # Append values that are exactly a template string.
+        # This is faster than a string replace.
+        elif vv in values:
+            outcmd.append(values[vv])
+        # Substitute everything else with replacement
+        else:
+            for key, value in values.items():
+                if key in ('@INPUT@', '@OUTPUT@'):
+                    # Already done above
+                    continue
+                vv = vv.replace(key, value)
+            outcmd.append(vv)
+    return outcmd
+
+def get_filenames_templates_dict(inputs, outputs):
+    '''
+    Create a dictionary with template strings as keys and values as values for
+    the following templates:
+
+    @INPUT@  - the full path to one or more input files, from @inputs
+    @OUTPUT@ - the full path to one or more output files, from @outputs
+    @OUTDIR@ - the full path to the directory containing the output files
+
+    If there is only one input file, the following keys are also created:
+
+    @PLAINNAME@ - the filename of the input file
+    @BASENAME@ - the filename of the input file with the extension removed
+
+    If there is more than one input file, the following keys are also created:
+
+    @INPUT0@, @INPUT1@, ... one for each input file
+
+    If there is more than one output file, the following keys are also created:
+
+    @OUTPUT0@, @OUTPUT1@, ... one for each output file
+    '''
+    values = {}
+    # Gather values derived from the input
+    if inputs:
+        # We want to substitute all the inputs.
+        values['@INPUT@'] = inputs
+        for (ii, vv) in enumerate(inputs):
+            # Write out @INPUT0@, @INPUT1@, ...
+            values['@INPUT{}@'.format(ii)] = vv
+        if len(inputs) == 1:
+            # Just one value, substitute @PLAINNAME@ and @BASENAME@
+            values['@PLAINNAME@'] = plain = os.path.split(inputs[0])[1]
+            values['@BASENAME@'] = os.path.splitext(plain)[0]
+    if outputs:
+        # Gather values derived from the outputs, similar to above.
+        values['@OUTPUT@'] = outputs
+        for (ii, vv) in enumerate(outputs):
+            values['@OUTPUT{}@'.format(ii)] = vv
+        # Outdir should be the same for all outputs
+        values['@OUTDIR@'] = os.path.split(outputs[0])[0]
+        # Many external programs fail on empty arguments.
+        if values['@OUTDIR@'] == '':
+            values['@OUTDIR@'] = '.'
+    return values

run_unittests.py

@@ -174,6 +174,157 @@ class InternalTests(unittest.TestCase):
         libdir = '/some/path/to/prefix/libdir'
         self.assertEqual(commonpath([prefix, libdir]), str(pathlib.PurePath(prefix)))
 
+    def test_string_templates_substitution(self):
+        dictfunc = mesonbuild.mesonlib.get_filenames_templates_dict
+        substfunc = mesonbuild.mesonlib.substitute_values
+        ME = mesonbuild.mesonlib.MesonException
+
+        # Identity
+        self.assertEqual(dictfunc([], []), {})
+
+        # One input, no outputs
+        inputs = ['bar/foo.c.in']
+        outputs = []
+        ret = dictfunc(inputs, outputs)
+        d = {'@INPUT@': inputs, '@INPUT0@': inputs[0],
+             '@PLAINNAME@': 'foo.c.in', '@BASENAME@': 'foo.c'}
+        # Check dictionary
+        self.assertEqual(ret, d)
+        # Check substitutions
+        cmd = ['some', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), cmd)
+        cmd = ['@INPUT@.out', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), [inputs[0] + '.out'] + cmd[1:])
+        cmd = ['@INPUT0@.out', '@PLAINNAME@.ok', 'strings']
+        self.assertEqual(substfunc(cmd, d),
+                         [inputs[0] + '.out'] + [d['@PLAINNAME@'] + '.ok'] + cmd[2:])
+        cmd = ['@INPUT@', '@BASENAME@.hah', 'strings']
+        self.assertEqual(substfunc(cmd, d),
+                         inputs + [d['@BASENAME@'] + '.hah'] + cmd[2:])
+        cmd = ['@OUTPUT@']
+        self.assertRaises(ME, substfunc, cmd, d)
+
+        # One input, one output
+        inputs = ['bar/foo.c.in']
+        outputs = ['out.c']
+        ret = dictfunc(inputs, outputs)
+        d = {'@INPUT@': inputs, '@INPUT0@': inputs[0],
+             '@PLAINNAME@': 'foo.c.in', '@BASENAME@': 'foo.c',
+             '@OUTPUT@': outputs, '@OUTPUT0@': outputs[0], '@OUTDIR@': '.'}
+        # Check dictionary
+        self.assertEqual(ret, d)
+        # Check substitutions
+        cmd = ['some', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), cmd)
+        cmd = ['@INPUT@.out', '@OUTPUT@', 'strings']
+        self.assertEqual(substfunc(cmd, d),
+                         [inputs[0] + '.out'] + outputs + cmd[2:])
+        cmd = ['@INPUT0@.out', '@PLAINNAME@.ok', '@OUTPUT0@']
+        self.assertEqual(substfunc(cmd, d),
+                         [inputs[0] + '.out', d['@PLAINNAME@'] + '.ok'] + outputs)
+        cmd = ['@INPUT@', '@BASENAME@.hah', 'strings']
+        self.assertEqual(substfunc(cmd, d),
+                         inputs + [d['@BASENAME@'] + '.hah'] + cmd[2:])
+
+        # One input, one output with a subdir
+        outputs = ['dir/out.c']
+        ret = dictfunc(inputs, outputs)
+        d = {'@INPUT@': inputs, '@INPUT0@': inputs[0],
+             '@PLAINNAME@': 'foo.c.in', '@BASENAME@': 'foo.c',
+             '@OUTPUT@': outputs, '@OUTPUT0@': outputs[0], '@OUTDIR@': 'dir'}
+        # Check dictionary
+        self.assertEqual(ret, d)
+
+        # Two inputs, no outputs
+        inputs = ['bar/foo.c.in', 'baz/foo.c.in']
+        outputs = []
+        ret = dictfunc(inputs, outputs)
+        d = {'@INPUT@': inputs, '@INPUT0@': inputs[0], '@INPUT1@': inputs[1]}
+        # Check dictionary
+        self.assertEqual(ret, d)
+        # Check substitutions
+        cmd = ['some', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), cmd)
+        cmd = ['@INPUT@', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), inputs + cmd[1:])
+        cmd = ['@INPUT0@.out', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), [inputs[0] + '.out'] + cmd[1:])
+        cmd = ['@INPUT0@.out', '@INPUT1@.ok', 'strings']
+        self.assertEqual(substfunc(cmd, d), [inputs[0] + '.out', inputs[1] + '.ok'] + cmd[2:])
+        cmd = ['@INPUT0@', '@INPUT1@', 'strings']
+        self.assertEqual(substfunc(cmd, d), inputs + cmd[2:])
+        # Many inputs, can't use @INPUT@ like this
+        cmd = ['@INPUT@.out', 'ordinary', 'strings']
+        # Not enough inputs
+        cmd = ['@INPUT2@.out', 'ordinary', 'strings']
+        self.assertRaises(ME, substfunc, cmd, d)
+        # Too many inputs
+        cmd = ['@PLAINNAME@']
+        self.assertRaises(ME, substfunc, cmd, d)
+        cmd = ['@BASENAME@']
+        self.assertRaises(ME, substfunc, cmd, d)
+        # No outputs
+        cmd = ['@OUTPUT@']
+        self.assertRaises(ME, substfunc, cmd, d)
+        cmd = ['@OUTPUT0@']
+        self.assertRaises(ME, substfunc, cmd, d)
+        cmd = ['@OUTDIR@']
+        self.assertRaises(ME, substfunc, cmd, d)
+
+        # Two inputs, one output
+        outputs = ['dir/out.c']
+        ret = dictfunc(inputs, outputs)
+        d = {'@INPUT@': inputs, '@INPUT0@': inputs[0], '@INPUT1@': inputs[1],
+             '@OUTPUT@': outputs, '@OUTPUT0@': outputs[0], '@OUTDIR@': 'dir'}
+        # Check dictionary
+        self.assertEqual(ret, d)
+        # Check substitutions
+        cmd = ['some', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), cmd)
+        cmd = ['@OUTPUT@', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), outputs + cmd[1:])
+        cmd = ['@OUTPUT@.out', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), [outputs[0] + '.out'] + cmd[1:])
+        cmd = ['@OUTPUT0@.out', '@INPUT1@.ok', 'strings']
+        self.assertEqual(substfunc(cmd, d), [outputs[0] + '.out', inputs[1] + '.ok'] + cmd[2:])
+        # Many inputs, can't use @INPUT@ like this
+        cmd = ['@INPUT@.out', 'ordinary', 'strings']
+        # Not enough inputs
+        cmd = ['@INPUT2@.out', 'ordinary', 'strings']
+        self.assertRaises(ME, substfunc, cmd, d)
+        # Not enough outputs
+        cmd = ['@OUTPUT2@.out', 'ordinary', 'strings']
+        self.assertRaises(ME, substfunc, cmd, d)
+
+        # Two inputs, two outputs
+        outputs = ['dir/out.c', 'dir/out2.c']
+        ret = dictfunc(inputs, outputs)
+        d = {'@INPUT@': inputs, '@INPUT0@': inputs[0], '@INPUT1@': inputs[1],
+             '@OUTPUT@': outputs, '@OUTPUT0@': outputs[0], '@OUTPUT1@': outputs[1],
+             '@OUTDIR@': 'dir'}
+        # Check dictionary
+        self.assertEqual(ret, d)
+        # Check substitutions
+        cmd = ['some', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), cmd)
+        cmd = ['@OUTPUT@', 'ordinary', 'strings']
+        self.assertEqual(substfunc(cmd, d), outputs + cmd[1:])
+        cmd = ['@OUTPUT0@', '@OUTPUT1@', 'strings']
+        self.assertEqual(substfunc(cmd, d), outputs + cmd[2:])
+        cmd = ['@OUTPUT0@.out', '@INPUT1@.ok', '@OUTDIR@']
+        self.assertEqual(substfunc(cmd, d), [outputs[0] + '.out', inputs[1] + '.ok', 'dir'])
+        # Many inputs, can't use @INPUT@ like this
+        cmd = ['@INPUT@.out', 'ordinary', 'strings']
+        # Not enough inputs
+        cmd = ['@INPUT2@.out', 'ordinary', 'strings']
+        self.assertRaises(ME, substfunc, cmd, d)
+        # Not enough outputs
+        cmd = ['@OUTPUT2@.out', 'ordinary', 'strings']
+        self.assertRaises(ME, substfunc, cmd, d)
+        # Many outputs, can't use @OUTPUT@ like this
+        cmd = ['@OUTPUT@.out', 'ordinary', 'strings']
+        self.assertRaises(ME, substfunc, cmd, d)
+
 
 class LinuxlikeTests(unittest.TestCase):
     def setUp(self):