Chiebot-Mirror
/
meson
mirror of https://github.com/mesonbuild/meson.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

ninja: Push ninja and shell quoting down into NinjaRule

Browse Source 
Rather than ad-hoc avoiding quoting where harmful, identify arguments
which contain shell constructs and ninja variables, and don't apply
quoting to those arguments.

This is made more complex by some arguments which might contain ninja
variables anywhere, not just at start, e.g. '/Fo$out'

(This implementation would fall down if there was an argument which
contained both a literal $ or shell metacharacter and a ninja variable,
but there are no instances of such a thing and it seems unlikely)

$DEPFILE needs special treatment.  It's used in the special variable
depfile, so it's value can't be shell quoted (as it used as a filename
to read by ninja).  So instead that variable needs to be shell quoted
when it appears in a command.

(Test common/129, which uses a depfile with a space in it's name,
exercises that)

If 'targetdep' is not in raw_names, test cases/rust all fail.
pull/7245/head
Jon Turney 6 years ago committed by Dan Kegel
parent 2f070c54bd
commit 9cec5f3521
1 changed files with 91 additions and 50 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 141
      mesonbuild/backend/ninjabackend.py

141
mesonbuild/backend/ninjabackend.py
Unescape View File

@ -17,6 +17,7 @@ import re
import pickle import pickle
import subprocess import subprocess
from collections import OrderedDict from collections import OrderedDict
from enum import Enum, unique
import itertools import itertools
from pathlib import PurePath, Path from pathlib import PurePath, Path
from functools import lru_cache from functools import lru_cache
@ -60,6 +61,12 @@ else:
# a conservative estimate of the command-line length limit on windows # a conservative estimate of the command-line length limit on windows
rsp_threshold = 4096 rsp_threshold = 4096
# ninja variables whose value should remain unquoted. The value of these ninja
# variables (or variables we use them in) is interpreted directly by ninja
# (e.g. the value of the depfile variable is a pathname that ninja will read
# from, etc.), so it must not be shell quoted.
raw_names = {'DEPFILE', 'DESC', 'pool', 'description', 'targetdep'}
def ninja_quote(text, is_build_line=False): def ninja_quote(text, is_build_line=False):
if is_build_line: if is_build_line:
qcs = ('$', ' ', ':') qcs = ('$', ' ', ':')
@ -76,6 +83,25 @@ Please report this error with a test case to the Meson bug tracker.'''.format(te
raise MesonException(errmsg) raise MesonException(errmsg)
return text return text
@unique
class Quoting(Enum):
both = 0
notShell = 1
notNinja = 2
none = 3
class NinjaCommandArg:
def __init__(self, s, quoting = Quoting.both):
self.s = s
self.quoting = quoting
def __str__(self):
return self.s
@staticmethod
def list(l, q):
return [NinjaCommandArg(i, q) for i in l]
class NinjaComment: class NinjaComment:
def __init__(self, comment): def __init__(self, comment):
self.comment = comment self.comment = comment
@ -90,9 +116,32 @@ class NinjaComment:
class NinjaRule: class NinjaRule:
def __init__(self, rule, command, args, description, def __init__(self, rule, command, args, description,
rspable = False, deps = None, depfile = None, extra = None): rspable = False, deps = None, depfile = None, extra = None):
def strToCommandArg(c):
if isinstance(c, NinjaCommandArg):
return c
# deal with common cases here, so we don't have to explicitly
# annotate the required quoting everywhere
if c == '&&':
# shell constructs shouldn't be shell quoted
return NinjaCommandArg(c, Quoting.notShell)
if c.startswith('$'):
var = re.search(r'\$\{?(\w*)\}?', c).group(1)
if var not in raw_names:
# ninja variables shouldn't be ninja quoted, and their value
# is already shell quoted
return NinjaCommandArg(c, Quoting.none)
else:
# shell quote the use of ninja variables whose value must
# not be shell quoted (as it also used by ninja)
return NinjaCommandArg(c, Quoting.notNinja)
return NinjaCommandArg(c)
self.name = rule self.name = rule
self.command = command # includes args which never go into a rspfile self.command = list(map(strToCommandArg, command)) # includes args which never go into a rspfile
self.args = args # args which will go into a rspfile, if used self.args = list(map(strToCommandArg, args)) # args which will go into a rspfile, if used
self.description = description self.description = description
self.deps = deps # depstyle 'gcc' or 'msvc' self.deps = deps # depstyle 'gcc' or 'msvc'
self.depfile = depfile self.depfile = depfile
@ -101,6 +150,18 @@ class NinjaRule:
self.refcount = 0 self.refcount = 0
self.rsprefcount = 0 self.rsprefcount = 0
@staticmethod
def _quoter(x):
if isinstance(x, NinjaCommandArg):
if x.quoting == Quoting.none:
return x.s
elif x.quoting == Quoting.notNinja:
return quote_func(x.s)
elif x.quoting == Quoting.notShell:
return ninja_quote(x.s)
# fallthrough
return ninja_quote(quote_func(str(x)))
def write(self, outfile): def write(self, outfile):
def rule_iter(): def rule_iter():
if self.refcount: if self.refcount:
@ -111,11 +172,11 @@ class NinjaRule:
for rsp in rule_iter(): for rsp in rule_iter():
outfile.write('rule {}{}\n'.format(self.name, rsp)) outfile.write('rule {}{}\n'.format(self.name, rsp))
if rsp == '_RSP': if rsp == '_RSP':
outfile.write(' command = {} @$out.rsp\n'.format(' '.join(self.command))) outfile.write(' command = {} @$out.rsp\n'.format(' '.join([self._quoter(x) for x in self.command])))
outfile.write(' rspfile = $out.rsp\n') outfile.write(' rspfile = $out.rsp\n')
outfile.write(' rspfile_content = {}\n'.format(' '.join(self.args))) outfile.write(' rspfile_content = {}\n'.format(' '.join([self._quoter(x) for x in self.args])))
else: else:
outfile.write(' command = {}\n'.format(' '.join(self.command + self.args))) outfile.write(' command = {}\n'.format(' '.join([self._quoter(x) for x in (self.command + self.args)])))
if self.deps: if self.deps:
outfile.write(' deps = {}\n'.format(self.deps)) outfile.write(' deps = {}\n'.format(self.deps))
if self.depfile: if self.depfile:
@ -141,9 +202,8 @@ class NinjaRule:
ninja_vars['in'] = infiles ninja_vars['in'] = infiles
ninja_vars['out'] = outfiles ninja_vars['out'] = outfiles
# expand variables in command (XXX: this ignores any escaping/quoting # expand variables in command
# that NinjaBuildElement.write() might do) command = ' '.join([self._quoter(x) for x in self.command + self.args])
command = ' '.join(self.command + self.args)
expanded_command = '' expanded_command = ''
for m in re.finditer(r'(\${\w*})|(\$\w*)|([^$]*)', command): for m in re.finditer(r'(\${\w*})|(\$\w*)|([^$]*)', command):
chunk = m.group() chunk = m.group()
@ -244,12 +304,6 @@ class NinjaBuildElement:
line = line.replace('\\', '/') line = line.replace('\\', '/')
outfile.write(line) outfile.write(line)
# ninja variables whose value should remain unquoted. The value of these
# ninja variables (or variables we use them in) is interpreted directly
# by ninja (e.g. the value of the depfile variable is a pathname that
# ninja will read from, etc.), so it must not be shell quoted.
raw_names = {'DEPFILE', 'DESC', 'pool', 'description', 'targetdep'}
for e in self.elems: for e in self.elems:
(name, elems) = e (name, elems) = e
should_quote = name not in raw_names should_quote = name not in raw_names
@ -945,13 +999,15 @@ int dummy;
deps='gcc', depfile='$DEPFILE', deps='gcc', depfile='$DEPFILE',
extra='restat = 1')) extra='restat = 1'))
c = [ninja_quote(quote_func(x)) for x in self.environment.get_build_command()] + \ c = self.environment.get_build_command() + \
['--internal', ['--internal',
'regenerate', 'regenerate',
ninja_quote(quote_func(self.environment.get_source_dir())), self.environment.get_source_dir(),
ninja_quote(quote_func(self.environment.get_build_dir()))] self.environment.get_build_dir(),
'--backend',
'ninja']
self.add_rule(NinjaRule('REGENERATE_BUILD', self.add_rule(NinjaRule('REGENERATE_BUILD',
c + ['--backend', 'ninja'], [], c, [],
'Regenerating build files.', 'Regenerating build files.',
extra='generator = 1')) extra='generator = 1'))
@ -1630,7 +1686,7 @@ int dummy;
cmdlist = execute_wrapper + [c.format('$out') for c in rmfile_prefix] cmdlist = execute_wrapper + [c.format('$out') for c in rmfile_prefix]
cmdlist += static_linker.get_exelist() cmdlist += static_linker.get_exelist()
cmdlist += ['$LINK_ARGS'] cmdlist += ['$LINK_ARGS']
cmdlist += static_linker.get_output_args('$out') cmdlist += NinjaCommandArg.list(static_linker.get_output_args('$out'), Quoting.none)
description = 'Linking static target $out' description = 'Linking static target $out'
if num_pools > 0: if num_pools > 0:
pool = 'pool = link_pool' pool = 'pool = link_pool'
@ -1652,7 +1708,7 @@ int dummy;
continue continue
rule = '{}_LINKER{}'.format(langname, self.get_rule_suffix(for_machine)) rule = '{}_LINKER{}'.format(langname, self.get_rule_suffix(for_machine))
command = compiler.get_linker_exelist() command = compiler.get_linker_exelist()
args = ['$ARGS'] + compiler.get_linker_output_args('$out') + ['$in', '$LINK_ARGS'] args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_linker_output_args('$out'), Quoting.none) + ['$in', '$LINK_ARGS']
description = 'Linking target $out' description = 'Linking target $out'
if num_pools > 0: if num_pools > 0:
pool = 'pool = link_pool' pool = 'pool = link_pool'
@ -1662,10 +1718,10 @@ int dummy;
rspable=compiler.can_linker_accept_rsp(), rspable=compiler.can_linker_accept_rsp(),
extra=pool)) extra=pool))
args = [ninja_quote(quote_func(x)) for x in self.environment.get_build_command()] + \ args = self.environment.get_build_command() + \
['--internal', ['--internal',
'symbolextractor', 'symbolextractor',
ninja_quote(quote_func(self.environment.get_build_dir())), self.environment.get_build_dir(),
'$in', '$in',
'$IMPLIB', '$IMPLIB',
'$out'] '$out']
@ -1677,15 +1733,13 @@ int dummy;
def generate_java_compile_rule(self, compiler): def generate_java_compile_rule(self, compiler):
rule = self.compiler_to_rule_name(compiler) rule = self.compiler_to_rule_name(compiler)
invoc = [ninja_quote(i) for i in compiler.get_exelist()] command = compiler.get_exelist() + ['$ARGS', '$in']
command = invoc + ['$ARGS', '$in']
description = 'Compiling Java object $in' description = 'Compiling Java object $in'
self.add_rule(NinjaRule(rule, command, [], description)) self.add_rule(NinjaRule(rule, command, [], description))
def generate_cs_compile_rule(self, compiler): def generate_cs_compile_rule(self, compiler):
rule = self.compiler_to_rule_name(compiler) rule = self.compiler_to_rule_name(compiler)
invoc = [ninja_quote(i) for i in compiler.get_exelist()] command = compiler.get_exelist()
command = invoc
args = ['$ARGS', '$in'] args = ['$ARGS', '$in']
description = 'Compiling C Sharp target $out' description = 'Compiling C Sharp target $out'
self.add_rule(NinjaRule(rule, command, args, description, self.add_rule(NinjaRule(rule, command, args, description,
@ -1693,15 +1747,13 @@ int dummy;
def generate_vala_compile_rules(self, compiler): def generate_vala_compile_rules(self, compiler):
rule = self.compiler_to_rule_name(compiler) rule = self.compiler_to_rule_name(compiler)
invoc = [ninja_quote(i) for i in compiler.get_exelist()] command = compiler.get_exelist() + ['$ARGS', '$in']
command = invoc + ['$ARGS', '$in']
description = 'Compiling Vala source $in' description = 'Compiling Vala source $in'
self.add_rule(NinjaRule(rule, command, [], description, extra='restat = 1')) self.add_rule(NinjaRule(rule, command, [], description, extra='restat = 1'))
def generate_rust_compile_rules(self, compiler): def generate_rust_compile_rules(self, compiler):
rule = self.compiler_to_rule_name(compiler) rule = self.compiler_to_rule_name(compiler)
invoc = [ninja_quote(i) for i in compiler.get_exelist()] command = compiler.get_exelist() + ['$ARGS', '$in']
command = invoc + ['$ARGS', '$in']
description = 'Compiling Rust source $in' description = 'Compiling Rust source $in'
depfile = '$targetdep' depfile = '$targetdep'
depstyle = 'gcc' depstyle = 'gcc'
@ -1710,12 +1762,12 @@ int dummy;
def generate_swift_compile_rules(self, compiler): def generate_swift_compile_rules(self, compiler):
rule = self.compiler_to_rule_name(compiler) rule = self.compiler_to_rule_name(compiler)
full_exe = [ninja_quote(x) for x in self.environment.get_build_command()] + [ full_exe = self.environment.get_build_command() + [
'--internal', '--internal',
'dirchanger', 'dirchanger',
'$RUNDIR', '$RUNDIR',
] ]
invoc = full_exe + [ninja_quote(i) for i in compiler.get_exelist()] invoc = full_exe + compiler.get_exelist()
command = invoc + ['$ARGS', '$in'] command = invoc + ['$ARGS', '$in']
description = 'Compiling Swift source $in' description = 'Compiling Swift source $in'
self.add_rule(NinjaRule(rule, command, [], description)) self.add_rule(NinjaRule(rule, command, [], description))
@ -1735,8 +1787,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
if self.created_llvm_ir_rule[compiler.for_machine]: if self.created_llvm_ir_rule[compiler.for_machine]:
return return
rule = self.get_compiler_rule_name('llvm_ir', compiler.for_machine) rule = self.get_compiler_rule_name('llvm_ir', compiler.for_machine)
command = [ninja_quote(i) for i in compiler.get_exelist()] command = compiler.get_exelist()
args = ['$ARGS'] + compiler.get_output_args('$out') + compiler.get_compile_only_args() + ['$in'] args = ['$ARGS'] + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
description = 'Compiling LLVM IR object $in' description = 'Compiling LLVM IR object $in'
self.add_rule(NinjaRule(rule, command, args, description, self.add_rule(NinjaRule(rule, command, args, description,
rspable=compiler.can_linker_accept_rsp())) rspable=compiler.can_linker_accept_rsp()))
@ -1765,15 +1817,9 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
if langname == 'fortran': if langname == 'fortran':
self.generate_fortran_dep_hack(crstr) self.generate_fortran_dep_hack(crstr)
rule = self.get_compiler_rule_name(langname, compiler.for_machine) rule = self.get_compiler_rule_name(langname, compiler.for_machine)
depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE') depargs = NinjaCommandArg.list(compiler.get_dependency_gen_args('$out', '$DEPFILE'), Quoting.none)
quoted_depargs = [] command = compiler.get_exelist()
for d in depargs: args = ['$ARGS'] + depargs + NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none) + compiler.get_compile_only_args() + ['$in']
if d != '$out' and d != '$in':
d = quote_func(d)
quoted_depargs.append(d)
command = [ninja_quote(i) for i in compiler.get_exelist()]
args = ['$ARGS'] + quoted_depargs + compiler.get_output_args('$out') + compiler.get_compile_only_args() + ['$in']
description = 'Compiling {} object $out'.format(compiler.get_display_language()) description = 'Compiling {} object $out'.format(compiler.get_display_language())
if isinstance(compiler, VisualStudioLikeCompiler): if isinstance(compiler, VisualStudioLikeCompiler):
deps = 'msvc' deps = 'msvc'
@ -1791,16 +1837,11 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
rule = self.compiler_to_pch_rule_name(compiler) rule = self.compiler_to_pch_rule_name(compiler)
depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE') depargs = compiler.get_dependency_gen_args('$out', '$DEPFILE')
quoted_depargs = []
for d in depargs:
if d != '$out' and d != '$in':
d = quote_func(d)
quoted_depargs.append(d)
if isinstance(compiler, VisualStudioLikeCompiler): if isinstance(compiler, VisualStudioLikeCompiler):
output = [] output = []
else: else:
output = compiler.get_output_args('$out') output = NinjaCommandArg.list(compiler.get_output_args('$out'), Quoting.none)
command = compiler.get_exelist() + ['$ARGS'] + quoted_depargs + output + compiler.get_compile_only_args() + ['$in'] command = compiler.get_exelist() + ['$ARGS'] + depargs + output + compiler.get_compile_only_args() + ['$in']
description = 'Precompiling header $in' description = 'Precompiling header $in'
if isinstance(compiler, VisualStudioLikeCompiler): if isinstance(compiler, VisualStudioLikeCompiler):
deps = 'msvc' deps = 'msvc'

Write Preview
Loading…
Cancel
Save