Use universal_newlines=True for all Popen calls

Instead of adding it everywhere manually, create a wrapper called
mesonlib.Popen_safe and use that everywhere that we call an executable
and extract its output.

This will also allow us to tweak it to do more/different things if
needed for some locales and/or systems.

Closes #1079

mesonbuild/backend/ninjabackend.py

@@ -18,7 +18,7 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..mesonlib import File, MesonException, get_compiler_for_source
+from ..mesonlib import File, MesonException, get_compiler_for_source, Popen_safe
 from .backends import InstallData
 from ..build import InvalidArguments
 import os, sys, pickle, re
@@ -159,18 +159,14 @@ class NinjaBackend(backends.Backend):
 int dummy;
 ''')
 
-        pc = subprocess.Popen(['cl', '/showIncludes', '/c', 'incdetect.c'],
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE,
-                              cwd=self.environment.get_scratch_dir())
+        pc, stdo = Popen_safe(['cl', '/showIncludes', '/c', 'incdetect.c'],
+                              cwd=self.environment.get_scratch_dir())[0:2]
 
-        (stdo, _) = pc.communicate()
-
-        for line in stdo.split(b'\r\n'):
-            if line.endswith(b'stdio.h'):
-                matchstr = b':'.join(line.split(b':')[0:2]) + b':'
-                with open(tempfilename, 'ab') as binfile:
-                    binfile.write(b'msvc_deps_prefix = ' + matchstr + b'\r\n')
+        for line in stdo.split('\n'):
+            if line.endswith('stdio.h'):
+                matchstr = ':'.join(line.split(':')[0:2]) + ':'
+                with open(tempfilename, 'a') as binfile:
+                    binfile.write('msvc_deps_prefix = ' + matchstr + '\n')
                 return open(tempfilename, 'a')
         raise MesonException('Could not determine vs dep dependency prefix string.')
 

mesonbuild/compilers.py

@@ -18,7 +18,7 @@ import subprocess, os.path
 import tempfile
 from .import mesonlib
 from . import mlog
-from .mesonlib import MesonException, version_compare
+from .mesonlib import MesonException, version_compare, Popen_safe
 from . import coredata
 
 """This file contains the data files of all compilers Meson knows
@@ -457,12 +457,7 @@ class Compiler():
                 mlog.debug('Working directory: ', tmpdirname)
                 mlog.debug('Command line: ', ' '.join(commands), '\n')
                 mlog.debug('Code:\n', code)
-                p = subprocess.Popen(commands, cwd=tmpdirname,
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE)
-                (stde, stdo) = p.communicate()
-                stde = stde.decode()
-                stdo = stdo.decode()
+                p, stdo, stde = Popen_safe(commands, cwd=tmpdirname)
                 mlog.debug('Compiler stdout:\n', stdo)
                 mlog.debug('Compiler stderr:\n', stde)
 
@@ -600,9 +595,7 @@ class CCompiler(Compiler):
         return ['-shared']
 
     def get_library_dirs(self):
-        output = subprocess.Popen(self.exelist + ['--print-search-dirs'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
-        (stdo, _) = output.communicate()
-        stdo = stdo.decode('utf-8')
+        stdo = Popen_safe(self.exelist + ['--print-search-dirs'])[1]
         for line in stdo.split('\n'):
             if line.startswith('libraries:'):
                 libstr = line.split('=', 1)[1]
@@ -659,10 +652,7 @@ class CCompiler(Compiler):
             ofile.write(code)
         # Compile sanity check
         cmdlist = self.exelist + extra_flags + [source_name] + self.get_output_args(binary_name)
-        pc = subprocess.Popen(cmdlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir)
-        (stdo, stde) = pc.communicate()
-        stdo = stdo.decode()
-        stde = stde.decode()
+        pc, stdo, stde = Popen_safe(cmdlist, cwd=work_dir)
         mlog.debug('Sanity check compiler command line:', ' '.join(cmdlist))
         mlog.debug('Sanity check compile stdout:')
         mlog.debug(stdo)
@@ -806,15 +796,11 @@ int main () {{
             else:
                 cmdlist = p.output_name
             try:
-                pe = subprocess.Popen(cmdlist, stdout=subprocess.PIPE,
-                                      stderr=subprocess.PIPE)
+                pe, so, se = Popen_safe(cmdlist)
             except Exception as e:
                 mlog.debug('Could not run: %s (error: %s)\n' % (cmdlist, e))
                 return RunResult(False)
 
-            (so, se) = pe.communicate()
-        so = so.decode()
-        se = se.decode()
         mlog.debug('Program stdout:\n')
         mlog.debug(so)
         mlog.debug('Program stderr:\n')
@@ -1931,7 +1917,7 @@ class VisualStudioCCompiler(CCompiler):
     # understand and you can't tell it to error out on those.
     # http://stackoverflow.com/questions/15259720/how-can-i-make-the-microsoft-c-compiler-treat-unknown-flags-as-errors-rather-t
     def has_argument(self, arg, env):
-        warning_text = b'9002'
+        warning_text = '9002'
         code = 'int i;\n'
         (fd, srcname) = tempfile.mkstemp(suffix='.'+self.default_suffix)
         os.close(fd)
@@ -1944,8 +1930,7 @@ class VisualStudioCCompiler(CCompiler):
         mlog.debug('Running VS compile:')
         mlog.debug('Command line: ', ' '.join(commands))
         mlog.debug('Code:\n', code)
-        p = subprocess.Popen(commands, cwd=os.path.split(srcname)[0], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (stde, stdo) = p.communicate()
+        p, stdo, stde = Popen_safe(commands, cwd=os.path.split(srcname)[0])
         if p.returncode != 0:
             raise MesonException('Compiling test app failed.')
         return not(warning_text in stde or warning_text in stdo)
@@ -2614,10 +2599,9 @@ class ArLinker():
     def __init__(self, exelist):
         self.exelist = exelist
         self.id = 'ar'
-        pc = subprocess.Popen(self.exelist + ['-h'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
-        (stdo, _) = pc.communicate()
+        pc, stdo = Popen_safe(self.exelist + ['-h'])[0:2]
         # Enable deterministic builds if they are available.
-        if b'[D]' in stdo:
+        if '[D]' in stdo:
             self.std_args = ['csrD']
         else:
             self.std_args = ['csr']

mesonbuild/dependencies.py

@@ -23,7 +23,7 @@ import re
 import os, stat, glob, subprocess, shutil
 import sysconfig
 from collections import OrderedDict
-from . mesonlib import MesonException, version_compare, version_compare_many
+from . mesonlib import MesonException, version_compare, version_compare_many, Popen_safe
 from . import mlog
 from . import mesonlib
 from .environment import detect_cpu_family, for_windows
@@ -170,17 +170,14 @@ class PkgConfigDependency(Dependency):
         self._set_libs()
 
     def _call_pkgbin(self, args):
-        p = subprocess.Popen([self.pkgbin] + args,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             env=os.environ, universal_newlines=True)
-        out = p.communicate()[0]
+        p, out = Popen_safe([self.pkgbin] + args, env=os.environ)[0:2]
         return (p.returncode, out.strip())
 
     def _set_cargs(self):
         ret, out = self._call_pkgbin(['--cflags', self.name])
         if ret != 0:
             raise DependencyException('Could not generate cargs for %s:\n\n%s' % \
-                                      (self.name, out.decode(errors='ignore')))
+                                      (self.name, out))
         self.cargs = out.split()
 
     def _set_libs(self):
@@ -190,7 +187,7 @@ class PkgConfigDependency(Dependency):
         ret, out = self._call_pkgbin(libcmd)
         if ret != 0:
             raise DependencyException('Could not generate libs for %s:\n\n%s' % \
-                                      (self.name, out.decode(errors='ignore')))
+                                      (self.name, out))
         self.libs = []
         for lib in out.split():
             if lib.endswith(".la"):
@@ -238,13 +235,11 @@ class PkgConfigDependency(Dependency):
                 pkgbin = os.environ[evar].strip()
             else:
                 pkgbin = 'pkg-config'
-            p = subprocess.Popen([pkgbin, '--version'], stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
-            out = p.communicate()[0]
+            p, out = Popen_safe([pkgbin, '--version'])[0:2]
             if p.returncode == 0:
                 if not self.silent:
                     mlog.log('Found pkg-config:', mlog.bold(shutil.which(pkgbin)),
-                             '(%s)' % out.decode().strip())
+                             '(%s)' % out.strip())
                 PkgConfigDependency.pkgconfig_found = True
                 return
         except (FileNotFoundError, PermissionError):
@@ -303,16 +298,13 @@ class WxDependency(Dependency):
             mlog.log("Neither wx-config-3.0 nor wx-config found; can't detect dependency")
             return
 
-        p = subprocess.Popen([self.wxc, '--version'],
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        out = p.communicate()[0]
+        p, out = Popen_safe([self.wxc, '--version'])[0:2]
         if p.returncode != 0:
             mlog.log('Dependency wxwidgets found:', mlog.red('NO'))
             self.cargs = []
             self.libs = []
         else:
-            self.modversion = out.decode().strip()
+            self.modversion = out.strip()
             version_req = kwargs.get('version', None)
             if version_req is not None:
                 if not version_compare(self.modversion, version_req, strict=True):
@@ -324,20 +316,15 @@ class WxDependency(Dependency):
             self.requested_modules = self.get_requested(kwargs)
             # wx-config seems to have a cflags as well but since it requires C++,
             # this should be good, at least for now.
-            p = subprocess.Popen([self.wxc, '--cxxflags'],
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
-            out = p.communicate()[0]
+            p, out = Popen_safe([self.wxc, '--cxxflags'])[0:2]
             if p.returncode != 0:
                 raise DependencyException('Could not generate cargs for wxwidgets.')
-            self.cargs = out.decode().split()
+            self.cargs = out.split()
 
-            p = subprocess.Popen([self.wxc, '--libs'] + self.requested_modules,
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            out = p.communicate()[0]
+            p, out = Popen_safe([self.wxc, '--libs'] + self.requested_modules)[0:2]
             if p.returncode != 0:
                 raise DependencyException('Could not generate libs for wxwidgets.')
-            self.libs = out.decode().split()
+            self.libs = out.split()
 
     def get_requested(self, kwargs):
         modules = 'modules'
@@ -363,12 +350,10 @@ class WxDependency(Dependency):
     def check_wxconfig(self):
         for wxc in ['wx-config-3.0', 'wx-config']:
             try:
-                p = subprocess.Popen([wxc, '--version'], stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE)
-                out = p.communicate()[0]
+                p, out = Popen_safe([wxc, '--version'])[0:2]
                 if p.returncode == 0:
                     mlog.log('Found wx-config:', mlog.bold(shutil.which(wxc)),
-                             '(%s)' % out.decode().strip())
+                             '(%s)' % out.strip())
                     self.wxc = wxc
                     WxDependency.wx_found = True
                     return
@@ -943,10 +928,7 @@ class QtBaseDependency(Dependency):
             if not self.qmake.found():
                 continue
             # Check that the qmake is for qt5
-            pc = subprocess.Popen(self.qmake.fullpath +  ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                                  universal_newlines=True)
-            stdo = pc.communicate()[0]
+            pc, stdo = Popen_safe(self.qmake.fullpath +  ['-v'])[0:2]
             if pc.returncode != 0:
                 continue
             if not 'Qt version ' + self.qtver in stdo:
@@ -959,9 +941,7 @@ class QtBaseDependency(Dependency):
             return
         self.version = re.search(self.qtver + '(\.\d+)+', stdo).group(0)
         # Query library path, header path, and binary path
-        stdo = subprocess.Popen(self.qmake.fullpath +  ['-query'],
-                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                                universal_newlines=True).communicate()[0]
+        stdo = Popen_safe(self.qmake.fullpath +  ['-query'])[1]
         qvars = {}
         for line in stdo.split('\n'):
             line = line.strip()
@@ -1051,9 +1031,7 @@ class GnuStepDependency(Dependency):
     def detect(self):
         confprog = 'gnustep-config'
         try:
-            gp = subprocess.Popen([confprog, '--help'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            gp.communicate()
+            gp = Popen_safe([confprog, '--help'])[0]
         except (FileNotFoundError, PermissionError):
             self.args = None
             mlog.log('Dependency GnuStep found:', mlog.red('NO'), '(no gnustep-config)')
@@ -1066,20 +1044,12 @@ class GnuStepDependency(Dependency):
             arg = '--gui-libs'
         else:
             arg = '--base-libs'
-        fp = subprocess.Popen([confprog, '--objc-flags'],
-                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (flagtxt, flagerr) = fp.communicate()
-        flagtxt = flagtxt.decode()
-        flagerr = flagerr.decode()
+        fp, flagtxt, flagerr = Popen_safe([confprog, '--objc-flags'])
         if fp.returncode != 0:
             raise DependencyException('Error getting objc-args: %s %s' % (flagtxt, flagerr))
         args = flagtxt.split()
         self.args = self.filter_arsg(args)
-        fp = subprocess.Popen([confprog, arg],
-                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (libtxt, liberr) = fp.communicate()
-        libtxt = libtxt.decode()
-        liberr = liberr.decode()
+        fp, libtxt, liberr = Popen_safe([confprog, arg])
         if fp.returncode != 0:
             raise DependencyException('Error getting objc-lib args: %s %s' % (libtxt, liberr))
         self.libs = self.weird_filter(libtxt.split())
@@ -1184,16 +1154,10 @@ class SDL2Dependency(Dependency):
             pass
         sdlconf = shutil.which('sdl2-config')
         if sdlconf:
-            pc = subprocess.Popen(['sdl2-config', '--cflags'],
-                                  stdout=subprocess.PIPE,
-                                  stderr=subprocess.DEVNULL)
-            (stdo, _) = pc.communicate()
-            self.cargs = stdo.decode().strip().split()
-            pc = subprocess.Popen(['sdl2-config', '--libs'],
-                                  stdout=subprocess.PIPE,
-                                  stderr=subprocess.DEVNULL)
-            (stdo, _) = pc.communicate()
-            self.linkargs = stdo.decode().strip().split()
+            pc, stdo = Popen_safe(['sdl2-config', '--cflags'])[0:2]
+            self.cargs = stdo.strip().split()
+            pc, stdo = Popen_safe(['sdl2-config', '--libs'])[0:2]
+            self.linkargs = stdo.strip().split()
             self.is_found = True
             mlog.log('Dependency', mlog.bold('sdl2'), 'found:', mlog.green('YES'), '(%s)' % sdlconf)
             self.version = '2' # FIXME

mesonbuild/environment.py

@@ -17,6 +17,7 @@ from . import coredata
 from . import mesonlib
 from . import mlog
 from .compilers import *
+from .mesonlib import Popen_safe
 import configparser
 import shutil
 
@@ -42,11 +43,10 @@ def find_coverage_tools():
 def detect_ninja():
     for n in ['ninja', 'ninja-build']:
         try:
-            p = subprocess.Popen([n, '--version'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+            p, version = Popen_safe([n, '--version'])[0:2]
         except (FileNotFoundError, PermissionError):
             # Doesn't exist in PATH or isn't executable
             continue
-        version = p.communicate()[0].decode(errors='ignore')
         # Perhaps we should add a way for the caller to know the failure mode
         # (not found or too old)
         if p.returncode == 0 and mesonlib.version_compare(version, ">=1.6"):
@@ -306,9 +306,7 @@ class Environment():
         # Arguments to output compiler pre-processor defines to stdout
         # gcc, g++, and gfortran all support these arguments
         args = compiler + ['-E', '-dM', '-']
-        p = subprocess.Popen(args, universal_newlines=True,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        output = p.communicate('')[0]
+        p, output = Popen_safe(args, write='', stdin=subprocess.PIPE)[0:2]
         if p.returncode != 0:
             raise EnvironmentException('Unable to detect GNU compiler type:\n' + output)
         # Parse several lines of the type:
@@ -372,14 +370,10 @@ class Environment():
                     arg = '/?'
                 else:
                     arg = '--version'
-                p = subprocess.Popen([compiler, arg], stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE)
+                p, out, err = Popen_safe([compiler, arg])
             except OSError as e:
                 popen_exceptions[' '.join([compiler, arg])] = e
                 continue
-            (out, err) = p.communicate()
-            out = out.decode(errors='ignore')
-            err = err.decode(errors='ignore')
             version = search_version(out)
             if 'Free Software Foundation' in out:
                 defines = self.get_gnu_compiler_defines([compiler])
@@ -428,15 +422,10 @@ class Environment():
         for compiler in compilers:
             for arg in ['--version', '-V']:
                 try:
-                    p = subprocess.Popen([compiler, arg],
-                                         stdout=subprocess.PIPE,
-                                         stderr=subprocess.PIPE)
+                    p, out, err = Popen_safe([compiler, arg])
                 except OSError as e:
                     popen_exceptions[' '.join([compiler, arg])] = e
                     continue
-                (out, err) = p.communicate()
-                out = out.decode(errors='ignore')
-                err = err.decode(errors='ignore')
 
                 version = search_version(out)
 
@@ -512,15 +501,10 @@ class Environment():
             else:
                 arg = '--version'
             try:
-                p = subprocess.Popen([compiler, arg],
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE)
+                p, out, err = Popen_safe([compiler, arg])
             except OSError as e:
                 popen_exceptions[' '.join([compiler, arg])] = e
                 continue
-            (out, err) = p.communicate()
-            out = out.decode(errors='ignore')
-            err = err.decode(errors='ignore')
             version = search_version(out)
             if 'Free Software Foundation' in out:
                 defines = self.get_gnu_compiler_defines([compiler])
@@ -559,12 +543,9 @@ class Environment():
             is_cross = False
             exe_wrap = None
         try:
-            p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out, err = Popen_safe(exelist + ['--version'])
         except OSError:
             raise EnvironmentException('Could not execute ObjC compiler "%s"' % ' '.join(exelist))
-        (out, err) = p.communicate()
-        out = out.decode(errors='ignore')
-        err = err.decode(errors='ignore')
         version = search_version(out)
         if 'Free Software Foundation' in out:
             defines = self.get_gnu_compiler_defines(exelist)
@@ -587,12 +568,9 @@ class Environment():
             is_cross = False
             exe_wrap = None
         try:
-            p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out, err = Popen_safe(exelist + ['--version'])
         except OSError:
             raise EnvironmentException('Could not execute ObjC++ compiler "%s"' % ' '.join(exelist))
-        (out, err) = p.communicate()
-        out = out.decode(errors='ignore')
-        err = err.decode(errors='ignore')
         version = search_version(out)
         if 'Free Software Foundation' in out:
             defines = self.get_gnu_compiler_defines(exelist)
@@ -604,12 +582,9 @@ class Environment():
     def detect_java_compiler(self):
         exelist = ['javac']
         try:
-            p = subprocess.Popen(exelist + ['-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out, err = Popen_safe(exelist + ['-version'])
         except OSError:
             raise EnvironmentException('Could not execute Java compiler "%s"' % ' '.join(exelist))
-        (out, err) = p.communicate()
-        out = out.decode(errors='ignore')
-        err = err.decode(errors='ignore')
         version = search_version(err)
         if 'javac' in err:
             return JavaCompiler(exelist, version)
@@ -618,12 +593,9 @@ class Environment():
     def detect_cs_compiler(self):
         exelist = ['mcs']
         try:
-            p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out, err = Popen_safe(exelist + ['--version'])
         except OSError:
             raise EnvironmentException('Could not execute C# compiler "%s"' % ' '.join(exelist))
-        (out, err) = p.communicate()
-        out = out.decode(errors='ignore')
-        err = err.decode(errors='ignore')
         version = search_version(out)
         if 'Mono' in out:
             return MonoCompiler(exelist, version)
@@ -632,11 +604,9 @@ class Environment():
     def detect_vala_compiler(self):
         exelist = ['valac']
         try:
-            p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out = Popen_safe(exelist + ['--version'])[0:2]
         except OSError:
             raise EnvironmentException('Could not execute Vala compiler "%s"' % ' '.join(exelist))
-        (out, _) = p.communicate()
-        out = out.decode(errors='ignore')
         version = search_version(out)
         if 'Vala' in out:
             return ValaCompiler(exelist, version)
@@ -645,11 +615,9 @@ class Environment():
     def detect_rust_compiler(self):
         exelist = ['rustc']
         try:
-            p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out = Popen_safe(exelist + ['--version'])[0:2]
         except OSError:
             raise EnvironmentException('Could not execute Rust compiler "%s"' % ' '.join(exelist))
-        (out, _) = p.communicate()
-        out = out.decode(errors='ignore')
         version = search_version(out)
         if 'rustc' in out:
             return RustCompiler(exelist, version)
@@ -679,11 +647,9 @@ class Environment():
             raise EnvironmentException('Could not find any supported D compiler.')
 
         try:
-            p = subprocess.Popen(exelist + ['--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out = Popen_safe(exelist + ['--version'])[0:2]
         except OSError:
             raise EnvironmentException('Could not execute D compiler "%s"' % ' '.join(exelist))
-        (out, _) = p.communicate()
-        out = out.decode(errors='ignore')
         version = search_version(out)
         if 'LLVM D compiler' in out:
             return LLVMDCompiler(exelist, version, is_cross)
@@ -696,11 +662,9 @@ class Environment():
     def detect_swift_compiler(self):
         exelist = ['swiftc']
         try:
-            p = subprocess.Popen(exelist + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, _, err = Popen_safe(exelist + ['-v'])
         except OSError:
             raise EnvironmentException('Could not execute Swift compiler "%s"' % ' '.join(exelist))
-        (_, err) = p.communicate()
-        err = err.decode(errors='ignore')
         version = search_version(err)
         if 'Swift' in err:
             return SwiftCompiler(exelist, version)
@@ -723,12 +687,9 @@ class Environment():
         else:
             arg = '--version'
         try:
-            p = subprocess.Popen([linker, arg], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            p, out, err = Popen_safe([linker, arg])
         except OSError:
             raise EnvironmentException('Could not execute static linker "%s".' % linker)
-        (out, err) = p.communicate()
-        out = out.decode(errors='ignore')
-        err = err.decode(errors='ignore')
         if '/OUT:' in out or '/OUT:' in err:
             return VisualStudioLinker([linker])
         if p.returncode == 0:

mesonbuild/interpreter.py

@@ -22,6 +22,7 @@ from . import optinterpreter
 from . import compilers
 from .wrap import wrap
 from . import mesonlib
+from .mesonlib import Popen_safe
 from .dependencies import InternalDependency, Dependency
 from .interpreterbase import InterpreterBase
 from .interpreterbase import check_stringlist, noPosargs, noKwargs, stringArgs
@@ -70,17 +71,8 @@ class RunProcess(InterpreterObject):
 
     def __init__(self, command_array, source_dir, build_dir, subdir, in_builddir=False):
         super().__init__()
-        pc = self.run_command(command_array, source_dir, build_dir, subdir, in_builddir)
-        (stdout, stderr) = pc.communicate()
+        pc, self.stdout, self.stderr = self.run_command(command_array, source_dir, build_dir, subdir, in_builddir)
         self.returncode = pc.returncode
-        if sys.stdout.encoding:
-            self.stdout = stdout.decode(encoding=sys.stdout.encoding, errors='ignore').replace('\r\n', '\n')
-        else:
-            self.stdout = stdout.decode(errors='ignore').replace('\r\n', '\n')
-        if sys.stderr.encoding:
-            self.stderr = stderr.decode(encoding=sys.stderr.encoding, errors='ignore').replace('\r\n', '\n')
-        else:
-            self.stderr = stderr.decode(errors='ignore').replace('\r\n', '\n')
         self.methods.update({'returncode' : self.returncode_method,
                              'stdout' : self.stdout_method,
                              'stderr' : self.stderr_method,
@@ -99,22 +91,19 @@ class RunProcess(InterpreterObject):
         child_env.update(env)
         mlog.debug('Running command:', ' '.join(command_array))
         try:
-            return subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                    env=child_env, cwd=cwd)
+            return Popen_safe(command_array, env=child_env, cwd=cwd)
         except FileNotFoundError:
             pass
         # Was not a command, is a program in path?
         exe = shutil.which(cmd_name)
         if exe is not None:
             command_array = [exe] + command_array[1:]
-            return subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                    env=child_env, cwd=cwd)
+            return Popen_safe(command_array, env=child_env, cwd=cwd)
         # No? Maybe it is a script in the source tree.
         fullpath = os.path.join(source_dir, subdir, cmd_name)
         command_array = [fullpath] + command_array[1:]
         try:
-            return subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                    env=child_env, cwd=cwd)
+            return Popen_safe(command_array, env=child_env, cwd=cwd)
         except FileNotFoundError:
             raise InterpreterException('Could not execute command "%s".' % cmd_name)
 

mesonbuild/mesonlib.py

@@ -385,3 +385,10 @@ def expand_arguments(args):
             print(e)
             return None
     return expended_args
+
+def Popen_safe(args, write=None, stderr=subprocess.PIPE, **kwargs):
+    p = subprocess.Popen(args, universal_newlines=True,
+                         stdout=subprocess.PIPE,
+                         stderr=stderr, **kwargs)
+    o, e = p.communicate(write)
+    return (p, o, e)

mesonbuild/modules/gnome.py

@@ -20,7 +20,7 @@ import os
 import sys
 import copy
 import subprocess
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, Popen_safe
 from .. import dependencies
 from .. import mlog
 from .. import mesonlib
@@ -197,9 +197,7 @@ can not be used with the current version of glib-compiled-resources, due to
             cmd += ['--sourcedir', os.path.join(state.subdir, source_dir)]
         cmd += ['--sourcedir', state.subdir] # Current dir
 
-        pc = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True,
-                              cwd=state.environment.get_source_dir())
-        (stdout, _) = pc.communicate()
+        pc, stdout = Popen_safe(cmd, cwd=state.environment.get_source_dir())[0:2]
         if pc.returncode != 0:
             mlog.warning('glib-compile-resources has failed to get the dependencies for {}'.format(cmd[1]))
             raise subprocess.CalledProcessError(pc.returncode, cmd)

mesonbuild/modules/qt4.py

@@ -15,7 +15,7 @@
 import os, subprocess
 from .. import mlog
 from .. import build
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, Popen_safe
 from ..dependencies import Qt4Dependency
 import xml.etree.ElementTree as ET
 
@@ -37,11 +37,9 @@ class Qt4Module():
         # Moc and rcc return a non-zero result when doing so.
         # What kind of an idiot thought that was a good idea?
         if self.moc.found():
-            mp = subprocess.Popen(self.moc.get_command() + ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdout, stderr) = mp.communicate()
-            stdout = stdout.decode().strip()
-            stderr = stderr.decode().strip()
+            stdout, stderr = Popen_safe(self.moc.get_command() + ['-v'])[1:3]
+            stdout = stdout.strip()
+            stderr = stderr.strip()
             if 'Qt Meta' in stderr:
                 moc_ver = stderr
             else:
@@ -52,11 +50,9 @@ class Qt4Module():
         else:
             mlog.log(' moc:', mlog.red('NO'))
         if self.uic.found():
-            up = subprocess.Popen(self.uic.get_command() + ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdout, stderr) = up.communicate()
-            stdout = stdout.decode().strip()
-            stderr = stderr.decode().strip()
+            stdout, stderr = Popen_safe(self.uic.get_command() + ['-v'])[1:3]
+            stdout = stdout.strip()
+            stderr = stderr.strip()
             if 'version 4.' in stderr:
                 uic_ver = stderr
             else:
@@ -67,11 +63,9 @@ class Qt4Module():
         else:
             mlog.log(' uic:', mlog.red('NO'))
         if self.rcc.found():
-            rp = subprocess.Popen(self.rcc.get_command() + ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdout, stderr) = rp.communicate()
-            stdout = stdout.decode().strip()
-            stderr = stderr.decode().strip()
+            stdout, stderr = Popen_safe(self.rcc.get_command() + ['-v'])[1:3]
+            stdout = stdout.strip()
+            stderr = stderr.strip()
             if 'version 4.' in stderr:
                 rcc_ver = stderr
             else:

mesonbuild/modules/qt5.py

@@ -15,7 +15,7 @@
 import os, subprocess
 from .. import mlog
 from .. import build
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, Popen_safe
 from ..dependencies import Qt5Dependency
 import xml.etree.ElementTree as ET
 
@@ -37,11 +37,9 @@ class Qt5Module():
         # Moc and rcc return a non-zero result when doing so.
         # What kind of an idiot thought that was a good idea?
         if self.moc.found():
-            mp = subprocess.Popen(self.moc.get_command() + ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdout, stderr) = mp.communicate()
-            stdout = stdout.decode().strip()
-            stderr = stderr.decode().strip()
+            stdout, stderr = Popen_safe(self.moc.get_command() + ['-v'])[1:3]
+            stdout = stdout.strip()
+            stderr = stderr.strip()
             if 'Qt 5' in stderr:
                 moc_ver = stderr
             elif '5.' in stdout:
@@ -54,11 +52,9 @@ class Qt5Module():
         else:
             mlog.log(' moc:', mlog.red('NO'))
         if self.uic.found():
-            up = subprocess.Popen(self.uic.get_command() + ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdout, stderr) = up.communicate()
-            stdout = stdout.decode().strip()
-            stderr = stderr.decode().strip()
+            stdout, stderr = Popen_safe(self.uic.get_command() + ['-v'])[1:3]
+            stdout = stdout.strip()
+            stderr = stderr.strip()
             if 'version 5.' in stderr:
                 uic_ver = stderr
             elif '5.' in stdout:
@@ -71,11 +67,9 @@ class Qt5Module():
         else:
             mlog.log(' uic:', mlog.red('NO'))
         if self.rcc.found():
-            rp = subprocess.Popen(self.rcc.get_command() + ['-v'],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdout, stderr) = rp.communicate()
-            stdout = stdout.decode().strip()
-            stderr = stderr.decode().strip()
+            stdout, stderr = Popen_safe(self.rcc.get_command() + ['-v'])[1:3]
+            stdout = stdout.strip()
+            stderr = stderr.strip()
             if 'version 5.' in stderr:
                 rcc_ver = stderr
             elif '5.' in stdout:

mesonbuild/scripts/gtkdochelper.py

@@ -17,7 +17,7 @@ import sys, os
 import subprocess
 import shutil
 import argparse
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, Popen_safe
 from . import destdir_join
 
 parser = argparse.ArgumentParser()
@@ -46,15 +46,13 @@ parser.add_argument('--mode', dest='mode', default='')
 parser.add_argument('--installdir', dest='install_dir')
 
 def gtkdoc_run_check(cmd, cwd):
-    p = subprocess.Popen(cmd, cwd=cwd,
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stde, stdo) = p.communicate()
+    # Put stderr into stdout since we want to print it out anyway.
+    # This preserves the order of messages.
+    p, out = Popen_safe(cmd, cwd=cwd, stderr=subprocess.STDOUT)[0:2]
     if p.returncode != 0:
         err_msg = ["{!r} failed with status {:d}".format(cmd[0], p.returncode)]
-        if stde:
-            err_msg.append(stde.decode(errors='ignore'))
-        if stdo:
-            err_msg.append(stdo.decode(errors='ignore'))
+        if out:
+            err_msg.append(out)
         raise MesonException('\n'.join(err_msg))
 
 def build_gtkdoc(source_root, build_root, doc_subdir, src_subdirs,

mesonbuild/scripts/meson_exe.py

@@ -21,7 +21,7 @@ import pickle
 import platform
 import subprocess
 
-import mesonbuild
+from ..mesonlib import MesonException, Popen_safe
 
 options = None
 
@@ -45,7 +45,7 @@ def run_exe(exe):
     else:
         if exe.is_cross:
             if exe.exe_runner is None:
-                raise Exception('BUG: Trying to run cross-compiled exes with no wrapper')
+                raise AssertionError('BUG: Trying to run cross-compiled exes with no wrapper')
             else:
                 cmd = [exe.exe_runner] + exe.fname
         else:
@@ -55,17 +55,12 @@ def run_exe(exe):
     if len(exe.extra_paths) > 0:
         child_env['PATH'] = (os.pathsep.join(exe.extra_paths + ['']) +
                              child_env['PATH'])
-    p = subprocess.Popen(cmd + exe.cmd_args,
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE,
-                         env=child_env,
-                         cwd=exe.workdir)
-    stdout, stderr = p.communicate()
+    p, stdout, stderr = Popen_safe(cmd + exe.cmd_args, env=child_env, cwd=exe.workdir)
     if exe.capture and p.returncode == 0:
-        with open(exe.capture, 'wb') as output:
+        with open(exe.capture, 'w') as output:
             output.write(stdout)
     if stderr:
-        sys.stderr.buffer.write(stderr)
+        sys.stderr.write(stderr)
     return p.returncode
 
 def run(args):

mesonbuild/scripts/meson_install.py

@@ -18,6 +18,7 @@ import sys, pickle, os, shutil, subprocess, gzip, platform
 from glob import glob
 from . import depfixer
 from . import destdir_join
+from ..mesonlib import MesonException, Popen_safe
 
 install_log_file = None
 
@@ -205,12 +206,11 @@ def install_targets(d):
         do_copy(fname, outname)
         if should_strip:
             print('Stripping target')
-            ps = subprocess.Popen(['strip', outname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (stdo, stde) = ps.communicate()
+            ps, stdo, stde = Popen_safe(['strip', outname])
             if ps.returncode != 0:
                 print('Could not strip file.\n')
-                print('Stdout:\n%s\n' % stdo.decode())
-                print('Stderr:\n%s\n' % stde.decode())
+                print('Stdout:\n%s\n' % stdo)
+                print('Stderr:\n%s\n' % stde)
                 sys.exit(1)
         printed_symlink_error = False
         for alias in aliases:

mesonbuild/scripts/symbolextractor.py

@@ -23,7 +23,8 @@
 # http://cgit.freedesktop.org/libreoffice/core/commit/?id=3213cd54b76bc80a6f0516aac75a48ff3b2ad67c
 
 import os, sys, subprocess
-from mesonbuild import mesonlib
+from .. import mesonlib
+from ..mesonlib import MesonException, Popen_safe
 import argparse
 
 parser = argparse.ArgumentParser()
@@ -59,23 +60,21 @@ def linux_syms(libfilename, outfilename):
         nmbin = os.environ[evar].strip()
     else:
         nmbin = 'nm'
-    pe = subprocess.Popen([readelfbin, '-d', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pe.communicate()[0].decode()
+    pe, output = Popen_safe([readelfbin, '-d', libfilename])[0:2]
     if pe.returncode != 0:
         raise RuntimeError('Readelf does not work')
     result = [x for x in output.split('\n') if 'SONAME' in x]
     assert(len(result) <= 1)
-    pnm = subprocess.Popen([nmbin, '--dynamic', '--extern-only', '--defined-only', '--format=posix', libfilename],
-                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pnm.communicate()[0].decode()
+    pnm, output = Popen_safe([nmbin, '--dynamic', '--extern-only',
+                              '--defined-only', '--format=posix',
+                              libfilename])[0:2]
     if pnm.returncode != 0:
         raise RuntimeError('nm does not work.')
     result += [' '.join(x.split()[0:2]) for x in output.split('\n') if len(x) > 0]
     write_if_changed('\n'.join(result) + '\n', outfilename)
 
 def osx_syms(libfilename, outfilename):
-    pe = subprocess.Popen(['otool', '-l', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pe.communicate()[0].decode()
+    pe, output = Popen_safe(['otool', '-l', libfilename])[0:2]
     if pe.returncode != 0:
         raise RuntimeError('Otool does not work.')
     arr = output.split('\n')
@@ -84,8 +83,7 @@ def osx_syms(libfilename, outfilename):
             match = i
             break
     result = [arr[match+2], arr[match+5]] # Libreoffice stores all 5 lines but the others seem irrelevant.
-    pnm = subprocess.Popen(['nm', '-g', '-P', libfilename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    output = pnm.communicate()[0].decode()
+    pnm, output = Popen_safe(['nm', '-g', '-P', libfilename])[0:2]
     if pnm.returncode != 0:
         raise RuntimeError('nm does not work.')
     result += [' '.join(x.split()[0:2]) for x in output.split('\n') if len(x) > 0 and not x.endswith('U')]