Harmonize data pickling.

Try to be more consistent on using save() and load() methods to pickle
data.

mesonbuild/backend/ninjabackend.py

@@ -2691,3 +2691,9 @@ rule FORTRAN_DEP_HACK
 
         elem = NinjaBuildElement(self.all_outputs, deps, 'phony', '')
         elem.write(outfile)
+
+def load(build_dir):
+    filename = os.path.join(build_dir, 'meson-private', 'install.dat')
+    with open(filename, 'rb') as f:
+        obj = pickle.load(f)
+    return obj

mesonbuild/build.py

@@ -15,6 +15,7 @@
 import copy, os, re
 from collections import OrderedDict
 import itertools, pathlib
+import pickle
 
 from . import environment
 from . import dependencies
@@ -1928,3 +1929,22 @@ def get_sources_string_names(sources):
         else:
             raise AssertionError('Unknown source type: {!r}'.format(s))
     return names
+
+def load(build_dir):
+    filename = os.path.join(build_dir, 'meson-private', 'build.dat')
+    load_fail_msg = 'Build data file {!r} is corrupted. Try with a fresh build tree.'.format(filename)
+    nonexisting_fail_msg = 'No such build data file as "{!r}".'.format(filename)
+    try:
+        with open(filename, 'rb') as f:
+            obj = pickle.load(f)
+    except FileNotFoundError:
+        raise MesonException(nonexisting_fail_msg)
+    except pickle.UnpicklingError:
+        raise MesonException(load_fail_msg)
+    if not isinstance(obj, Build):
+        raise MesonException(load_fail_msg)
+    return obj
+
+def save(obj, filename):
+    with open(filename, 'wb') as f:
+        pickle.dump(obj, f)

mesonbuild/coredata.py

@@ -340,7 +340,8 @@ class CoreData:
                 return opt.validate_value(override_value)
         raise MesonException('Tried to validate unknown option %s.' % option_name)
 
-def load(filename):
+def load(build_dir):
+    filename = os.path.join(build_dir, 'meson-private', 'coredata.dat')
     load_fail_msg = 'Coredata file {!r} is corrupted. Try with a fresh build tree.'.format(filename)
     try:
         with open(filename, 'rb') as f:
@@ -354,7 +355,8 @@ def load(filename):
                              (obj.version, version))
     return obj
 
-def save(obj, filename):
+def save(obj, build_dir):
+    filename = os.path.join(build_dir, 'meson-private', 'coredata.dat')
     if obj.version != version:
         raise MesonException('Fatal version mismatch corruption.')
     with open(filename, 'wb') as f:

mesonbuild/environment.py

@@ -251,8 +251,7 @@ class Environment:
         os.makedirs(self.scratch_dir, exist_ok=True)
         os.makedirs(self.log_dir, exist_ok=True)
         try:
-            cdf = os.path.join(self.get_build_dir(), Environment.coredata_file)
-            self.coredata = coredata.load(cdf)
+            self.coredata = coredata.load(self.get_build_dir())
             self.first_invocation = False
         except FileNotFoundError:
             # WARNING: Don't use any values from coredata in __init__. It gets
@@ -315,9 +314,8 @@ class Environment:
         return self.cross_info is not None
 
     def dump_coredata(self):
-        cdf = os.path.join(self.get_build_dir(), Environment.coredata_file)
-        coredata.save(self.coredata, cdf)
-        return cdf
+        coredata.save(self.coredata, self.get_build_dir())
+        return os.path.join(self.get_build_dir(), Environment.coredata_file)
 
     def get_script_dir(self):
         import mesonbuild.scripts

mesonbuild/mconf.py

@@ -13,9 +13,8 @@
 # limitations under the License.
 
 import sys, os
-import pickle
 import argparse
-from . import coredata, mesonlib
+from . import coredata, mesonlib, build
 
 parser = argparse.ArgumentParser(prog='meson configure')
 
@@ -31,25 +30,17 @@ class ConfException(mesonlib.MesonException):
 class Conf:
     def __init__(self, build_dir):
         self.build_dir = build_dir
-        self.coredata_file = os.path.join(build_dir, 'meson-private/coredata.dat')
-        self.build_file = os.path.join(build_dir, 'meson-private/build.dat')
-        if not os.path.isfile(self.coredata_file) or not os.path.isfile(self.build_file):
+        if not os.path.isdir(os.path.join(build_dir, 'meson-private')):
             raise ConfException('Directory %s does not seem to be a Meson build directory.' % build_dir)
-        with open(self.coredata_file, 'rb') as f:
-            self.coredata = pickle.load(f)
-        with open(self.build_file, 'rb') as f:
-            self.build = pickle.load(f)
-        if self.coredata.version != coredata.version:
-            raise ConfException('Version mismatch (%s vs %s)' %
-                                (coredata.version, self.coredata.version))
+        self.build = build.load(self.build_dir)
+        self.coredata = coredata.load(self.build_dir)
 
     def clear_cache(self):
         self.coredata.deps = {}
 
     def save(self):
         # Only called if something has changed so overwrite unconditionally.
-        with open(self.coredata_file, 'wb') as f:
-            pickle.dump(self.coredata, f)
+        coredata.save(self.coredata, self.build_dir)
         # We don't write the build file because any changes to it
         # are erased when Meson is executed the next time, i.e. when
         # Ninja is run.

mesonbuild/mesonmain.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys, stat, traceback, pickle, argparse
-import time, datetime
+import sys, stat, traceback, argparse
+import datetime
 import os.path
 from . import environment, interpreter, mesonlib
 from . import build
@@ -196,6 +196,7 @@ class MesonApp:
         mlog.log('Build machine cpu:', mlog.bold(intr.builtin['build_machine'].cpu_method([], {})))
         intr.run()
         try:
+            dumpfile = os.path.join(env.get_scratch_dir(), 'build.dat')
             # We would like to write coredata as late as possible since we use the existence of
             # this file to check if we generated the build file successfully. Since coredata
             # includes settings, the build files must depend on it and appear newer. However, due
@@ -204,16 +205,13 @@ class MesonApp:
             # possible, but before build files, and if any error occurs, delete it.
             cdf = env.dump_coredata()
             g.generate(intr)
-            dumpfile = os.path.join(env.get_scratch_dir(), 'build.dat')
-            with open(dumpfile, 'wb') as f:
-                pickle.dump(b, f)
+            build.save(b, dumpfile)
             # Post-conf scripts must be run after writing coredata or else introspection fails.
             g.run_postconf_scripts()
         except:
             os.unlink(cdf)
             raise
 
-
 def run_script_command(args):
     cmdname = args[0]
     cmdargs = args[1:]

mesonbuild/mintro.py

@@ -19,8 +19,9 @@ tests and so on. All output is in JSON for simple parsing.
 Currently only works for the Ninja backend. Others use generated
 project files and don't need this info."""
 
-import json, pickle
-from . import coredata, build
+import json
+from . import build, mtest, coredata as cdata
+from .backend import ninjabackend
 import argparse
 import sys, os
 import pathlib
@@ -132,16 +133,16 @@ def add_keys(optlist, options):
     for key in keys:
         opt = options[key]
         optdict = {'name': key, 'value': opt.value}
-        if isinstance(opt, coredata.UserStringOption):
+        if isinstance(opt, cdata.UserStringOption):
             typestr = 'string'
-        elif isinstance(opt, coredata.UserBooleanOption):
+        elif isinstance(opt, cdata.UserBooleanOption):
             typestr = 'boolean'
-        elif isinstance(opt, coredata.UserComboOption):
+        elif isinstance(opt, cdata.UserComboOption):
             optdict['choices'] = opt.choices
             typestr = 'combo'
-        elif isinstance(opt, coredata.UserIntegerOption):
+        elif isinstance(opt, cdata.UserIntegerOption):
             typestr = 'integer'
-        elif isinstance(opt, coredata.UserArrayOption):
+        elif isinstance(opt, cdata.UserArrayOption):
             typestr = 'array'
         else:
             raise RuntimeError("Unknown option type")
@@ -149,7 +150,7 @@ def add_keys(optlist, options):
         optdict['description'] = opt.description
         optlist.append(optdict)
 
-def list_buildsystem_files(coredata, builddata):
+def list_buildsystem_files(builddata):
     src_dir = builddata.environment.get_source_dir()
     # I feel dirty about this. But only slightly.
     filelist = []
@@ -208,26 +209,15 @@ def run(args):
               'change the working directory to it.')
         return 1
 
-    corefile = os.path.join(datadir, 'coredata.dat')
-    buildfile = os.path.join(datadir, 'build.dat')
-    installfile = os.path.join(datadir, 'install.dat')
-    testfile = os.path.join(datadir, 'meson_test_setup.dat')
-    benchmarkfile = os.path.join(datadir, 'meson_benchmark_setup.dat')
+    coredata = cdata.load(options.builddir)
+    builddata = build.load(options.builddir)
+    testdata = mtest.load_tests(options.builddir)
+    benchmarkdata = mtest.load_benchmarks(options.builddir)
 
-    # Load all data files
-    with open(corefile, 'rb') as f:
-        coredata = pickle.load(f)
-    with open(buildfile, 'rb') as f:
-        builddata = pickle.load(f)
-    with open(testfile, 'rb') as f:
-        testdata = pickle.load(f)
-    with open(benchmarkfile, 'rb') as f:
-        benchmarkdata = pickle.load(f)
     # Install data is only available with the Ninja backend
-    if os.path.isfile(installfile):
-        with open(installfile, 'rb') as f:
-            installdata = pickle.load(f)
-    else:
+    try:
+        installdata = ninjabackend.load(options.builddir)
+    except FileNotFoundError:
         installdata = None
 
     if options.list_targets:
@@ -237,7 +227,7 @@ def run(args):
     elif options.target_files is not None:
         list_target_files(options.target_files, coredata, builddata)
     elif options.buildsystem_files:
-        list_buildsystem_files(coredata, builddata)
+        list_buildsystem_files(builddata)
     elif options.buildoptions:
         list_buildoptions(coredata, builddata)
     elif options.tests:

mesonbuild/mtest.py

@@ -165,6 +165,22 @@ def run_with_mono(fname):
         return True
     return False
 
+def load_benchmarks(build_dir):
+    datafile = os.path.join(build_dir, 'meson-private', 'meson_benchmark_setup.dat')
+    if not os.path.isfile(datafile):
+        raise TestException('Directory ${!r} does not seem to be a Meson build directory.'.format(build_dir))
+    with open(datafile, 'rb') as f:
+        obj = pickle.load(f)
+    return obj
+
+def load_tests(build_dir):
+    datafile = os.path.join(build_dir, 'meson-private', 'meson_test_setup.dat')
+    if not os.path.isfile(datafile):
+        raise TestException('Directory ${!r} does not seem to be a Meson build directory.'.format(build_dir))
+    with open(datafile, 'rb') as f:
+        obj = pickle.load(f)
+    return obj
+
 class TestHarness:
     def __init__(self, options):
         self.options = options
@@ -180,12 +196,10 @@ class TestHarness:
         self.logfile = None
         self.jsonlogfile = None
         if self.options.benchmark:
-            datafile = os.path.join(options.wd, 'meson-private', 'meson_benchmark_setup.dat')
+            self.tests = load_benchmarks(options.wd)
         else:
-            datafile = os.path.join(options.wd, 'meson-private', 'meson_test_setup.dat')
-        if not os.path.isfile(datafile):
-            raise TestException('Directory %s does not seem to be a Meson build directory.' % options.wd)
-        self.load_datafile(datafile)
+            self.tests = load_tests(options.wd)
+        self.load_suites()
 
     def __del__(self):
         if self.logfile:
@@ -374,9 +388,6 @@ TIMEOUT: %4d
     def doit(self):
         if self.is_run:
             raise RuntimeError('Test harness object can only be used once.')
-        if not os.path.isfile(self.datafile):
-            print('Test data file. Probably this means that you did not run this in the build directory.')
-            return 1
         self.is_run = True
         tests = self.get_tests()
         if not tests:
@@ -415,15 +426,6 @@ TIMEOUT: %4d
                 ss.add(s)
         self.suites = list(ss)
 
-    def load_tests(self):
-        with open(self.datafile, 'rb') as f:
-            self.tests = pickle.load(f)
-
-    def load_datafile(self, datafile):
-        self.datafile = datafile
-        self.load_tests()
-        self.load_suites()
-
     def get_tests(self):
         if not self.tests:
             print('No tests defined.')
@@ -559,18 +561,16 @@ def list_tests(th):
         print(th.get_pretty_suite(t))
 
 def merge_suite_options(options, test):
-    buildfile = os.path.join(options.wd, 'meson-private/build.dat')
-    with open(buildfile, 'rb') as f:
-        build = pickle.load(f)
+    bld = build.load(options.wd)
     if ":" in options.setup:
-        if options.setup not in build.test_setups:
+        if options.setup not in bld.test_setups:
             sys.exit("Unknown test setup '%s'." % options.setup)
-        current = build.test_setups[options.setup]
+        current = bld.test_setups[options.setup]
     else:
         full_name = test.project_name + ":" + options.setup
-        if full_name not in build.test_setups:
+        if full_name not in bld.test_setups:
             sys.exit("Test setup '%s' not found from project '%s'." % (options.setup, test.project_name))
-        current = build.test_setups[full_name]
+        current = bld.test_setups[full_name]
     if not options.gdb:
         options.gdb = current.gdb
     if options.timeout_multiplier is None: