Split the monolithic introspection file into chunks

mesonbuild/environment.py

@@ -308,6 +308,7 @@ def search_version(text):
 class Environment:
     private_dir = 'meson-private'
     log_dir = 'meson-logs'
+    info_dir = 'meson-info'
 
     def __init__(self, source_dir, build_dir, options):
         self.source_dir = source_dir
@@ -318,8 +319,10 @@ class Environment:
         if build_dir is not None:
             self.scratch_dir = os.path.join(build_dir, Environment.private_dir)
             self.log_dir = os.path.join(build_dir, Environment.log_dir)
+            self.info_dir = os.path.join(build_dir, Environment.info_dir)
             os.makedirs(self.scratch_dir, exist_ok=True)
             os.makedirs(self.log_dir, exist_ok=True)
+            os.makedirs(self.info_dir, exist_ok=True)
             try:
                 self.coredata = coredata.load(self.get_build_dir())
                 self.first_invocation = False
@@ -333,8 +336,16 @@ class Environment:
                     mlog.log('Reason:', mlog.red(str(e)))
                     coredata.read_cmd_line_file(self.build_dir, options)
                     self.create_new_coredata(options)
-                else:
-                    raise e
+                except MesonException as e:
+                    # If we stored previous command line options, we can recover from
+                    # a broken/outdated coredata.
+                    if os.path.isfile(coredata.get_cmd_line_file(self.build_dir)):
+                        mlog.warning('Regenerating configuration from scratch.')
+                        mlog.log('Reason:', mlog.red(str(e)))
+                        coredata.read_cmd_line_file(self.build_dir, options)
+                        self.create_new_coredata(options)
+                    else:
+                        raise e
         else:
             # Just create a fresh coredata in this case
             self.create_new_coredata(options)

mesonbuild/mconf.py

@@ -163,7 +163,7 @@ def run(options):
             c.print_conf()
         if save:
             c.save()
-            mintro.update_build_options(c.coredata, builddir)
+            mintro.update_build_options(c.coredata, c.build.environment.info_dir)
     except ConfException as e:
         print('Meson configurator encountered an error:')
         raise e

mesonbuild/mintro.py

@@ -276,7 +276,7 @@ def list_target_files(target_name, targets):
 
     return ('target_files', result)
 
-def list_buildoptions(coredata):
+def list_buildoptions(coredata: cdata.CoreData):
     optlist = []
 
     dir_option_names = ['bindir',
@@ -346,7 +346,7 @@ def list_buildsystem_files(builddata: build.Build):
     filelist = find_buildsystem_files_list(src_dir)
     return ('buildsystem_files', filelist)
 
-def list_deps(coredata):
+def list_deps(coredata: cdata.CoreData):
     result = []
     for d in coredata.deps.values():
         if d.found():
@@ -451,10 +451,10 @@ def list_projinfo_from_source(sourcedir):
 
 def run(options):
     datadir = 'meson-private'
-    introfile = INTROSPECTION_OUTPUT_FILE
+    infodir = 'meson-info'
     if options.builddir is not None:
         datadir = os.path.join(options.builddir, datadir)
-        introfile = os.path.join(options.builddir, introfile)
+        infodir = os.path.join(options.builddir, infodir)
     if options.builddir.endswith('/meson.build') or options.builddir.endswith('\\meson.build') or options.builddir == 'meson.build':
         sourcedir = '.' if options.builddir == 'meson.build' else options.builddir[:-11]
         if options.projectinfo:
@@ -463,15 +463,11 @@ def run(options):
         if options.buildoptions:
             list_buildoptions_from_source(sourcedir, options.backend)
             return 0
-    if not os.path.isdir(datadir) or not os.path.isfile(introfile):
+    if not os.path.isdir(datadir) or not os.path.isdir(infodir):
         print('Current directory is not a build dir. Please specify it or '
               'change the working directory to it.')
         return 1
 
-    rawdata = {}
-    with open(introfile, 'r') as fp:
-        rawdata = json.load(fp)
-
     results = []
     toextract = []
 
@@ -490,14 +486,20 @@ def run(options):
     if options.all or options.list_targets:
         toextract += ['targets']
     if options.target_files is not None:
-        results += [list_target_files(options.target_files, rawdata['targets'])]
+        targets_file = os.path.join(infodir, 'intro-targets.json')
+        with open(targets_file, 'r') as fp:
+            targets = json.load(fp)
+        results += [list_target_files(options.target_files, targets)]
     if options.all or options.tests:
         toextract += ['tests']
 
     for i in toextract:
-        if i not in rawdata:
-            raise RuntimeError('Key "{}" missing in introspection file. Please report this a bug.'.format(i))
-        results += [(i, rawdata[i])]
+        curr = os.path.join(infodir, 'intro-{}.json'.format(i))
+        if not os.path.isfile(curr):
+            print('Introspection file {} does not exist.'.format(curr))
+            return 1
+        with open(curr, 'r') as fp:
+            results += [(i, json.load(fp))]
 
     indent = options.indent if options.indent > 0 else None
 
@@ -514,6 +516,12 @@ def run(options):
         print(json.dumps(out, indent=indent))
     return 0
 
+def write_intro_info(intro_info, info_dir):
+    for i in intro_info:
+        out_file = os.path.join(info_dir, 'intro-{}.json'.format(i[0]))
+        with open(out_file, 'w') as fp:
+            json.dump(i[1], fp)
+
 def generate_introspection_file(builddata: build.Build, backend: backends.Backend):
     coredata = builddata.environment.get_coredata()
     benchmarkdata = backend.create_test_serialisation(builddata.get_benchmarks())
@@ -531,29 +539,11 @@ def generate_introspection_file(builddata: build.Build, backend: backends.Backen
         list_tests(testdata)
     ]
 
-    outdict = {}
-    for i in intro_info:
-        outdict[i[0]] = i[1]
-
-    outfile = os.path.join(builddata.environment.get_build_dir(), INTROSPECTION_OUTPUT_FILE)
-    outfile = os.path.abspath(outfile)
-
-    with open(outfile, 'w') as fp:
-        json.dump(outdict, fp)
-
-def update_build_options(coredata, builddir):
-    outfile = os.path.join(builddir, INTROSPECTION_OUTPUT_FILE)
-    outfile = os.path.abspath(outfile)
-
-    with open(outfile, 'r') as fp:
-        outdict = json.load(fp)
+    write_intro_info(intro_info, builddata.environment.info_dir)
 
+def update_build_options(coredata: cdata.CoreData, info_dir):
     intro_info = [
         list_buildoptions(coredata)
     ]
 
-    for i in intro_info:
-        outdict[i[0]] = i[1]
-
-    with open(outfile, 'w') as fp:
-        json.dump(outdict, fp)
+    write_intro_info(intro_info, info_dir)

run_unittests.py

@@ -3116,10 +3116,8 @@ recommended as it is not supported on some platforms''')
     def test_introspect_json_dump(self):
         testdir = os.path.join(self.unit_test_dir, '49 introspection')
         self.init(testdir)
-        introfile = os.path.join(self.builddir, 'meson-introspection.json')
-        self.assertPathExists(introfile)
-        with open(introfile, 'r') as fp:
-            res = json.load(fp)
+        infodir = os.path.join(self.builddir, 'meson-info')
+        self.assertPathExists(infodir)
 
         def assertKeyTypes(key_type_list, obj):
             for i in key_type_list:
@@ -3184,6 +3182,14 @@ recommended as it is not supported on some platforms''')
             ('source_files', list),
         ]
 
+        # First load all files
+        res = {}
+        for i in root_keylist:
+            curr = os.path.join(infodir, 'intro-{}.json'.format(i[0]))
+            self.assertPathExists(curr)
+            with open(curr, 'r') as fp:
+                res[i[0]] = json.load(fp)
+
         assertKeyTypes(root_keylist, res)
 
         # Check Tests and benchmarks
@@ -3252,16 +3258,30 @@ recommended as it is not supported on some platforms''')
         res_all = self.introspect('--all')
         res_file = {}
 
-        introfile = os.path.join(self.builddir, 'meson-introspection.json')
-        self.assertPathExists(introfile)
-        with open(introfile, 'r') as fp:
-            res_file = json.load(fp)
+        root_keylist = [
+            'benchmarks',
+            'buildoptions',
+            'buildsystem_files',
+            'dependencies',
+            'installed',
+            'projectinfo',
+            'targets',
+            'tests',
+        ]
+
+        infodir = os.path.join(self.builddir, 'meson-info')
+        self.assertPathExists(infodir)
+        for i in root_keylist:
+            curr = os.path.join(infodir, 'intro-{}.json'.format(i))
+            self.assertPathExists(curr)
+            with open(curr, 'r') as fp:
+                res_file[i] = json.load(fp)
 
         self.assertEqual(res_all, res_file)
 
     def test_introspect_config_update(self):
         testdir = os.path.join(self.unit_test_dir, '49 introspection')
-        introfile = os.path.join(self.builddir, 'meson-introspection.json')
+        introfile = os.path.join(self.builddir, 'meson-info', 'intro-buildoptions.json')
         self.init(testdir)
         self.assertPathExists(introfile)
         with open(introfile, 'r') as fp:
@@ -3270,20 +3290,20 @@ recommended as it is not supported on some platforms''')
         self.setconf('-Dcpp_std=c++14')
         self.setconf('-Dbuildtype=release')
 
-        for idx, i in enumerate(res1['buildoptions']):
+        for idx, i in enumerate(res1):
             if i['name'] == 'cpp_std':
-                res1['buildoptions'][idx]['value'] = 'c++14'
+                res1[idx]['value'] = 'c++14'
             if i['name'] == 'buildtype':
-                res1['buildoptions'][idx]['value'] = 'release'
+                res1[idx]['value'] = 'release'
             if i['name'] == 'optimization':
-                res1['buildoptions'][idx]['value'] = '3'
+                res1[idx]['value'] = '3'
             if i['name'] == 'debug':
-                res1['buildoptions'][idx]['value'] = False
+                res1[idx]['value'] = False
 
         with open(introfile, 'r') as fp:
             res2 = json.load(fp)
 
-        self.assertDictEqual(res1, res2)
+        self.assertListEqual(res1, res2)
 
 class FailureTests(BasePlatformTests):
     '''