Merge pull request #7461 from mensinda/noMoreSetuptools

Remove the setuptools dependency with mesondata.py

mesonbuild/cmake/interpreter.py

@@ -15,8 +15,6 @@
 # This class contains the basic functionality needed to run any interpreter
 # or an interpreter-based tool.
 
-import pkg_resources
-
 from .common import CMakeException, CMakeTarget, TargetOptions
 from .client import CMakeClient, RequestCMakeInputs, RequestConfigure, RequestCompute, RequestCodeModel
 from .fileapi import CMakeFileAPI
@@ -25,6 +23,7 @@ from .traceparser import CMakeTraceParser, CMakeGeneratorTarget
 from .. import mlog, mesonlib
 from ..environment import Environment
 from ..mesonlib import MachineChoice, OrderedSet, version_compare
+from ..mesondata import mesondata
 from ..compilers.compilers import lang_suffixes, header_suffixes, obj_suffixes, lib_suffixes, is_header
 from enum import Enum
 from functools import lru_cache
@@ -814,7 +813,7 @@ class CMakeInterpreter:
             raise CMakeException('Unable to find CMake')
         self.trace = CMakeTraceParser(cmake_exe.version(), self.build_dir, permissive=True)
 
-        preload_file = pkg_resources.resource_filename('mesonbuild', 'cmake/data/preload.cmake')
+        preload_file = mesondata['cmake/data/preload.cmake'].write_to_private(self.env)
 
         # Prefere CMAKE_PROJECT_INCLUDE over CMAKE_TOOLCHAIN_FILE if possible,
         # since CMAKE_PROJECT_INCLUDE was actually designed for code injection.

mesonbuild/dependencies/base.py

@@ -29,8 +29,6 @@ import typing as T
 from enum import Enum
 from pathlib import Path, PurePath
 
-import pkg_resources
-
 from .. import mlog
 from .. import mesonlib
 from ..compilers import clib_langs
@@ -40,6 +38,7 @@ from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException
 from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine
 from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list, split_args
 from ..mesonlib import Version, LibType
+from ..mesondata import mesondata
 
 if T.TYPE_CHECKING:
     from ..compilers.compilers import CompilerType  # noqa: F401
@@ -1522,8 +1521,7 @@ class CMakeDependency(ExternalDependency):
         build_dir = self._get_build_dir()
 
         # Insert language parameters into the CMakeLists.txt and write new CMakeLists.txt
-        # Per the warning in pkg_resources, this is *not* a path and os.path and Pathlib are *not* safe to use here.
+        cmake_txt = mesondata['dependencies/data/' + cmake_file].data
-        cmake_txt = pkg_resources.resource_string('mesonbuild', 'dependencies/data/' + cmake_file).decode()
 
         # In general, some Fortran CMake find_package() also require C language enabled,
         # even if nothing from C is directly used. An easy Fortran example that fails

mesonbuild/mesondata.py

@@ -0,0 +1,374 @@
+# Copyright 2020 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+####
+####  WARNING: This is an automatically generated file! Do not edit!
+####           Generated by tools/gen_data.py
+####
+
+
+from pathlib import Path
+import typing as T
+
+if T.TYPE_CHECKING:
+    from .environment import Environment
+
+######################
+# BEGIN Data section #
+######################
+
+file_0_data_CMakeListsLLVM_txt = '''\
+cmake_minimum_required(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION} )
+
+set(PACKAGE_FOUND FALSE)
+
+while(TRUE)
+  find_package(LLVM REQUIRED CONFIG QUIET)
+
+  # ARCHS has to be set via the CMD interface
+  if(LLVM_FOUND OR "${ARCHS}" STREQUAL "")
+    break()
+  endif()
+
+  list(GET       ARCHS 0 CMAKE_LIBRARY_ARCHITECTURE)
+  list(REMOVE_AT ARCHS 0)
+endwhile()
+
+if(LLVM_FOUND)
+  set(PACKAGE_FOUND TRUE)
+
+  foreach(mod IN LISTS LLVM_MESON_MODULES)
+    # Reset variables
+    set(out_mods)
+    set(real_mods)
+
+    # Generate a lower and upper case version
+    string(TOLOWER "${mod}" mod_L)
+    string(TOUPPER "${mod}" mod_U)
+
+    # Get the mapped components
+    llvm_map_components_to_libnames(out_mods ${mod} ${mod_L} ${mod_U})
+    list(SORT              out_mods)
+    list(REMOVE_DUPLICATES out_mods)
+
+    # Make sure that the modules exist
+    foreach(i IN LISTS out_mods)
+      if(TARGET ${i})
+        list(APPEND real_mods ${i})
+      endif()
+    endforeach()
+
+    # Set the output variables
+    set(MESON_LLVM_TARGETS_${mod} ${real_mods})
+    foreach(i IN LISTS real_mods)
+      set(MESON_TARGET_TO_LLVM_${i} ${mod})
+    endforeach()
+  endforeach()
+
+  # Check the following variables:
+  # LLVM_PACKAGE_VERSION
+  # LLVM_VERSION
+  # LLVM_VERSION_STRING
+  if(NOT DEFINED PACKAGE_VERSION)
+    if(DEFINED LLVM_PACKAGE_VERSION)
+      set(PACKAGE_VERSION "${LLVM_PACKAGE_VERSION}")
+    elseif(DEFINED LLVM_VERSION)
+      set(PACKAGE_VERSION "${LLVM_VERSION}")
+    elseif(DEFINED LLVM_VERSION_STRING)
+      set(PACKAGE_VERSION "${LLVM_VERSION_STRING}")
+    endif()
+  endif()
+
+  # Check the following variables:
+  # LLVM_LIBRARIES
+  # LLVM_LIBS
+  set(libs)
+  if(DEFINED LLVM_LIBRARIES)
+    set(libs LLVM_LIBRARIES)
+  elseif(DEFINED LLVM_LIBS)
+    set(libs LLVM_LIBS)
+  endif()
+
+  # Check the following variables:
+  # LLVM_INCLUDE_DIRS
+  # LLVM_INCLUDES
+  # LLVM_INCLUDE_DIR
+  set(includes)
+  if(DEFINED LLVM_INCLUDE_DIRS)
+    set(includes LLVM_INCLUDE_DIRS)
+  elseif(DEFINED LLVM_INCLUDES)
+    set(includes LLVM_INCLUDES)
+  elseif(DEFINED LLVM_INCLUDE_DIR)
+    set(includes LLVM_INCLUDE_DIR)
+  endif()
+
+  # Check the following variables:
+  # LLVM_DEFINITIONS
+  set(definitions)
+  if(DEFINED LLVM_DEFINITIONS)
+    set(definitions LLVM_DEFINITIONS)
+  endif()
+
+  set(PACKAGE_INCLUDE_DIRS "${${includes}}")
+  set(PACKAGE_DEFINITIONS  "${${definitions}}")
+  set(PACKAGE_LIBRARIES    "${${libs}}")
+endif()
+'''
+
+file_1_data_CMakePathInfo_txt = '''\
+cmake_minimum_required(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION})
+
+set(TMP_PATHS_LIST)
+list(APPEND TMP_PATHS_LIST ${CMAKE_PREFIX_PATH})
+list(APPEND TMP_PATHS_LIST ${CMAKE_FRAMEWORK_PATH})
+list(APPEND TMP_PATHS_LIST ${CMAKE_APPBUNDLE_PATH})
+list(APPEND TMP_PATHS_LIST $ENV{CMAKE_PREFIX_PATH})
+list(APPEND TMP_PATHS_LIST $ENV{CMAKE_FRAMEWORK_PATH})
+list(APPEND TMP_PATHS_LIST $ENV{CMAKE_APPBUNDLE_PATH})
+list(APPEND TMP_PATHS_LIST ${CMAKE_SYSTEM_PREFIX_PATH})
+list(APPEND TMP_PATHS_LIST ${CMAKE_SYSTEM_FRAMEWORK_PATH})
+list(APPEND TMP_PATHS_LIST ${CMAKE_SYSTEM_APPBUNDLE_PATH})
+
+set(LIB_ARCH_LIST)
+if(CMAKE_LIBRARY_ARCHITECTURE_REGEX)
+  file(GLOB implicit_dirs RELATIVE /lib /lib/*-linux-gnu* )
+  foreach(dir ${implicit_dirs})
+    if("${dir}" MATCHES "${CMAKE_LIBRARY_ARCHITECTURE_REGEX}")
+      list(APPEND LIB_ARCH_LIST "${dir}")
+    endif()
+  endforeach()
+endif()
+
+# "Export" these variables:
+set(MESON_ARCH_LIST ${LIB_ARCH_LIST})
+set(MESON_PATHS_LIST ${TMP_PATHS_LIST})
+set(MESON_CMAKE_ROOT ${CMAKE_ROOT})
+set(MESON_CMAKE_SYSROOT ${CMAKE_SYSROOT})
+set(MESON_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH})
+
+message(STATUS ${TMP_PATHS_LIST})
+'''
+
+file_2_data_CMakeLists_txt = '''\
+# fail noisily if attempt to use this file without setting:
+# cmake_minimum_required(VERSION ${CMAKE_VERSION})
+# project(... LANGUAGES ...)
+
+cmake_policy(SET CMP0000 NEW)
+
+set(PACKAGE_FOUND FALSE)
+set(_packageName "${NAME}")
+string(TOUPPER "${_packageName}" PACKAGE_NAME)
+
+while(TRUE)
+  find_package("${NAME}" QUIET COMPONENTS ${COMPS})
+
+  # ARCHS has to be set via the CMD interface
+  if(${_packageName}_FOUND OR ${PACKAGE_NAME}_FOUND OR "${ARCHS}" STREQUAL "")
+    break()
+  endif()
+
+  list(GET       ARCHS 0 CMAKE_LIBRARY_ARCHITECTURE)
+  list(REMOVE_AT ARCHS 0)
+endwhile()
+
+if(${_packageName}_FOUND  OR  ${PACKAGE_NAME}_FOUND)
+  set(PACKAGE_FOUND TRUE)
+
+  # Check the following variables:
+  # FOO_VERSION
+  # Foo_VERSION
+  # FOO_VERSION_STRING
+  # Foo_VERSION_STRING
+  if(NOT DEFINED PACKAGE_VERSION)
+    if(DEFINED ${_packageName}_VERSION)
+      set(PACKAGE_VERSION "${${_packageName}_VERSION}")
+    elseif(DEFINED ${PACKAGE_NAME}_VERSION)
+      set(PACKAGE_VERSION "${${PACKAGE_NAME}_VERSION}")
+    elseif(DEFINED ${_packageName}_VERSION_STRING)
+      set(PACKAGE_VERSION "${${_packageName}_VERSION_STRING}")
+    elseif(DEFINED ${PACKAGE_NAME}_VERSION_STRING)
+      set(PACKAGE_VERSION "${${PACKAGE_NAME}_VERSION_STRING}")
+    endif()
+  endif()
+
+  # Check the following variables:
+  # FOO_LIBRARIES
+  # Foo_LIBRARIES
+  # FOO_LIBS
+  # Foo_LIBS
+  set(libs)
+  if(DEFINED ${_packageName}_LIBRARIES)
+    set(libs ${_packageName}_LIBRARIES)
+  elseif(DEFINED ${PACKAGE_NAME}_LIBRARIES)
+    set(libs ${PACKAGE_NAME}_LIBRARIES)
+  elseif(DEFINED ${_packageName}_LIBS)
+    set(libs ${_packageName}_LIBS)
+  elseif(DEFINED ${PACKAGE_NAME}_LIBS)
+    set(libs ${PACKAGE_NAME}_LIBS)
+  endif()
+
+  # Check the following variables:
+  # FOO_INCLUDE_DIRS
+  # Foo_INCLUDE_DIRS
+  # FOO_INCLUDES
+  # Foo_INCLUDES
+  # FOO_INCLUDE_DIR
+  # Foo_INCLUDE_DIR
+  set(includes)
+  if(DEFINED ${_packageName}_INCLUDE_DIRS)
+    set(includes ${_packageName}_INCLUDE_DIRS)
+  elseif(DEFINED ${PACKAGE_NAME}_INCLUDE_DIRS)
+    set(includes ${PACKAGE_NAME}_INCLUDE_DIRS)
+  elseif(DEFINED ${_packageName}_INCLUDES)
+    set(includes ${_packageName}_INCLUDES)
+  elseif(DEFINED ${PACKAGE_NAME}_INCLUDES)
+    set(includes ${PACKAGE_NAME}_INCLUDES)
+  elseif(DEFINED ${_packageName}_INCLUDE_DIR)
+    set(includes ${_packageName}_INCLUDE_DIR)
+  elseif(DEFINED ${PACKAGE_NAME}_INCLUDE_DIR)
+    set(includes ${PACKAGE_NAME}_INCLUDE_DIR)
+  endif()
+
+  # Check the following variables:
+  # FOO_DEFINITIONS
+  # Foo_DEFINITIONS
+  set(definitions)
+  if(DEFINED ${_packageName}_DEFINITIONS)
+    set(definitions ${_packageName}_DEFINITIONS)
+  elseif(DEFINED ${PACKAGE_NAME}_DEFINITIONS)
+    set(definitions ${PACKAGE_NAME}_DEFINITIONS)
+  endif()
+
+  set(PACKAGE_INCLUDE_DIRS "${${includes}}")
+  set(PACKAGE_DEFINITIONS  "${${definitions}}")
+  set(PACKAGE_LIBRARIES    "${${libs}}")
+endif()
+'''
+
+file_3_data_preload_cmake = '''\
+if(MESON_PS_LOADED)
+  return()
+endif()
+
+set(MESON_PS_LOADED ON)
+
+# Dummy macros that have a special meaning in the meson code
+macro(meson_ps_execute_delayed_calls)
+endmacro()
+
+macro(meson_ps_reload_vars)
+endmacro()
+
+# Helper macro to inspect the current CMake state
+macro(meson_ps_inspect_vars)
+  set(MESON_PS_CMAKE_CURRENT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+  set(MESON_PS_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+  meson_ps_execute_delayed_calls()
+endmacro()
+
+
+# Override some system functions with custom code and forward the args
+# to the original function
+macro(add_custom_command)
+  meson_ps_inspect_vars()
+  _add_custom_command(${ARGV})
+endmacro()
+
+macro(add_custom_target)
+  meson_ps_inspect_vars()
+  _add_custom_target(${ARGV})
+endmacro()
+
+macro(set_property)
+  meson_ps_inspect_vars()
+  _set_property(${ARGV})
+endmacro()
+
+function(set_source_files_properties)
+  set(FILES)
+  set(I 0)
+  set(PROPERTIES OFF)
+
+  while(I LESS ARGC)
+    if(NOT PROPERTIES)
+      if("${ARGV${I}}" STREQUAL "PROPERTIES")
+        set(PROPERTIES ON)
+      else()
+        list(APPEND FILES "${ARGV${I}}")
+      endif()
+
+      math(EXPR I "${I} + 1")
+    else()
+      set(ID_IDX ${I})
+      math(EXPR PROP_IDX "${ID_IDX} + 1")
+
+      set(ID   "${ARGV${ID_IDX}}")
+      set(PROP "${ARGV${PROP_IDX}}")
+
+      set_property(SOURCE ${FILES} PROPERTY "${ID}" "${PROP}")
+      math(EXPR I "${I} + 2")
+    endif()
+  endwhile()
+endfunction()
+
+set(MESON_PS_DELAYED_CALLS add_custom_command;add_custom_target;set_property)
+meson_ps_reload_vars()
+'''
+
+
+####################
+# END Data section #
+####################
+
+class DataFile:
+    def __init__(self, path: Path, sha256sum: str, data: str) -> None:
+        self.path = path
+        self.sha256sum = sha256sum
+        self.data = data
+
+    def write_once(self, path: Path) -> None:
+        if not path.exists():
+            path.write_text(self.data)
+
+    def write_to_private(self, env: 'Environment') -> Path:
+        out_file = Path(env.scratch_dir) / 'data' / self.path.name
+        out_file.parent.mkdir(exist_ok=True)
+        self.write_once(out_file)
+        return out_file
+
+
+mesondata = {
+    'dependencies/data/CMakeListsLLVM.txt': DataFile(
+        Path('dependencies/data/CMakeListsLLVM.txt'),
+        '412cec3315597041a978d018cdaca282dcd47693793540da88ae2f80d0cbd7cd',
+        file_0_data_CMakeListsLLVM_txt,
+    ),
+    'dependencies/data/CMakePathInfo.txt': DataFile(
+        Path('dependencies/data/CMakePathInfo.txt'),
+        '90da8b443982d9c87139b7dc84228eb58cab4315764949637208f25e2bda7db2',
+        file_1_data_CMakePathInfo_txt,
+    ),
+    'dependencies/data/CMakeLists.txt': DataFile(
+        Path('dependencies/data/CMakeLists.txt'),
+        '71a2d58381f912bbfb1c8709884d34d721f682edf2fca001e1f582f0bffd0da7',
+        file_2_data_CMakeLists_txt,
+    ),
+    'cmake/data/preload.cmake': DataFile(
+        Path('cmake/data/preload.cmake'),
+        '064d047b18a5c919ad016b838bed50c5d40aebe9e53da0e70eff9d52a2c1ca1f',
+        file_3_data_preload_cmake,
+    ),
+}

msi/createmsi.py

@@ -151,8 +151,6 @@ class PackageGenerator:
             if os.path.exists(sdir):
                 shutil.rmtree(sdir)
         main_stage, ninja_stage = self.staging_dirs
-        dep_data_dir = 'mesonbuild/dependencies/data'
-        cmake_data_dir = 'mesonbuild/cmake/data'
         modules = self.get_all_modules_from_dir('mesonbuild/modules')
         modules += self.get_all_modules_from_dir('mesonbuild/scripts')
         modules += self.get_more_modules()
@@ -174,8 +172,6 @@ class PackageGenerator:
         pyinst_cmd += ['meson.py']
         subprocess.check_call(pyinst_cmd)
         shutil.move(pyinstaller_tmpdir + '/meson', main_stage)
-        shutil.copytree(dep_data_dir, main_stage + '/mesonbuild/dependencies/data')
-        shutil.copytree(cmake_data_dir, main_stage + '/mesonbuild/cmake/data')
         if not os.path.exists(os.path.join(main_stage, 'meson.exe')):
             sys.exit('Meson exe missing from staging dir.')
         os.mkdir(ninja_stage)

run_unittests.py

@@ -1485,6 +1485,38 @@ class DataTests(unittest.TestCase):
         astint = AstInterpreter('.', '', '')
         self.assertEqual(set(interp.funcs.keys()), set(astint.funcs.keys()))
 
+    def test_mesondata_is_up_to_date(self):
+        from mesonbuild.mesondata import mesondata
+        err_msg = textwrap.dedent('''
+
+            ###########################################################
+            ###        mesonbuild.mesondata is not up-to-date       ###
+            ###  Please regenerate it by running tools/gen_data.py  ###
+            ###########################################################
+
+        ''')
+
+        root_dir = Path(__file__).resolve().parent
+        mesonbuild_dir = root_dir / 'mesonbuild'
+
+        data_dirs = mesonbuild_dir.glob('**/data')
+        data_files = []  # type: T.List[T.Tuple(str, str)]
+
+        for i in data_dirs:
+            for p in i.iterdir():
+                data_files += [(p.relative_to(mesonbuild_dir).as_posix(), hashlib.sha256(p.read_bytes()).hexdigest())]
+
+        from pprint import pprint
+        current_files = set(mesondata.keys())
+        scanned_files = set([x[0] for x in data_files])
+
+        self.assertSetEqual(current_files, scanned_files, err_msg + 'Data files were added or removed\n')
+        errors = []
+        for i in data_files:
+            if mesondata[i[0]].sha256sum != i[1]:
+                errors += [i[0]]
+
+        self.assertListEqual(errors, [], err_msg + 'Files were changed')
 
 class BasePlatformTests(unittest.TestCase):
     prefix = '/usr'

setup.py

@@ -37,10 +37,6 @@ packages = ['mesonbuild',
             'mesonbuild.scripts',
             'mesonbuild.templates',
             'mesonbuild.wrap']
-package_data = {
-    'mesonbuild.dependencies': ['data/CMakeLists.txt', 'data/CMakeListsLLVM.txt', 'data/CMakePathInfo.txt'],
-    'mesonbuild.cmake': ['data/run_ctgt.py', 'data/preload.cmake'],
-}
 data_files = []
 if sys.platform != 'win32':
     # Only useful on UNIX-like systems
@@ -51,6 +47,5 @@ if __name__ == '__main__':
     setup(name='meson',
           version=version,
           packages=packages,
-          package_data=package_data,
           entry_points=entries,
           data_files=data_files,)

tools/gen_data.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 Daniel Mensinger
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import hashlib
+import textwrap
+import re
+from pathlib import Path
+from datetime import datetime
+import typing as T
+
+class DataFile:
+    file_counter = 0
+
+    def __init__(self, path: Path, root: Path):
+        self.path = path
+        self.id = self.path.relative_to(root)
+        self.data_str = f'file_{DataFile.file_counter}_data_' + re.sub('[^a-zA-Z0-9]', '_', self.path.name)
+        DataFile.file_counter += 1
+
+        b = self.path.read_bytes()
+        self.data = b.decode()
+        self.sha256sum = hashlib.sha256(b).hexdigest()
+
+    def __repr__(self) -> str:
+        return f'<{type(self).__name__}: [{self.sha256sum}] {self.id}>'
+
+def main() -> int:
+    root_dir = Path(__file__).resolve().parents[1]
+    mesonbuild_dir = root_dir / 'mesonbuild'
+    out_file = mesonbuild_dir / 'mesondata.py'
+
+    data_dirs = mesonbuild_dir.glob('**/data')
+
+    data_files: T.List[DataFile] = []
+
+    for d in data_dirs:
+        for p in d.iterdir():
+            data_files += [DataFile(p, mesonbuild_dir)]
+
+    print(f'Found {len(data_files)} data files')
+
+    # Generate the data script
+    data = ''
+
+    data += textwrap.dedent(f'''\
+        # Copyright {datetime.today().year} The Meson development team
+
+        # Licensed under the Apache License, Version 2.0 (the "License");
+        # you may not use this file except in compliance with the License.
+        # You may obtain a copy of the License at
+
+        #     http://www.apache.org/licenses/LICENSE-2.0
+
+        # Unless required by applicable law or agreed to in writing, software
+        # distributed under the License is distributed on an "AS IS" BASIS,
+        # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        # See the License for the specific language governing permissions and
+        # limitations under the License.
+
+
+        ####
+        ####  WARNING: This is an automatically generated file! Do not edit!
+        ####           Generated by {Path(__file__).resolve().relative_to(root_dir)}
+        ####
+
+
+        from pathlib import Path
+        import typing as T
+
+        if T.TYPE_CHECKING:
+            from .environment import Environment
+
+        ######################
+        # BEGIN Data section #
+        ######################
+
+    ''')
+
+    for i in data_files:
+        data += f"{i.data_str} = '''\\\n{i.data}'''\n\n"
+
+    data += textwrap.dedent(f'''
+        ####################
+        # END Data section #
+        ####################
+
+        class DataFile:
+            def __init__(self, path: Path, sha256sum: str, data: str) -> None:
+                self.path = path
+                self.sha256sum = sha256sum
+                self.data = data
+
+            def write_once(self, path: Path) -> None:
+                if not path.exists():
+                    path.write_text(self.data)
+
+            def write_to_private(self, env: 'Environment') -> Path:
+                out_file = Path(env.scratch_dir) / 'data' / self.path.name
+                out_file.parent.mkdir(exist_ok=True)
+                self.write_once(out_file)
+                return out_file
+
+
+        mesondata = {{
+    ''')
+
+    for i in data_files:
+        data += textwrap.indent(textwrap.dedent(f"""\
+            '{i.id}': DataFile(
+                Path('{i.id}'),
+                '{i.sha256sum}',
+                {i.data_str},
+            ),
+        """), '    ')
+
+    data += textwrap.dedent('''\
+        }
+    ''')
+
+    print(f'Updating {out_file}')
+    out_file.write_text(data)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())