From 035df5369e34fdb7bda954601800cd00b8d8ef8e Mon Sep 17 00:00:00 2001
From: Dylan Baker <dylan@pnwbakers.com>
Date: Tue, 17 Aug 2021 12:37:21 -0700
Subject: [PATCH] backends/ninja: write depscan input files to json

Currently, we write each file to the command line, but this can result in
situations where the number of files passed exceeds OS imposed command
line limits. For compilers, we solve this with response files. For
depscan I've chosen to use a JSON list instead. JSON has several
advantages in that it's standardized, there's a built-in python module
for it, and it's familiar. I've also chosen to always use the JSON file
instead of having a heuristic to decide between JSON and not JSON,
while there may be a small performance trade off here, keeping the
implementation simple with only one path is wort it.

Fixes #9129
---
 mesonbuild/backend/ninjabackend.py | 27 ++++++++++++++++++---------
 mesonbuild/scripts/depscan.py      | 10 ++++++----
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py
index 9b249c3ba..fb0e77b51 100644
--- a/mesonbuild/backend/ninjabackend.py
+++ b/mesonbuild/backend/ninjabackend.py
@@ -11,18 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import typing as T
+
+from collections import OrderedDict
+from enum import Enum, unique
+from functools import lru_cache
+from pathlib import PurePath, Path
+from textwrap import dedent
+import itertools
+import json
 import os
-import re
 import pickle
+import re
 import shlex
 import subprocess
-from collections import OrderedDict
-from enum import Enum, unique
-import itertools
-from textwrap import dedent
-from pathlib import PurePath, Path
-from functools import lru_cache
+import typing as T
 
 from . import backends
 from .. import modules
@@ -915,9 +917,16 @@ class NinjaBackend(backends.Backend):
         pickle_base = target.name + '.dat'
         pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/')
         pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/')
+        json_abs = os.path.join(self.get_target_private_dir_abs(target), f'{target.name}-deps.json').replace('\\', '/')
         rule_name = 'depscan'
         scan_sources = self.select_sources_to_scan(compiled_sources)
-        elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources)
+
+        # Dump the sources as a json list. This avoids potential probllems where
+        # the number of sources passed to depscan exceedes the limit imposed by
+        # the OS.
+        with open(json_abs, 'w', encoding='utf-8') as f:
+            json.dump(scan_sources, f)
+        elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, json_abs)
         elem.add_item('picklefile', pickle_file)
         scaninfo = TargetDependencyScannerInfo(self.get_target_private_dir(target), source2object)
         with open(pickle_abs, 'wb') as p:
diff --git a/mesonbuild/scripts/depscan.py b/mesonbuild/scripts/depscan.py
index 9fc435b5d..68e7dc49d 100644
--- a/mesonbuild/scripts/depscan.py
+++ b/mesonbuild/scripts/depscan.py
@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
 import pathlib
 import pickle
 import re
-import os
 import sys
 import typing as T
 
@@ -194,8 +195,9 @@ class DependencyScanner:
         return 0
 
 def run(args: T.List[str]) -> int:
-    pickle_file = args[0]
-    outfile = args[1]
-    sources = args[2:]
+    assert len(args) == 3, 'got wrong number of arguments!'
+    pickle_file, outfile, jsonfile = args
+    with open(jsonfile, 'r', encoding='utf-8') as f:
+        sources = json.load(f)
     scanner = DependencyScanner(pickle_file, outfile, sources)
     return scanner.scan()