From 583b9a689ab2083839975b0b3cfa0790d978acd9 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 1 Jun 2015 12:33:12 -0700 Subject: [PATCH] Addressed comments: fixed memory leak introduced by changes. --- tools/run_tests/jobset.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 61a95b0d555..51d61db7f6c 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -66,6 +66,7 @@ def shuffle_iteratable(it): # p as we take elements - this gives us a somewhat random set of values before # we've seen all the values, but starts producing values without having to # compute ALL of them at once, allowing tests to start a little earlier + LARGE_THRESHOLD = 1000 nextit = [] p = 1 for val in it: @@ -74,6 +75,17 @@ def shuffle_iteratable(it): yield val else: nextit.append(val) + # if the input iterates over a large number of values (potentially + # infinite, we'd be in the loop for a while (again, potentially forever). + # We need to reset "nextit" every so often to, in the case of an infinite + # iterator, avoid growing "nextit" without ever freeing it. + if len(nextit) > LARGE_THRESHOLD: + random.shuffle(nextit) + for val in nextit: + yield val + nextit = [] + p = 1 + # after taking a random sampling, we shuffle the rest of the elements and # yield them random.shuffle(nextit)