From 5c65432b72b4c6559d4e939c7d61ee3065f82e39 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Sat, 16 Jul 2022 23:21:42 +0800 Subject: [PATCH] Add tutorial tests --- tests/deploy/test_predictor.py | 43 ++++++++------------ tests/test_tutorials.py | 73 +++++++++++++++++++++++++++++++++- tests/testing_utils.py | 4 +- 3 files changed, 90 insertions(+), 30 deletions(-) diff --git a/tests/deploy/test_predictor.py b/tests/deploy/test_predictor.py index b1a41a6..b967a53 100644 --- a/tests/deploy/test_predictor.py +++ b/tests/deploy/test_predictor.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path as osp import tempfile import unittest.mock as mock @@ -28,14 +29,18 @@ class TestPredictor(CommonTest): @staticmethod def add_tests(cls): + """ + Automatically patch testing functions to cls. + """ + def _test_predictor(trainer_name): def _test_predictor_impl(self): trainer_class = getattr(self.MODULE, trainer_name) # Construct trainer with default parameters trainer = trainer_class() with tempfile.TemporaryDirectory() as td: - dynamic_model_dir = f"{td}/dynamic" - static_model_dir = f"{td}/static" + dynamic_model_dir = osp.join(td, "dynamic") + static_model_dir = osp.join(td, "static") # HACK: BaseModel.save_model() requires BaseModel().optimizer to be set optimizer = mock.Mock() optimizer.state_dict.return_value = {'foo': 'bar'} @@ -114,9 +119,9 @@ class TestCDPredictor(TestPredictor): out_single_file_list_t[0]) # Single input (ndarrays) - input_ = (cv2.imread(t1_path).astype('float32'), - cv2.imread(t2_path).astype('float32') - ) # Reuse the name `input_` + input_ = ( + cv2.imread(t1_path).astype('float32'), + cv2.imread(t2_path).astype('float32')) # Reuse the name `input_` out_single_array_p = predictor.predict(input_, transforms=transforms) self.check_dict_equal(out_single_array_p, out_single_file_p) out_single_array_t = trainer.predict(input_, transforms=transforms) @@ -164,7 +169,7 @@ class TestClasPredictor(TestPredictor): trainer.labels = labels predictor._model.labels = labels - # Single input (file paths) + # Single input (file path) input_ = single_input out_single_file_p = predictor.predict(input_, transforms=transforms) out_single_file_t = trainer.predict(input_, transforms=transforms) @@ -178,7 +183,7 @@ class TestClasPredictor(TestPredictor): self.check_dict_equal(out_single_file_list_p[0], out_single_file_list_t[0]) - # Single input (ndarrays) + # Single input (ndarray) input_ = cv2.imread(single_input).astype( 'float32') # Reuse the name `input_` out_single_array_p = predictor.predict(input_, transforms=transforms) @@ -227,7 +232,7 @@ class TestDetPredictor(TestPredictor): trainer.labels = labels predictor._model.labels = labels - # Single input (file paths) + # Single input (file path) input_ = single_input out_single_file_p = predictor.predict(input_, transforms=transforms) out_single_file_t = trainer.predict(input_, transforms=transforms) @@ -241,23 +246,7 @@ class TestDetPredictor(TestPredictor): self.check_dict_equal(out_single_file_list_p[0], out_single_file_list_t[0]) - # Single input (ndarrays) - input_ = cv2.imread(single_input).astype( - 'float32') # Reuse the name `input_` - out_single_array_p = predictor.predict(input_, transforms=transforms) - self.check_dict_equal(out_single_array_p, out_single_file_p) - out_single_array_t = trainer.predict(input_, transforms=transforms) - self.check_dict_equal(out_single_array_p, out_single_array_t) - out_single_array_list_p = predictor.predict( - [input_], transforms=transforms) - self.assertEqual(len(out_single_array_list_p), 1) - self.check_dict_equal(out_single_array_list_p[0], out_single_array_p) - out_single_array_list_t = trainer.predict( - [input_], transforms=transforms) - self.check_dict_equal(out_single_array_list_p[0], - out_single_array_list_t[0]) - - # Single input (ndarrays) + # Single input (ndarray) input_ = cv2.imread(single_input).astype( 'float32') # Reuse the name `input_` out_single_array_p = predictor.predict(input_, transforms=transforms) @@ -303,7 +292,7 @@ class TestSegPredictor(TestPredictor): num_inputs = 2 transforms = pdrs.transforms.Compose([pdrs.transforms.Normalize()]) - # Single input (file paths) + # Single input (file path) input_ = single_input out_single_file_p = predictor.predict(input_, transforms=transforms) out_single_file_t = trainer.predict(input_, transforms=transforms) @@ -317,7 +306,7 @@ class TestSegPredictor(TestPredictor): self.check_dict_equal(out_single_file_list_p[0], out_single_file_list_t[0]) - # Single input (ndarrays) + # Single input (ndarray) input_ = cv2.imread(single_input).astype( 'float32') # Reuse the name `input_` out_single_array_p = predictor.predict(input_, transforms=transforms) diff --git a/tests/test_tutorials.py b/tests/test_tutorials.py index eeae9aa..1c1543f 100644 --- a/tests/test_tutorials.py +++ b/tests/test_tutorials.py @@ -10,4 +10,75 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. + +import os.path as osp +import tempfile +import shutil +from glob import iglob + +from testing_utils import run_script, CpuCommonTest + + +class TestTutorial(CpuCommonTest): + SUBDIR = "./" + TIMEOUT = 300 + PATTERN = "*.py" + + @classmethod + def setUpClass(cls): + cls._td = tempfile.TemporaryDirectory(dir='./') + # Recursively copy the content of `cls.SUBDIR` to td. + # This is necessary for running scripts in td. + cls._TSUBDIR = osp.join(cls._td.name, osp.basename(cls.SUBDIR)) + shutil.copytree(cls.SUBDIR, cls._TSUBDIR) + return super().setUpClass() + + @classmethod + def tearDownClass(cls): + cls._td.cleanup() + + @staticmethod + def add_tests(cls): + """ + Automatically patch testing functions to cls. + """ + + def _test_tutorial(script_name): + def _test_tutorial_impl(self): + # Set working directory to `cls._TSUBDIR` such that the + # files generated by the script will be automatically cleaned. + run_script(f"python {script_name}", wd=cls._TSUBDIR) + + return _test_tutorial_impl + + for script_path in iglob(osp.join(cls.SUBDIR, cls.PATTERN)): + script_name = osp.basename(script_path) + if osp.normpath(osp.join(cls.SUBDIR, script_name)) != osp.normpath( + script_path): + raise ValueError( + f"{script_name} should be directly contained in {cls.SUBDIR}" + ) + setattr(cls, 'test_' + script_name, _test_tutorial(script_name)) + + return cls + + +@TestTutorial.add_tests +class TestCDTutorial(TestTutorial): + SUBDIR = "../tutorials/train/change_detection" + + +@TestTutorial.add_tests +class TestClasTutorial(TestTutorial): + SUBDIR = "../tutorials/train/classification" + + +@TestTutorial.add_tests +class TestDetTutorial(TestTutorial): + SUBDIR = "../tutorials/train/object_detection" + + +@TestTutorial.add_tests +class TestSegTutorial(TestTutorial): + SUBDIR = "../tutorials/train/semantic_segmentation" diff --git a/tests/testing_utils.py b/tests/testing_utils.py index a8b94b4..d8f8414 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -24,9 +24,9 @@ import paddle __all__ = ['CommonTest', 'CpuCommonTest', 'run_script'] -def run_script(cmd, silent=True, wd=None): +def run_script(cmd, silent=True, wd=None, timeout=None): # XXX: This function is not safe!!! - cfg = dict(check=True, shell=True) + cfg = dict(check=True, shell=True, timeout=timeout) if silent: cfg['stdout'] = subprocess.DEVNULL if wd is not None: