From f49b26182b33b4a4c4b4f2eee0b4818f3bc8bad0 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 25 Dec 2023 04:57:02 +0000 Subject: [PATCH] dnn(test): skip very long debug tests, reduce test time --- modules/dnn/perf/perf_convolution.cpp | 51 ++++++++++++++-------- modules/dnn/perf/perf_layer.cpp | 3 ++ modules/dnn/perf/perf_net.cpp | 41 ++++++++++++++--- modules/dnn/test/test_backends.cpp | 4 +- modules/dnn/test/test_caffe_importer.cpp | 13 ++++-- modules/dnn/test/test_darknet_importer.cpp | 11 ++--- modules/dnn/test/test_int8_layers.cpp | 10 ++++- modules/dnn/test/test_model.cpp | 15 +++++-- modules/dnn/test/test_onnx_importer.cpp | 9 +++- modules/dnn/test/test_tf_importer.cpp | 2 +- 10 files changed, 118 insertions(+), 41 deletions(-) diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index 211207307c..2c33969a76 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -4,6 +4,7 @@ #include "perf_precomp.hpp" #include +#include namespace opencv_test { @@ -773,16 +774,14 @@ struct ConvParamGenerator ::testing::internal::ParamGenerator all() const { -#if 1 - // default to 20 for each type of convolution - const int NUM = 20; -#else - const int NUM = size; -#endif - - ConvParam_t v_[NUM]; + int NUM = size; + static size_t DNN_LIMIT_CONV = utils::getConfigurationParameterSizeT("OPENCV_TEST_DNN_LIMIT_CONV", 0); + if (DNN_LIMIT_CONV > 0) + NUM = std::min(NUM, (int)DNN_LIMIT_CONV); + + std::vector v_(NUM); for (int i = 0; i < NUM; ++i) { v_[i] = testConfigs[i]; } // reduce generated code size - return ::testing::ValuesIn(v_, v_ + NUM); + return ::testing::ValuesIn(v_); } }; static inline void PrintTo(const ConvParam_t& p, std::ostream* os) @@ -807,9 +806,20 @@ static inline void PrintTo(const ConvParam_t& p, std::ostream* os) *os << ", BIAS"; } -Net build_net(const ConvParam_t& params, Backend backendId, Target targetId) +static +Net build_net( + const ConvParam_t& params, Backend backendId, Target targetId, + const std::function& configure_network_cb = std::function(), + double flops_limit_debug_long = 2e9, double flops_limit_debug_verylong = 6e9 +) { double declared_flops = params.declared_flops; + + if (flops_limit_debug_verylong > 0 && declared_flops >= flops_limit_debug_verylong) + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + if (flops_limit_debug_long > 0 && declared_flops >= flops_limit_debug_long) + applyTestTag(CV_TEST_TAG_DEBUG_LONG); + Size kernel = params.kernel; MatShape inputShape = MatShape(params.shapeIn.dims, params.shapeIn.dims + 4); int outChannels = params.outCN; @@ -863,9 +873,14 @@ Net build_net(const ConvParam_t& params, Backend backendId, Target targetId) Net net; net.addLayerToPrev(lp.name, lp.type, lp); - net.setInput(input); net.setPreferableBackend(backendId); net.setPreferableTarget(targetId); + if (configure_network_cb) + { + configure_network_cb(net); + } + + net.setInput(input); // warmup Mat output = net.forward(); @@ -928,11 +943,12 @@ PERF_TEST_P_(Conv_3x3S1D1, conv) Backend backendId = get<0>(get<1>(GetParam())); Target targetId = get<1>(get<1>(GetParam())); bool winograd = get<2>(GetParam()); - Net net = build_net(params, backendId, targetId); - net.enableWinograd(winograd); - - // warmup again since configuration is changed - net.forward(); + Net net = build_net(params, backendId, targetId, + [=](Net& net) + { + net.enableWinograd(winograd); + } + ); TEST_CYCLE() { @@ -946,7 +962,8 @@ PERF_TEST_P_(Conv_Depthwise, conv) const ConvParam_t& params = get<0>(GetParam()); Backend backendId = get<0>(get<1>(GetParam())); Target targetId = get<1>(get<1>(GetParam())); - Net net = build_net(params, backendId, targetId); + Net net = build_net(params, backendId, targetId, std::function(), + 0/*flops_limit_debug_long*/, 0/*flops_limit_debug_verylong*/); TEST_CYCLE() { diff --git a/modules/dnn/perf/perf_layer.cpp b/modules/dnn/perf/perf_layer.cpp index 5849b8ff47..66b5ad62c2 100644 --- a/modules/dnn/perf/perf_layer.cpp +++ b/modules/dnn/perf/perf_layer.cpp @@ -829,6 +829,9 @@ PERF_TEST_P_(Layer_FullyConnected, fc) int backendId = get<0>(get<3>(GetParam())); int targetId = get<1>(get<3>(GetParam())); + if (inpShape.size() == 4 && inpShape[0] == 5 && inpShape[1] == 16 && inpShape[2] == 512 && inpShape[3] == 128 && outDims >= 512) + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + std::vector weightShape; if (isMatMul) { weightShape = inpShape; diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index 33e49526e9..6bf56f0719 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -140,6 +140,8 @@ PERF_TEST_P_(DNNTestNetwork, ENet) PERF_TEST_P_(DNNTestNetwork, SSD) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", "disabled", cv::Size(300, 300)); } @@ -190,6 +192,8 @@ PERF_TEST_P_(DNNTestNetwork, DenseNet_121) PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + if (backend == DNN_BACKEND_HALIDE || (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL))) throw SkipTestException(""); @@ -209,6 +213,8 @@ PERF_TEST_P_(DNNTestNetwork, opencv_face_detector) PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "ssd_inception_v2_coco_2017_11_17.pbtxt", "", @@ -217,7 +223,10 @@ PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow) PERF_TEST_P_(DNNTestNetwork, YOLOv3) { - applyTestTag(CV_TEST_TAG_MEMORY_2GB); + applyTestTag( + CV_TEST_TAG_MEMORY_2GB, + CV_TEST_TAG_DEBUG_VERYLONG + ); if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure @@ -238,7 +247,10 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv3) PERF_TEST_P_(DNNTestNetwork, YOLOv4) { - applyTestTag(CV_TEST_TAG_MEMORY_2GB); + applyTestTag( + CV_TEST_TAG_MEMORY_2GB, + CV_TEST_TAG_DEBUG_VERYLONG + ); if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); if (target == DNN_TARGET_MYRIAD) // not enough resources @@ -274,15 +286,23 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv5) { processNet("", "dnn/yolov5n.onnx", "", inp); } -PERF_TEST_P_(DNNTestNetwork, YOLOv8) { - applyTestTag(CV_TEST_TAG_MEMORY_512MB); +PERF_TEST_P_(DNNTestNetwork, YOLOv8) +{ + applyTestTag( + CV_TEST_TAG_MEMORY_512MB, + CV_TEST_TAG_DEBUG_LONG + ); + Mat sample = imread(findDataFile("dnn/dog416.png")); Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(640, 640), Scalar(), true); processNet("", "dnn/yolov8n.onnx", "", inp); } PERF_TEST_P_(DNNTestNetwork, YOLOX) { - applyTestTag(CV_TEST_TAG_MEMORY_512MB); + applyTestTag( + CV_TEST_TAG_MEMORY_512MB, + CV_TEST_TAG_DEBUG_VERYLONG + ); Mat sample = imread(findDataFile("dnn/dog416.png")); Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(640, 640), Scalar(), true); processNet("", "dnn/yolox_s.onnx", "", inp); @@ -290,6 +310,8 @@ PERF_TEST_P_(DNNTestNetwork, YOLOX) { PERF_TEST_P_(DNNTestNetwork, EAST_text_detection) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); processNet("dnn/frozen_east_text_detection.pb", "", "", cv::Size(320, 320)); @@ -297,6 +319,8 @@ PERF_TEST_P_(DNNTestNetwork, EAST_text_detection) PERF_TEST_P_(DNNTestNetwork, FastNeuralStyle_eccv16) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); processNet("dnn/fast_neural_style_eccv16_starry_night.t7", "", "", cv::Size(320, 240)); @@ -304,6 +328,8 @@ PERF_TEST_P_(DNNTestNetwork, FastNeuralStyle_eccv16) PERF_TEST_P_(DNNTestNetwork, Inception_v2_Faster_RCNN) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019010000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) throw SkipTestException("Test is disabled in OpenVINO 2019R1"); @@ -411,7 +437,10 @@ PERF_TEST_P_(DNNTestNetwork, EfficientDet_int8) processNet("", "dnn/tflite/coco_efficientdet_lite0_v1_1.0_quant_2021_09_06.tflite", "", inp); } -PERF_TEST_P_(DNNTestNetwork, VIT_B_32) { +PERF_TEST_P_(DNNTestNetwork, VIT_B_32) +{ + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + processNet("", "dnn/onnx/models/vit_b_32.onnx", "", cv::Size(224, 224)); } diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index d62892bf84..f255ab87aa 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -117,7 +117,7 @@ TEST_P(DNNTestNetwork, ResNet_50) { applyTestTag( (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), - CV_TEST_TAG_DEBUG_LONG + CV_TEST_TAG_DEBUG_VERYLONG ); processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", @@ -455,7 +455,7 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow) { applyTestTag( (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), - CV_TEST_TAG_DEBUG_LONG + CV_TEST_TAG_DEBUG_VERYLONG ); #if defined(INF_ENGINE_RELEASE) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 34b84622fa..4db7796e58 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -270,7 +270,11 @@ TEST(Reproducibility_FCN, Accuracy) TEST(Reproducibility_SSD, Accuracy) { - applyTestTag(CV_TEST_TAG_MEMORY_512MB, CV_TEST_TAG_DEBUG_LONG); + applyTestTag( + CV_TEST_TAG_MEMORY_512MB, + CV_TEST_TAG_DEBUG_VERYLONG + ); + Net net; { const string proto = findDataFile("dnn/ssd_vgg16.prototxt"); @@ -500,7 +504,10 @@ TEST(Reproducibility_GoogLeNet_fp16, Accuracy) // https://github.com/richzhang/colorization TEST_P(Test_Caffe_nets, Colorization) { - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); + applyTestTag( + target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB, + CV_TEST_TAG_DEBUG_VERYLONG + ); checkBackend(); Mat inp = blobFromNPY(_tf("colorization_inp.npy")); @@ -777,7 +784,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_zf) #else (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), #endif - CV_TEST_TAG_DEBUG_LONG + CV_TEST_TAG_DEBUG_VERYLONG ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) // IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 40c07977f8..ba2a7f14c6 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -357,7 +357,8 @@ TEST_P(Test_Darknet_nets, YoloVoc) #else CV_TEST_TAG_MEMORY_1GB, #endif - CV_TEST_TAG_LONG + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure @@ -919,10 +920,10 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) TEST_P(Test_Darknet_nets, YOLOv4x_mish) { applyTestTag( - CV_TEST_TAG_LONG, - CV_TEST_TAG_MEMORY_2GB, - CV_TEST_TAG_DEBUG_VERYLONG - ); + CV_TEST_TAG_MEMORY_2GB, + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) // IE exception: Ngraph operation Transpose with name permute_168 has dynamic output shape on 0 port, but CPU plug-in supports only static shape diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 3c062cdb40..bc5d9388a9 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -748,7 +748,10 @@ TEST_P(Test_Int8_nets, GoogLeNet) TEST_P(Test_Int8_nets, ResNet50) { - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); + applyTestTag( + target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB, + CV_TEST_TAG_DEBUG_VERYLONG + ); if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); @@ -1200,7 +1203,10 @@ TEST_P(Test_Int8_nets, YoloVoc) TEST_P(Test_Int8_nets, TinyYoloVoc) { - applyTestTag(CV_TEST_TAG_MEMORY_512MB); + applyTestTag( + CV_TEST_TAG_MEMORY_512MB, + CV_TEST_TAG_DEBUG_VERYLONG + ); if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index f891d55ab5..6cfdc38219 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -288,8 +288,9 @@ TEST_P(Test_Model, Classify) TEST_P(Test_Model, DetectRegion) { applyTestTag( + CV_TEST_TAG_MEMORY_2GB, CV_TEST_TAG_LONG, - CV_TEST_TAG_MEMORY_2GB + CV_TEST_TAG_DEBUG_VERYLONG ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) @@ -348,8 +349,9 @@ TEST_P(Test_Model, DetectRegion) TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) { applyTestTag( + CV_TEST_TAG_MEMORY_2GB, CV_TEST_TAG_LONG, - CV_TEST_TAG_MEMORY_2GB + CV_TEST_TAG_DEBUG_VERYLONG ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) @@ -408,6 +410,8 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) TEST_P(Test_Model, DetectionOutput) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) // Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:427: // While validating node 'v1::Reshape bbox_pred_reshape (ave_bbox_pred_rois[0]:f32{1,8,1,1}, Constant_388[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape': @@ -631,7 +635,8 @@ TEST_P(Test_Model, Detection_normalized) TEST_P(Test_Model, Segmentation) { applyTestTag( - CV_TEST_TAG_MEMORY_2GB + CV_TEST_TAG_MEMORY_2GB, + CV_TEST_TAG_DEBUG_VERYLONG ); float norm = 0; @@ -746,6 +751,8 @@ TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) TEST_P(Test_Model, TextDetectionByDB) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); if (target == DNN_TARGET_CPU_FP16) @@ -788,6 +795,8 @@ TEST_P(Test_Model, TextDetectionByDB) TEST_P(Test_Model, TextDetectionByEAST) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + std::string imgPath = _tf("text_det_test2.jpg"); std::string weightPath = _tf("frozen_east_text_detection.pb", false); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 39565b12eb..ad6efbe77a 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -2390,7 +2390,7 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR) #else (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), #endif - CV_TEST_TAG_DEBUG_LONG + CV_TEST_TAG_DEBUG_VERYLONG ); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { @@ -2713,6 +2713,8 @@ void yoloPostProcessing( TEST_P(Test_ONNX_nets, YOLOX) { + applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); + std::string weightPath = _tf("models/yolox_s_inf_decoder.onnx", false); Size targetSize{640, 640}; @@ -2786,7 +2788,10 @@ TEST_P(Test_ONNX_nets, YOLOv8) // 4. 1D mat dimension issue with the output of range operator TEST_P(Test_ONNX_nets, YOLOv7) { - applyTestTag(CV_TEST_TAG_MEMORY_2GB); + applyTestTag( + CV_TEST_TAG_MEMORY_2GB, + CV_TEST_TAG_DEBUG_VERYLONG + ); std::string weightPath = _tf("models/yolov7_not_simplified.onnx", false); // Reference, which is collected with input size of 640x640 diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 34f24dd26b..964fcbbbe8 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -1282,7 +1282,7 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection) { applyTestTag( (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), - CV_TEST_TAG_DEBUG_LONG + CV_TEST_TAG_DEBUG_VERYLONG ); #if defined(INF_ENGINE_RELEASE)