Merge pull request #24763 from opencv-pushbot:gitee/alalek/test_dnn_skip_update_debug

modules/dnn/perf/perf_convolution.cpp

@@ -4,6 +4,7 @@
 
 #include "perf_precomp.hpp"
 #include <opencv2/dnn/shape_utils.hpp>
+#include <opencv2/core/utils/configuration.private.hpp>
 
 namespace opencv_test {
 
@@ -773,16 +774,14 @@ struct ConvParamGenerator
 
     ::testing::internal::ParamGenerator<ConvParam_t> all() const
     {
-#if 1
+        int NUM = size;
-        // default to 20 for each type of convolution
+        static size_t DNN_LIMIT_CONV = utils::getConfigurationParameterSizeT("OPENCV_TEST_DNN_LIMIT_CONV", 0);
-        const int NUM = 20;
+        if (DNN_LIMIT_CONV > 0)
-#else
+            NUM = std::min(NUM, (int)DNN_LIMIT_CONV);
-        const int NUM = size;
+
-#endif
+        std::vector<ConvParam_t> v_(NUM);
-
-        ConvParam_t v_[NUM];
         for (int i = 0; i < NUM; ++i) { v_[i] = testConfigs[i]; } // reduce generated code size
-        return ::testing::ValuesIn(v_, v_ + NUM);
+        return ::testing::ValuesIn(v_);
     }
 };
 static inline void PrintTo(const ConvParam_t& p, std::ostream* os)
@@ -807,9 +806,20 @@ static inline void PrintTo(const ConvParam_t& p, std::ostream* os)
         *os << ", BIAS";
 }
 
-Net build_net(const ConvParam_t& params, Backend backendId, Target targetId)
+static
+Net build_net(
+    const ConvParam_t& params, Backend backendId, Target targetId,
+    const std::function<void(Net&)>& configure_network_cb = std::function<void(Net&)>(),
+    double flops_limit_debug_long = 2e9, double flops_limit_debug_verylong = 6e9
+)
 {
     double declared_flops = params.declared_flops;
+
+    if (flops_limit_debug_verylong > 0 && declared_flops >= flops_limit_debug_verylong)
+        applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+    if (flops_limit_debug_long > 0 && declared_flops >= flops_limit_debug_long)
+        applyTestTag(CV_TEST_TAG_DEBUG_LONG);
+
     Size kernel = params.kernel;
     MatShape inputShape = MatShape(params.shapeIn.dims, params.shapeIn.dims + 4);
     int outChannels = params.outCN;
@@ -863,9 +873,14 @@ Net build_net(const ConvParam_t& params, Backend backendId, Target targetId)
     Net net;
     net.addLayerToPrev(lp.name, lp.type, lp);
 
-    net.setInput(input);
     net.setPreferableBackend(backendId);
     net.setPreferableTarget(targetId);
+    if (configure_network_cb)
+    {
+        configure_network_cb(net);
+    }
+
+    net.setInput(input);
 
     // warmup
     Mat output = net.forward();
@@ -928,11 +943,12 @@ PERF_TEST_P_(Conv_3x3S1D1, conv)
     Backend backendId = get<0>(get<1>(GetParam()));
     Target targetId = get<1>(get<1>(GetParam()));
     bool winograd = get<2>(GetParam());
-    Net net = build_net(params, backendId, targetId);
+    Net net = build_net(params, backendId, targetId,
-    net.enableWinograd(winograd);
+        [=](Net& net)
-
+        {
-    // warmup again since configuration is changed
+            net.enableWinograd(winograd);
-    net.forward();
+        }
+    );
 
     TEST_CYCLE()
     {
@@ -946,7 +962,8 @@ PERF_TEST_P_(Conv_Depthwise, conv)
     const ConvParam_t& params = get<0>(GetParam());
     Backend backendId = get<0>(get<1>(GetParam()));
     Target targetId = get<1>(get<1>(GetParam()));
-    Net net = build_net(params, backendId, targetId);
+    Net net = build_net(params, backendId, targetId, std::function<void(Net&)>(),
+        0/*flops_limit_debug_long*/, 0/*flops_limit_debug_verylong*/);
 
     TEST_CYCLE()
     {

modules/dnn/perf/perf_layer.cpp

@@ -829,6 +829,9 @@ PERF_TEST_P_(Layer_FullyConnected, fc)
     int backendId = get<0>(get<3>(GetParam()));
     int targetId = get<1>(get<3>(GetParam()));
 
+    if (inpShape.size() == 4 && inpShape[0] == 5 && inpShape[1] == 16 && inpShape[2] == 512 && inpShape[3] == 128 && outDims >= 512)
+        applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     std::vector<int> weightShape;
     if (isMatMul) {
         weightShape = inpShape;

modules/dnn/perf/perf_net.cpp

@@ -140,6 +140,8 @@ PERF_TEST_P_(DNNTestNetwork, ENet)
 
 PERF_TEST_P_(DNNTestNetwork, SSD)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", "disabled",
             cv::Size(300, 300));
 }
@@ -190,6 +192,8 @@ PERF_TEST_P_(DNNTestNetwork, DenseNet_121)
 
 PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     if (backend == DNN_BACKEND_HALIDE ||
         (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL)))
         throw SkipTestException("");
@@ -209,6 +213,8 @@ PERF_TEST_P_(DNNTestNetwork, opencv_face_detector)
 
 PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "ssd_inception_v2_coco_2017_11_17.pbtxt", "",
@@ -217,7 +223,10 @@ PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
 
 PERF_TEST_P_(DNNTestNetwork, YOLOv3)
 {
-    applyTestTag(CV_TEST_TAG_MEMORY_2GB);
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_2GB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
     if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
@@ -238,7 +247,10 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv3)
 
 PERF_TEST_P_(DNNTestNetwork, YOLOv4)
 {
-    applyTestTag(CV_TEST_TAG_MEMORY_2GB);
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_2GB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
     if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     if (target == DNN_TARGET_MYRIAD)  // not enough resources
@@ -274,15 +286,23 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv5) {
     processNet("", "dnn/yolov5n.onnx", "", inp);
 }
 
-PERF_TEST_P_(DNNTestNetwork, YOLOv8) {
+PERF_TEST_P_(DNNTestNetwork, YOLOv8)
-    applyTestTag(CV_TEST_TAG_MEMORY_512MB);
+{
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_512MB,
+        CV_TEST_TAG_DEBUG_LONG
+    );
+
     Mat sample = imread(findDataFile("dnn/dog416.png"));
     Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(640, 640), Scalar(), true);
     processNet("", "dnn/yolov8n.onnx", "", inp);
 }
 
 PERF_TEST_P_(DNNTestNetwork, YOLOX) {
-    applyTestTag(CV_TEST_TAG_MEMORY_512MB);
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_512MB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
     Mat sample = imread(findDataFile("dnn/dog416.png"));
     Mat inp = blobFromImage(sample, 1.0 / 255.0, Size(640, 640), Scalar(), true);
     processNet("", "dnn/yolox_s.onnx", "", inp);
@@ -290,6 +310,8 @@ PERF_TEST_P_(DNNTestNetwork, YOLOX) {
 
 PERF_TEST_P_(DNNTestNetwork, EAST_text_detection)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     processNet("dnn/frozen_east_text_detection.pb", "", "", cv::Size(320, 320));
@@ -297,6 +319,8 @@ PERF_TEST_P_(DNNTestNetwork, EAST_text_detection)
 
 PERF_TEST_P_(DNNTestNetwork, FastNeuralStyle_eccv16)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     processNet("dnn/fast_neural_style_eccv16_starry_night.t7", "", "", cv::Size(320, 240));
@@ -304,6 +328,8 @@ PERF_TEST_P_(DNNTestNetwork, FastNeuralStyle_eccv16)
 
 PERF_TEST_P_(DNNTestNetwork, Inception_v2_Faster_RCNN)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019010000)
     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
         throw SkipTestException("Test is disabled in OpenVINO 2019R1");
@@ -411,7 +437,10 @@ PERF_TEST_P_(DNNTestNetwork, EfficientDet_int8)
     processNet("", "dnn/tflite/coco_efficientdet_lite0_v1_1.0_quant_2021_09_06.tflite", "", inp);
 }
 
-PERF_TEST_P_(DNNTestNetwork, VIT_B_32) {
+PERF_TEST_P_(DNNTestNetwork, VIT_B_32)
+{
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     processNet("", "dnn/onnx/models/vit_b_32.onnx", "", cv::Size(224, 224));
 }
 

modules/dnn/test/test_backends.cpp

@@ -117,7 +117,7 @@ TEST_P(DNNTestNetwork, ResNet_50)
 {
     applyTestTag(
         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
-        CV_TEST_TAG_DEBUG_LONG
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 
     processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
@@ -455,7 +455,7 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
 {
     applyTestTag(
         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
-        CV_TEST_TAG_DEBUG_LONG
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 #if defined(INF_ENGINE_RELEASE)
     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD

modules/dnn/test/test_caffe_importer.cpp

@@ -270,7 +270,11 @@ TEST(Reproducibility_FCN, Accuracy)
 
 TEST(Reproducibility_SSD, Accuracy)
 {
-    applyTestTag(CV_TEST_TAG_MEMORY_512MB, CV_TEST_TAG_DEBUG_LONG);
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_512MB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
+
     Net net;
     {
         const string proto = findDataFile("dnn/ssd_vgg16.prototxt");
@@ -500,7 +504,10 @@ TEST(Reproducibility_GoogLeNet_fp16, Accuracy)
 // https://github.com/richzhang/colorization
 TEST_P(Test_Caffe_nets, Colorization)
 {
-    applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
+    applyTestTag(
+        target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
     checkBackend();
 
     Mat inp = blobFromNPY(_tf("colorization_inp.npy"));
@@ -777,7 +784,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_zf)
 #else
         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 #endif
-        CV_TEST_TAG_DEBUG_LONG
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
     // IE exception: Ngraph operation Reshape with name rpn_cls_score_reshape has dynamic output shape on 0 port, but CPU plug-in supports only static shape

modules/dnn/test/test_darknet_importer.cpp

@@ -357,7 +357,8 @@ TEST_P(Test_Darknet_nets, YoloVoc)
 #else
         CV_TEST_TAG_MEMORY_1GB,
 #endif
-        CV_TEST_TAG_LONG
+        CV_TEST_TAG_LONG,
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
@@ -919,10 +920,10 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny)
 TEST_P(Test_Darknet_nets, YOLOv4x_mish)
 {
     applyTestTag(
-            CV_TEST_TAG_LONG,
+        CV_TEST_TAG_MEMORY_2GB,
-            CV_TEST_TAG_MEMORY_2GB,
+        CV_TEST_TAG_LONG,
-            CV_TEST_TAG_DEBUG_VERYLONG
+        CV_TEST_TAG_DEBUG_VERYLONG
-            );
+    );
 
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
     // IE exception: Ngraph operation Transpose with name permute_168 has dynamic output shape on 0 port, but CPU plug-in supports only static shape

modules/dnn/test/test_int8_layers.cpp

@@ -748,7 +748,10 @@ TEST_P(Test_Int8_nets, GoogLeNet)
 
 TEST_P(Test_Int8_nets, ResNet50)
 {
-    applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
+    applyTestTag(
+        target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
 
     if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel())
         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
@@ -1200,7 +1203,10 @@ TEST_P(Test_Int8_nets, YoloVoc)
 
 TEST_P(Test_Int8_nets, TinyYoloVoc)
 {
-    applyTestTag(CV_TEST_TAG_MEMORY_512MB);
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_512MB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
 
     if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel())
         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);

modules/dnn/test/test_model.cpp

@@ -288,8 +288,9 @@ TEST_P(Test_Model, Classify)
 TEST_P(Test_Model, DetectRegion)
 {
     applyTestTag(
+        CV_TEST_TAG_MEMORY_2GB,
         CV_TEST_TAG_LONG,
-        CV_TEST_TAG_MEMORY_2GB
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
@@ -348,8 +349,9 @@ TEST_P(Test_Model, DetectRegion)
 TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses)
 {
     applyTestTag(
+        CV_TEST_TAG_MEMORY_2GB,
         CV_TEST_TAG_LONG,
-        CV_TEST_TAG_MEMORY_2GB
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
@@ -408,6 +410,8 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses)
 
 TEST_P(Test_Model, DetectionOutput)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
     // Check 'backward_compatible_check || in_out_elements_equal' failed at core/src/op/reshape.cpp:427:
     // While validating node 'v1::Reshape bbox_pred_reshape (ave_bbox_pred_rois[0]:f32{1,8,1,1}, Constant_388[0]:i64{4}) -> (f32{?,?,?,?})' with friendly_name 'bbox_pred_reshape':
@@ -631,7 +635,8 @@ TEST_P(Test_Model, Detection_normalized)
 TEST_P(Test_Model, Segmentation)
 {
     applyTestTag(
-        CV_TEST_TAG_MEMORY_2GB
+        CV_TEST_TAG_MEMORY_2GB,
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 
     float norm = 0;
@@ -746,6 +751,8 @@ TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch)
 
 TEST_P(Test_Model, TextDetectionByDB)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     if (target == DNN_TARGET_OPENCL_FP16)
         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
     if (target == DNN_TARGET_CPU_FP16)
@@ -788,6 +795,8 @@ TEST_P(Test_Model, TextDetectionByDB)
 
 TEST_P(Test_Model, TextDetectionByEAST)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     std::string imgPath = _tf("text_det_test2.jpg");
     std::string weightPath = _tf("frozen_east_text_detection.pb", false);
 

modules/dnn/test/test_onnx_importer.cpp

@@ -2390,7 +2390,7 @@ TEST_P(Test_ONNX_nets, LResNet100E_IR)
 #else
         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 #endif
-        CV_TEST_TAG_DEBUG_LONG
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
     {
@@ -2713,6 +2713,8 @@ void yoloPostProcessing(
 
 TEST_P(Test_ONNX_nets, YOLOX)
 {
+    applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
+
     std::string weightPath = _tf("models/yolox_s_inf_decoder.onnx", false);
 
     Size targetSize{640, 640};
@@ -2786,7 +2788,10 @@ TEST_P(Test_ONNX_nets, YOLOv8)
 //  4. 1D mat dimension issue with the output of range operator
 TEST_P(Test_ONNX_nets, YOLOv7)
 {
-    applyTestTag(CV_TEST_TAG_MEMORY_2GB);
+    applyTestTag(
+        CV_TEST_TAG_MEMORY_2GB,
+        CV_TEST_TAG_DEBUG_VERYLONG
+    );
 
     std::string weightPath = _tf("models/yolov7_not_simplified.onnx", false);
     // Reference, which is collected with input size of 640x640

modules/dnn/test/test_tf_importer.cpp

@@ -1282,7 +1282,7 @@ TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 {
     applyTestTag(
         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
-        CV_TEST_TAG_DEBUG_LONG
+        CV_TEST_TAG_DEBUG_VERYLONG
     );
 
 #if defined(INF_ENGINE_RELEASE)