Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dnn(test-tags): add time / memory tags

Browse Source
pull/12783/head
Alexander Alekhin 7 years ago
parent b38de57f9a
commit e0841f3d6e
6 changed files with 65 additions and 12 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      modules/dnn/test/test_backends.cpp
  2. 17
      modules/dnn/test/test_caffe_importer.cpp
  3. 7
      modules/dnn/test/test_darknet_importer.cpp
  4. 29
      modules/dnn/test/test_onnx_importer.cpp
  5. 6
      modules/dnn/test/test_tf_importer.cpp
  6. 1
      modules/dnn/test/test_torch_importer.cpp

17
modules/dnn/test/test_backends.cpp
Unescape View File

@ -98,6 +98,7 @@ public:
TEST_P(DNNTestNetwork, AlexNet)
{
applyTestTag(CV_TEST_TAG_MEMORY_1GB);
processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt",
Size(227, 227), "prob",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_alexnet.yml" :
@ -106,6 +107,7 @@ TEST_P(DNNTestNetwork, AlexNet)
TEST_P(DNNTestNetwork, ResNet_50)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
Size(224, 224), "prob",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_resnet_50.yml" :
@ -122,12 +124,14 @@ TEST_P(DNNTestNetwork, SqueezeNet_v1_1)
TEST_P(DNNTestNetwork, GoogLeNet)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt",
Size(224, 224), "prob");
}
TEST_P(DNNTestNetwork, Inception_5h)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
double l1 = default_l1, lInf = default_lInf;
if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_CPU || target == DNN_TARGET_OPENCL))
{
@ -142,6 +146,7 @@ TEST_P(DNNTestNetwork, Inception_5h)
TEST_P(DNNTestNetwork, ENet)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
if ((backend == DNN_BACKEND_INFERENCE_ENGINE) ||
(backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("");
@ -153,6 +158,7 @@ TEST_P(DNNTestNetwork, ENet)
TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
@ -184,6 +190,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height)
TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
@ -214,6 +221,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height)
TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
@ -226,6 +234,8 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
TEST_P(DNNTestNetwork, SSD_VGG16)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU)
throw SkipTestException("");
double scoreThreshold = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0325 : 0.0;
@ -238,6 +248,8 @@ TEST_P(DNNTestNetwork, SSD_VGG16)
TEST_P(DNNTestNetwork, OpenPose_pose_coco)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
@ -254,6 +266,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco)
TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
CV_TEST_TAG_DEBUG_VERYLONG);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
@ -270,6 +284,7 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
@ -318,6 +333,7 @@ TEST_P(DNNTestNetwork, opencv_face_detector)
TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
@ -335,6 +351,7 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
TEST_P(DNNTestNetwork, DenseNet_121)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
if (backend == DNN_BACKEND_HALIDE)
throw SkipTestException("");
// Reference output values are in range [-3.807, 4.605]

17
modules/dnn/test/test_caffe_importer.cpp
Unescape View File

@ -112,6 +112,8 @@ TEST(Test_Caffe, read_googlenet)
typedef testing::TestWithParam<tuple<bool, Target> > Reproducibility_AlexNet;
TEST_P(Reproducibility_AlexNet, Accuracy)
{
Target targetId = get<1>(GetParam());
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
bool readFromMemory = get<0>(GetParam());
Net net;
{
@ -132,7 +134,6 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
ASSERT_FALSE(net.empty());
}
int targetId = get<1>(GetParam());
const float l1 = 1e-5;
const float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 3e-3 : 1e-4;
@ -151,9 +152,9 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(),
Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16)));
#if !defined(_WIN32) || defined(_WIN64)
TEST(Reproducibility_FCN, Accuracy)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_2GB);
Net net;
{
const string proto = findDataFile("dnn/fcn8s-heavy-pascal.prototxt", false);
@ -179,10 +180,10 @@ TEST(Reproducibility_FCN, Accuracy)
normAssert(ref, out);
}
#endif
TEST(Reproducibility_SSD, Accuracy)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
Net net;
{
const string proto = findDataFile("dnn/ssd_vgg16.prototxt", false);
@ -264,10 +265,11 @@ INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD,
typedef testing::TestWithParam<Target> Reproducibility_ResNet50;
TEST_P(Reproducibility_ResNet50, Accuracy)
{
Target targetId = GetParam();
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt", false),
findDataFile("dnn/ResNet-50-model.caffemodel", false));
int targetId = GetParam();
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId);
@ -330,6 +332,7 @@ INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_SqueezeNet_v1_1,
TEST(Reproducibility_AlexNet_fp16, Accuracy)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
const float l1 = 1e-5;
const float lInf = 3e-3;
@ -375,6 +378,7 @@ TEST(Reproducibility_GoogLeNet_fp16, Accuracy)
// https://github.com/richzhang/colorization
TEST_P(Test_Caffe_nets, Colorization)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
checkBackend();
Mat inp = blobFromNPY(_tf("colorization_inp.npy"));
@ -405,6 +409,7 @@ TEST_P(Test_Caffe_nets, Colorization)
TEST_P(Test_Caffe_nets, DenseNet_121)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
checkBackend();
const string proto = findDataFile("dnn/DenseNet_121.prototxt", false);
const string model = findDataFile("dnn/DenseNet_121.caffemodel", false);
@ -520,6 +525,8 @@ INSTANTIATE_TEST_CASE_P(Test_Caffe, opencv_face_detector,
TEST_P(Test_Caffe_nets, FasterRCNN_vgg16)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("Test is disabled for DLIE OpenCL targets"); // very slow
@ -536,6 +543,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_vgg16)
TEST_P(Test_Caffe_nets, FasterRCNN_zf)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
if ((backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) ||
(backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD))
throw SkipTestException("");
@ -547,6 +555,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_zf)
TEST_P(Test_Caffe_nets, RFCN)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB));
if ((backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) ||
(backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD))
throw SkipTestException("");

7
modules/dnn/test/test_darknet_importer.cpp
Unescape View File

@ -78,6 +78,7 @@ TEST(Test_Darknet, read_yolo_voc)
TEST(Test_Darknet, read_yolo_voc_stream)
{
applyTestTag(CV_TEST_TAG_MEMORY_1GB);
Mat ref;
Mat sample = imread(_tf("dog416.png"));
Mat inp = blobFromImage(sample, 1.0/255, Size(416, 416), Scalar(), true, false);
@ -267,6 +268,8 @@ public:
TEST_P(Test_Darknet_nets, YoloVoc)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16)
throw SkipTestException("Test is disabled");
@ -305,6 +308,8 @@ TEST_P(Test_Darknet_nets, YoloVoc)
TEST_P(Test_Darknet_nets, TinyYoloVoc)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
@ -339,6 +344,8 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc)
TEST_P(Test_Darknet_nets, YOLOv3)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)

29
modules/dnn/test/test_onnx_importer.cpp
Unescape View File

@ -217,6 +217,7 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets());
class Test_ONNX_nets : public Test_ONNX_layers {};
TEST_P(Test_ONNX_nets, Alexnet)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
const String model = _tf("models/alexnet.onnx");
Net net = readNetFromONNX(model);
@ -270,31 +271,30 @@ TEST_P(Test_ONNX_nets, Googlenet)
TEST_P(Test_ONNX_nets, CaffeNet)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
testONNXModels("caffenet", pb);
}
TEST_P(Test_ONNX_nets, RCNN_ILSVRC13)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
// Reference output values are in range [-4.992, -1.161]
testONNXModels("rcnn_ilsvrc13", pb, 0.0045);
}
#ifdef OPENCV_32BIT_CONFIGURATION
TEST_P(Test_ONNX_nets, DISABLED_VGG16) // memory usage >2Gb
#else
TEST_P(Test_ONNX_nets, VGG16)
#endif
{
applyTestTag(CV_TEST_TAG_MEMORY_6GB); // > 2.3Gb
// output range: [-69; 72], after Softmax [0; 0.96]
testONNXModels("vgg16", pb, default_l1, default_lInf, true);
}
#ifdef OPENCV_32BIT_CONFIGURATION
TEST_P(Test_ONNX_nets, DISABLED_VGG16_bn) // memory usage >2Gb
#else
TEST_P(Test_ONNX_nets, VGG16_bn)
#endif
{
applyTestTag(CV_TEST_TAG_MEMORY_6GB); // > 2.3Gb
// output range: [-16; 27], after Softmax [0; 0.67]
const double lInf = (target == DNN_TARGET_MYRIAD) ? 0.038 : default_lInf;
testONNXModels("vgg16-bn", pb, default_l1, lInf, true);
@ -302,23 +302,30 @@ TEST_P(Test_ONNX_nets, VGG16_bn)
TEST_P(Test_ONNX_nets, ZFNet)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
testONNXModels("zfnet512", pb);
}
TEST_P(Test_ONNX_nets, ResNet18v1)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
// output range: [-16; 22], after Softmax [0, 0.51]
testONNXModels("resnet18v1", pb, default_l1, default_lInf, true);
}
TEST_P(Test_ONNX_nets, ResNet50v1)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
// output range: [-67; 75], after Softmax [0, 0.98]
testONNXModels("resnet50v1", pb, default_l1, default_lInf, true);
}
TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC)
{
applyTestTag(CV_TEST_TAG_VERYLONG);
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE)
throw SkipTestException("Test is disabled for DLIE targets");
@ -334,6 +341,8 @@ TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC)
TEST_P(Test_ONNX_nets, TinyYolov2)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
if (cvtest::skipUnstableTests)
throw SkipTestException("Skip unstable test");
#if defined(INF_ENGINE_RELEASE)
@ -347,6 +356,7 @@ TEST_P(Test_ONNX_nets, TinyYolov2)
)
throw SkipTestException("Test is disabled for MyriadX");
#endif
// output range: [-11; 8]
double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.017 : default_l1;
double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.14 : default_lInf;
@ -367,6 +377,7 @@ TEST_P(Test_ONNX_nets, MobileNet_v2)
TEST_P(Test_ONNX_nets, LResNet100E_IR)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
(target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_OPENCL || target == DNN_TARGET_MYRIAD))
throw SkipTestException("");
@ -419,6 +430,8 @@ TEST_P(Test_ONNX_nets, Inception_v2)
TEST_P(Test_ONNX_nets, DenseNet121)
{
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
// output range: [-87; 138], after Softmax [0; 1]
testONNXModels("densenet121", pb, default_l1, default_lInf, true);
}

6
modules/dnn/test/test_tf_importer.cpp
Unescape View File

@ -361,6 +361,8 @@ TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
&& getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
@ -426,6 +428,7 @@ TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
TEST_P(Test_TensorFlow_nets, Faster_RCNN)
{
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); // FIXIT split test
static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
"faster_rcnn_resnet50_coco_2018_01_28"};
@ -521,6 +524,8 @@ TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
// np.save('east_text_detection.geometry.npy', geometry)
TEST_P(Test_TensorFlow_nets, EAST_text_detection)
{
applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
#if defined(INF_ENGINE_RELEASE)
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
throw SkipTestException("Test is disabled for Myriad targets");
@ -695,6 +700,7 @@ TEST(Test_TensorFlow, two_inputs)
TEST(Test_TensorFlow, Mask_RCNN)
{
applyTestTag(CV_TEST_TAG_MEMORY_1GB);
std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt", false);
std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);

1
modules/dnn/test/test_torch_importer.cpp
Unescape View File

@ -345,6 +345,7 @@ static void normAssertSegmentation(const Mat& ref, const Mat& test)
TEST_P(Test_Torch_nets, ENet_accuracy)
{
applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
checkBackend();
if (backend == DNN_BACKEND_INFERENCE_ENGINE ||
(backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))

Write Preview
Loading…
Cancel
Save