From 31b2d6be75c8f271f83b9118f9d950f73b22b49d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 25 Nov 2021 19:56:27 +0000 Subject: [PATCH] dnn(test): update InferenceEngine tests (4.x) --- modules/dnn/test/test_model.cpp | 46 +++++++++++++++++++++++---- modules/dnn/test/test_tf_importer.cpp | 5 ++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 4c08cc30b9..4f5922182a 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -287,6 +287,12 @@ TEST_P(Test_Model, DetectRegion) CV_TEST_TAG_MEMORY_2GB ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -295,6 +301,7 @@ TEST_P(Test_Model, DetectRegion) #endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + // FIXIT DNN_BACKEND_INFERENCE_ENGINE is misused if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); #endif @@ -340,6 +347,12 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) CV_TEST_TAG_MEMORY_2GB ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -390,7 +403,14 @@ TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) TEST_P(Test_Model, DetectionOutput) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // Exception: Function contains several inputs and outputs with one friendly name! (HETERO bug?) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if defined(INF_ENGINE_RELEASE) + // FIXIT DNN_BACKEND_INFERENCE_ENGINE is misused if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -468,9 +488,9 @@ TEST_P(Test_Model, DetectionMobilenetSSD) } else if (target == DNN_TARGET_MYRIAD) { - scoreDiff = 1.7e-2; + scoreDiff = 0.017; if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - iouDiff = 6.91e-2; + iouDiff = 0.1; } else if (target == DNN_TARGET_CUDA_FP16) { @@ -579,7 +599,8 @@ TEST_P(Test_Model, Detection_normalized) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) { - iouDiff = 0.095f; + scoreDiff = 0.02; + iouDiff = 0.1f; } #endif testDetectModel(weights_file, config_file, img_path, refClassIds, refConfidences, refBoxes, @@ -608,8 +629,13 @@ TEST_P(Test_Model, Segmentation) TEST_P(Test_Model, TextRecognition) { - if (target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name 71 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif std::string imgPath = _tf("text_rec_test.png"); std::string weightPath = _tf("onnx/models/crnn.onnx", false); @@ -627,8 +653,14 @@ TEST_P(Test_Model, TextRecognition) TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) { - if (target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // IE Exception: Ngraph operation Reshape with name 71 has dynamic output shape on 0 port, but CPU plug-in supports only static shape + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + std::string imgPath = _tf("text_rec_test.png"); std::string weightPath = _tf("onnx/models/crnn.onnx", false); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index ed8e8648e0..cbd7401232 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -731,6 +731,10 @@ TEST_P(Test_TensorFlow_layers, BiasAdd) TEST_P(Test_TensorFlow_layers, ExpandDims) { +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Layout::ANY is broken on CPU +#endif #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X @@ -1042,7 +1046,6 @@ TEST_P(Test_TensorFlow_nets, Faster_RCNN_resnet50_coco_2018_01_28) double iouDiff = 1e-4; if (target == DNN_TARGET_CUDA) { - // for faster_rcnn_resnet50_coco_2018_01_28 scoresDiff = 0.06; iouDiff = 0.08; }