From 28e08ae0bd5ae3106700229c9cf94513833730c2 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 20 Jul 2018 18:58:37 +0300 Subject: [PATCH 1/3] Add a sample which tests OpenVINO models --- modules/dnn/CMakeLists.txt | 6 + modules/dnn/src/op_inf_engine.cpp | 7 +- modules/dnn/test/test_ie_models.cpp | 220 ++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 modules/dnn/test/test_ie_models.cpp diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index a2f741cda1..a4cdc18cf7 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -120,3 +120,9 @@ if(BUILD_PERF_TESTS) endif() endif() endif() + +# Test Intel's Inference Engine models +if(HAVE_INF_ENGINE AND TARGET opencv_test_dnn) + ocv_target_include_directories(opencv_test_dnn PRIVATE ${INF_ENGINE_INCLUDE_DIRS}) + ocv_target_link_libraries(opencv_test_dnn LINK_PRIVATE ${INF_ENGINE_LIBRARIES}) +endif() diff --git a/modules/dnn/src/op_inf_engine.cpp b/modules/dnn/src/op_inf_engine.cpp index bcf2c2a3d9..a7c13f3a13 100644 --- a/modules/dnn/src/op_inf_engine.cpp +++ b/modules/dnn/src/op_inf_engine.cpp @@ -428,9 +428,8 @@ void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net) try { - static std::map sharedPlugins; - std::string deviceName = InferenceEngine::getDeviceName(targetDevice); - auto pluginIt = sharedPlugins.find(deviceName); + static std::map sharedPlugins; + auto pluginIt = sharedPlugins.find(targetDevice); if (pluginIt != sharedPlugins.end()) { enginePtr = pluginIt->second; @@ -438,7 +437,7 @@ void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net) else { enginePtr = InferenceEngine::PluginDispatcher({""}).getSuitablePlugin(targetDevice); - sharedPlugins[deviceName] = enginePtr; + sharedPlugins[targetDevice] = enginePtr; if (targetDevice == InferenceEngine::TargetDevice::eCPU) { diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp new file mode 100644 index 0000000000..80c8ef3bc1 --- /dev/null +++ b/modules/dnn/test/test_ie_models.cpp @@ -0,0 +1,220 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2018, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +#include "test_precomp.hpp" + +#ifdef HAVE_INF_ENGINE +#include + +#include +#include +#include + +static std::string extraTestDataPath = +#ifdef WINRT + NULL; +#else + getenv("INTEL_CVSDK_DIR"); +#endif + +namespace opencv_test { namespace { + +using namespace cv; +using namespace cv::dnn; +using namespace InferenceEngine; + +static inline void genData(const std::vector& dims, Mat& m, Blob::Ptr& dataPtr) +{ + std::vector reversedDims(dims.begin(), dims.end()); + std::reverse(reversedDims.begin(), reversedDims.end()); + + m.create(reversedDims, CV_32F); + randu(m, -1, 1); + + dataPtr = make_shared_blob(Precision::FP32, dims, (float*)m.data); +} + +void runIE(Target target, const std::string& xmlPath, const std::string& binPath, + std::map& inputsMap, std::map& outputsMap) +{ + CNNNetReader reader; + reader.ReadNetwork(xmlPath); + reader.ReadWeights(binPath); + + CNNNetwork net = reader.getNetwork(); + + InferenceEnginePluginPtr enginePtr; + InferencePlugin plugin; + ExecutableNetwork netExec; + InferRequest infRequest; + TargetDevice targetDevice; + switch (target) + { + case DNN_TARGET_CPU: + targetDevice = TargetDevice::eCPU; + break; + case DNN_TARGET_OPENCL: + case DNN_TARGET_OPENCL_FP16: + targetDevice = TargetDevice::eGPU; + break; + case DNN_TARGET_MYRIAD: + targetDevice = TargetDevice::eMYRIAD; + break; + default: + CV_Error(Error::StsNotImplemented, "Unknown target"); + }; + + try + { + enginePtr = PluginDispatcher({""}).getSuitablePlugin(targetDevice); + + if (targetDevice == TargetDevice::eCPU) + { + std::string suffixes[] = {"_avx2", "_sse4", ""}; + bool haveFeature[] = { + checkHardwareSupport(CPU_AVX2), + checkHardwareSupport(CPU_SSE4_2), + true + }; + for (int i = 0; i < 3; ++i) + { + if (!haveFeature[i]) + continue; +#ifdef _WIN32 + std::string libName = "cpu_extension" + suffixes[i] + ".dll"; +#else + std::string libName = "libcpu_extension" + suffixes[i] + ".so"; +#endif // _WIN32 + try + { + IExtensionPtr extension = make_so_pointer(libName); + enginePtr->AddExtension(extension, 0); + break; + } + catch(...) {} + } + // Some of networks can work without a library of extra layers. + } + plugin = InferencePlugin(enginePtr); + + netExec = plugin.LoadNetwork(net, {}); + infRequest = netExec.CreateInferRequest(); + } + catch (const std::exception& ex) + { + CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what())); + } + + // Fill input blobs. + inputsMap.clear(); + BlobMap inputBlobs; + for (auto& it : net.getInputsInfo()) + { + genData(it.second->getDims(), inputsMap[it.first], inputBlobs[it.first]); + } + infRequest.SetInput(inputBlobs); + + // Fill output blobs. + outputsMap.clear(); + BlobMap outputBlobs; + for (auto& it : net.getOutputsInfo()) + { + genData(it.second->dims, outputsMap[it.first], outputBlobs[it.first]); + } + infRequest.SetOutput(outputBlobs); + + infRequest.Infer(); +} + +std::vector getOutputsNames(const Net& net) +{ + std::vector names; + if (names.empty()) + { + std::vector outLayers = net.getUnconnectedOutLayers(); + std::vector layersNames = net.getLayerNames(); + names.resize(outLayers.size()); + for (size_t i = 0; i < outLayers.size(); ++i) + names[i] = layersNames[outLayers[i] - 1]; + } + return names; +} + +void runCV(Target target, const std::string& xmlPath, const std::string& binPath, + const std::map& inputsMap, + std::map& outputsMap) +{ + Net net = readNet(xmlPath, binPath); + for (auto& it : inputsMap) + net.setInput(it.second, it.first); + net.setPreferableTarget(target); + + std::vector outNames = getOutputsNames(net); + std::vector outs; + net.forward(outs, outNames); + + outputsMap.clear(); + EXPECT_EQ(outs.size(), outNames.size()); + for (int i = 0; i < outs.size(); ++i) + { + EXPECT_TRUE(outputsMap.insert({outNames[i], outs[i]}).second); + } +} + +typedef TestWithParam > DNNTestOpenVINO; +TEST_P(DNNTestOpenVINO, models) +{ + Target target = (dnn::Target)(int)get<0>(GetParam()); + std::string modelName = get<1>(GetParam()); + + if (modelName == "semantic-segmentation-adas-0001" && target == DNN_TARGET_OPENCL_FP16) + throw SkipTestException(""); + + std::string precision = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? "FP16" : "FP32"; + std::string prefix = utils::fs::join(extraTestDataPath, + utils::fs::join("deployment_tools", + utils::fs::join("intel_models", + utils::fs::join(modelName, + utils::fs::join(precision, modelName))))); + std::string xmlPath = prefix + ".xml"; + std::string binPath = prefix + ".bin"; + + std::map inputsMap; + std::map ieOutputsMap, cvOutputsMap; + runIE(target, xmlPath, binPath, inputsMap, ieOutputsMap); + runCV(target, xmlPath, binPath, inputsMap, cvOutputsMap); + + EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size()); + for (auto& srcIt : ieOutputsMap) + { + auto dstIt = cvOutputsMap.find(srcIt.first); + CV_Assert(dstIt != cvOutputsMap.end()); + double normInf = cvtest::norm(srcIt.second, dstIt->second, cv::NORM_INF); + EXPECT_EQ(normInf, 0); + } +} + +static testing::internal::ParamGenerator intelModels() +{ + String path = utils::fs::join(utils::fs::join(extraTestDataPath, "deployment_tools"), "intel_models"); + + std::vector modelsNames; + cv::utils::fs::glob_relative(path, "", modelsNames, false, true); + + std::vector::iterator end = + std::remove_if(modelsNames.begin(), modelsNames.end(), + [&](const String& dir){ return !utils::fs::isDirectory(utils::fs::join(path, dir)); }); + modelsNames = std::vector(modelsNames.begin(), end); + + return testing::ValuesIn(modelsNames); +} + +INSTANTIATE_TEST_CASE_P(/**/, DNNTestOpenVINO, Combine( + Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16), intelModels() +)); + +}} +#endif // HAVE_INF_ENGINE From 6e767e2376dd5d06f0c4e0da9cb54471ea284fda Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 23 Jul 2018 17:58:10 +0300 Subject: [PATCH 2/3] ts: add findDataDirectory() function --- modules/ts/include/opencv2/ts.hpp | 5 +++++ modules/ts/src/ts.cpp | 32 ++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/modules/ts/include/opencv2/ts.hpp b/modules/ts/include/opencv2/ts.hpp index 7b3f732ce0..5d88396630 100644 --- a/modules/ts/include/opencv2/ts.hpp +++ b/modules/ts/include/opencv2/ts.hpp @@ -654,6 +654,11 @@ void addDataSearchSubDirectory(const std::string& subdir); */ std::string findDataFile(const std::string& relative_path, bool required = true); +/*! @brief Try to find requested data directory +@sa findDataFile + */ +std::string findDataDirectory(const std::string& relative_path, bool required = true); + #ifndef __CV_TEST_EXEC_ARGS #if defined(_MSC_VER) && (_MSC_VER <= 1400) diff --git a/modules/ts/src/ts.cpp b/modules/ts/src/ts.cpp index 06f9118a28..b1ea96bb15 100644 --- a/modules/ts/src/ts.cpp +++ b/modules/ts/src/ts.cpp @@ -772,16 +772,24 @@ void addDataSearchSubDirectory(const std::string& subdir) TS::ptr()->data_search_subdir.push_back(subdir); } -std::string findDataFile(const std::string& relative_path, bool required) +static std::string findData(const std::string& relative_path, bool required, bool findDirectory) { #define TEST_TRY_FILE_WITH_PREFIX(prefix) \ { \ std::string path = path_join(prefix, relative_path); \ /*printf("Trying %s\n", path.c_str());*/ \ - FILE* f = fopen(path.c_str(), "rb"); \ - if(f) { \ - fclose(f); \ - return path; \ + if (findDirectory) \ + { \ + if (isDirectory(path)) \ + return path; \ + } \ + else \ + { \ + FILE* f = fopen(path.c_str(), "rb"); \ + if(f) { \ + fclose(f); \ + return path; \ + } \ } \ } @@ -842,11 +850,21 @@ std::string findDataFile(const std::string& relative_path, bool required) } #endif #endif + const char* type = findDirectory ? "directory" : "data file"; if (required) - CV_Error(cv::Error::StsError, cv::format("OpenCV tests: Can't find required data file: %s", relative_path.c_str())); - throw SkipTestException(cv::format("OpenCV tests: Can't find data file: %s", relative_path.c_str())); + CV_Error(cv::Error::StsError, cv::format("OpenCV tests: Can't find required %s: %s", type, relative_path.c_str())); + throw SkipTestException(cv::format("OpenCV tests: Can't find %s: %s", type, relative_path.c_str())); +} + +std::string findDataFile(const std::string& relative_path, bool required) +{ + return findData(relative_path, required, false); } +std::string findDataDirectory(const std::string& relative_path, bool required) +{ + return findData(relative_path, required, true); +} } //namespace cvtest From 4283309daaa8dfca1701a4b6d3ff884f3eae1bca Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 23 Jul 2018 18:17:51 +0300 Subject: [PATCH 3/3] dnn: update tests for OpenVINO models --- modules/dnn/test/test_ie_models.cpp | 56 +++++++++++++++++++---------- modules/ts/include/opencv2/ts.hpp | 1 + 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index 80c8ef3bc1..9013ce9774 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -13,15 +13,22 @@ #include #include -static std::string extraTestDataPath = -#ifdef WINRT - NULL; -#else - getenv("INTEL_CVSDK_DIR"); -#endif - namespace opencv_test { namespace { +static void initDLDTDataPath() +{ +#ifndef WINRT + static bool initialized = false; + if (!initialized) + { + const char* dldtTestDataPath = getenv("INTEL_CVSDK_DIR"); + if (dldtTestDataPath) + cvtest::addDataSearchPath(cv::utils::fs::join(dldtTestDataPath, "deployment_tools")); + initialized = true; + } +#endif +} + using namespace cv; using namespace cv::dnn; using namespace InferenceEngine; @@ -174,13 +181,11 @@ TEST_P(DNNTestOpenVINO, models) throw SkipTestException(""); std::string precision = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? "FP16" : "FP32"; - std::string prefix = utils::fs::join(extraTestDataPath, - utils::fs::join("deployment_tools", - utils::fs::join("intel_models", + std::string prefix = utils::fs::join("intel_models", utils::fs::join(modelName, - utils::fs::join(precision, modelName))))); - std::string xmlPath = prefix + ".xml"; - std::string binPath = prefix + ".bin"; + utils::fs::join(precision, modelName))); + std::string xmlPath = findDataFile(prefix + ".xml"); + std::string binPath = findDataFile(prefix + ".bin"); std::map inputsMap; std::map ieOutputsMap, cvOutputsMap; @@ -199,17 +204,30 @@ TEST_P(DNNTestOpenVINO, models) static testing::internal::ParamGenerator intelModels() { - String path = utils::fs::join(utils::fs::join(extraTestDataPath, "deployment_tools"), "intel_models"); - + initDLDTDataPath(); std::vector modelsNames; + + std::string path; + try + { + path = findDataDirectory("intel_models", false); + } + catch (...) + { + std::cerr << "ERROR: Can't find OpenVINO models. Check INTEL_CVSDK_DIR environment variable (run setup.sh)" << std::endl; + return ValuesIn(modelsNames); // empty list + } + cv::utils::fs::glob_relative(path, "", modelsNames, false, true); - std::vector::iterator end = + modelsNames.erase( std::remove_if(modelsNames.begin(), modelsNames.end(), - [&](const String& dir){ return !utils::fs::isDirectory(utils::fs::join(path, dir)); }); - modelsNames = std::vector(modelsNames.begin(), end); + [&](const String& dir){ return !utils::fs::isDirectory(utils::fs::join(path, dir)); }), + modelsNames.end() + ); + CV_Assert(!modelsNames.empty()); - return testing::ValuesIn(modelsNames); + return ValuesIn(modelsNames); } INSTANTIATE_TEST_CASE_P(/**/, DNNTestOpenVINO, Combine( diff --git a/modules/ts/include/opencv2/ts.hpp b/modules/ts/include/opencv2/ts.hpp index 5d88396630..3fbea894e8 100644 --- a/modules/ts/include/opencv2/ts.hpp +++ b/modules/ts/include/opencv2/ts.hpp @@ -103,6 +103,7 @@ using std::pair; using std::make_pair; using testing::TestWithParam; using testing::Values; +using testing::ValuesIn; using testing::Combine; using cv::Mat;