Merge pull request #12037 from dkurt:test_openvino_models

modules/dnn/CMakeLists.txt

@@ -120,3 +120,9 @@ if(BUILD_PERF_TESTS)
     endif()
   endif()
 endif()
+
+# Test Intel's Inference Engine models
+if(HAVE_INF_ENGINE AND TARGET opencv_test_dnn)
+  ocv_target_include_directories(opencv_test_dnn PRIVATE ${INF_ENGINE_INCLUDE_DIRS})
+  ocv_target_link_libraries(opencv_test_dnn LINK_PRIVATE ${INF_ENGINE_LIBRARIES})
+endif()

modules/dnn/src/op_inf_engine.cpp

@@ -428,9 +428,8 @@ void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net)
 
     try
     {
-        static std::map<std::string, InferenceEngine::InferenceEnginePluginPtr> sharedPlugins;
+        static std::map<InferenceEngine::TargetDevice, InferenceEngine::InferenceEnginePluginPtr> sharedPlugins;
-        std::string deviceName = InferenceEngine::getDeviceName(targetDevice);
+        auto pluginIt = sharedPlugins.find(targetDevice);
-        auto pluginIt = sharedPlugins.find(deviceName);
         if (pluginIt != sharedPlugins.end())
         {
             enginePtr = pluginIt->second;
@@ -438,7 +437,7 @@ void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net)
         else
         {
             enginePtr = InferenceEngine::PluginDispatcher({""}).getSuitablePlugin(targetDevice);
-            sharedPlugins[deviceName] = enginePtr;
+            sharedPlugins[targetDevice] = enginePtr;
 
             if (targetDevice == InferenceEngine::TargetDevice::eCPU)
             {

modules/dnn/test/test_ie_models.cpp

@@ -0,0 +1,238 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2018, Intel Corporation, all rights reserved.
+// Third party copyrights are property of their respective owners.
+#include "test_precomp.hpp"
+
+#ifdef HAVE_INF_ENGINE
+#include <opencv2/core/utils/filesystem.hpp>
+
+#include <inference_engine.hpp>
+#include <ie_icnn_network.hpp>
+#include <ie_extension.h>
+
+namespace opencv_test { namespace {
+
+static void initDLDTDataPath()
+{
+#ifndef WINRT
+    static bool initialized = false;
+    if (!initialized)
+    {
+        const char* dldtTestDataPath = getenv("INTEL_CVSDK_DIR");
+        if (dldtTestDataPath)
+            cvtest::addDataSearchPath(cv::utils::fs::join(dldtTestDataPath, "deployment_tools"));
+        initialized = true;
+    }
+#endif
+}
+
+using namespace cv;
+using namespace cv::dnn;
+using namespace InferenceEngine;
+
+static inline void genData(const std::vector<size_t>& dims, Mat& m, Blob::Ptr& dataPtr)
+{
+    std::vector<int> reversedDims(dims.begin(), dims.end());
+    std::reverse(reversedDims.begin(), reversedDims.end());
+
+    m.create(reversedDims, CV_32F);
+    randu(m, -1, 1);
+
+    dataPtr = make_shared_blob<float>(Precision::FP32, dims, (float*)m.data);
+}
+
+void runIE(Target target, const std::string& xmlPath, const std::string& binPath,
+           std::map<std::string, cv::Mat>& inputsMap, std::map<std::string, cv::Mat>& outputsMap)
+{
+    CNNNetReader reader;
+    reader.ReadNetwork(xmlPath);
+    reader.ReadWeights(binPath);
+
+    CNNNetwork net = reader.getNetwork();
+
+    InferenceEnginePluginPtr enginePtr;
+    InferencePlugin plugin;
+    ExecutableNetwork netExec;
+    InferRequest infRequest;
+    TargetDevice targetDevice;
+    switch (target)
+    {
+        case DNN_TARGET_CPU:
+            targetDevice = TargetDevice::eCPU;
+            break;
+        case DNN_TARGET_OPENCL:
+        case DNN_TARGET_OPENCL_FP16:
+            targetDevice = TargetDevice::eGPU;
+            break;
+        case DNN_TARGET_MYRIAD:
+            targetDevice = TargetDevice::eMYRIAD;
+            break;
+        default:
+            CV_Error(Error::StsNotImplemented, "Unknown target");
+    };
+
+    try
+    {
+        enginePtr = PluginDispatcher({""}).getSuitablePlugin(targetDevice);
+
+        if (targetDevice == TargetDevice::eCPU)
+        {
+            std::string suffixes[] = {"_avx2", "_sse4", ""};
+            bool haveFeature[] = {
+                checkHardwareSupport(CPU_AVX2),
+                checkHardwareSupport(CPU_SSE4_2),
+                true
+            };
+            for (int i = 0; i < 3; ++i)
+            {
+                if (!haveFeature[i])
+                    continue;
+#ifdef _WIN32
+                std::string libName = "cpu_extension" + suffixes[i] + ".dll";
+#else
+                std::string libName = "libcpu_extension" + suffixes[i] + ".so";
+#endif  // _WIN32
+                try
+                {
+                    IExtensionPtr extension = make_so_pointer<IExtension>(libName);
+                    enginePtr->AddExtension(extension, 0);
+                    break;
+                }
+                catch(...) {}
+            }
+            // Some of networks can work without a library of extra layers.
+        }
+        plugin = InferencePlugin(enginePtr);
+
+        netExec = plugin.LoadNetwork(net, {});
+        infRequest = netExec.CreateInferRequest();
+    }
+    catch (const std::exception& ex)
+    {
+        CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
+    }
+
+    // Fill input blobs.
+    inputsMap.clear();
+    BlobMap inputBlobs;
+    for (auto& it : net.getInputsInfo())
+    {
+        genData(it.second->getDims(), inputsMap[it.first], inputBlobs[it.first]);
+    }
+    infRequest.SetInput(inputBlobs);
+
+    // Fill output blobs.
+    outputsMap.clear();
+    BlobMap outputBlobs;
+    for (auto& it : net.getOutputsInfo())
+    {
+        genData(it.second->dims, outputsMap[it.first], outputBlobs[it.first]);
+    }
+    infRequest.SetOutput(outputBlobs);
+
+    infRequest.Infer();
+}
+
+std::vector<String> getOutputsNames(const Net& net)
+{
+    std::vector<String> names;
+    if (names.empty())
+    {
+        std::vector<int> outLayers = net.getUnconnectedOutLayers();
+        std::vector<String> layersNames = net.getLayerNames();
+        names.resize(outLayers.size());
+        for (size_t i = 0; i < outLayers.size(); ++i)
+            names[i] = layersNames[outLayers[i] - 1];
+    }
+    return names;
+}
+
+void runCV(Target target, const std::string& xmlPath, const std::string& binPath,
+           const std::map<std::string, cv::Mat>& inputsMap,
+           std::map<std::string, cv::Mat>& outputsMap)
+{
+    Net net = readNet(xmlPath, binPath);
+    for (auto& it : inputsMap)
+        net.setInput(it.second, it.first);
+    net.setPreferableTarget(target);
+
+    std::vector<String> outNames = getOutputsNames(net);
+    std::vector<Mat> outs;
+    net.forward(outs, outNames);
+
+    outputsMap.clear();
+    EXPECT_EQ(outs.size(), outNames.size());
+    for (int i = 0; i < outs.size(); ++i)
+    {
+        EXPECT_TRUE(outputsMap.insert({outNames[i], outs[i]}).second);
+    }
+}
+
+typedef TestWithParam<tuple<Target, String> > DNNTestOpenVINO;
+TEST_P(DNNTestOpenVINO, models)
+{
+    Target target = (dnn::Target)(int)get<0>(GetParam());
+    std::string modelName = get<1>(GetParam());
+
+    if (modelName == "semantic-segmentation-adas-0001" && target == DNN_TARGET_OPENCL_FP16)
+        throw SkipTestException("");
+
+    std::string precision = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? "FP16" : "FP32";
+    std::string prefix = utils::fs::join("intel_models",
+                         utils::fs::join(modelName,
+                         utils::fs::join(precision, modelName)));
+    std::string xmlPath = findDataFile(prefix + ".xml");
+    std::string binPath = findDataFile(prefix + ".bin");
+
+    std::map<std::string, cv::Mat> inputsMap;
+    std::map<std::string, cv::Mat> ieOutputsMap, cvOutputsMap;
+    runIE(target, xmlPath, binPath, inputsMap, ieOutputsMap);
+    runCV(target, xmlPath, binPath, inputsMap, cvOutputsMap);
+
+    EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size());
+    for (auto& srcIt : ieOutputsMap)
+    {
+        auto dstIt = cvOutputsMap.find(srcIt.first);
+        CV_Assert(dstIt != cvOutputsMap.end());
+        double normInf = cvtest::norm(srcIt.second, dstIt->second, cv::NORM_INF);
+        EXPECT_EQ(normInf, 0);
+    }
+}
+
+static testing::internal::ParamGenerator<String> intelModels()
+{
+    initDLDTDataPath();
+    std::vector<String> modelsNames;
+
+    std::string path;
+    try
+    {
+        path = findDataDirectory("intel_models", false);
+    }
+    catch (...)
+    {
+        std::cerr << "ERROR: Can't find OpenVINO models. Check INTEL_CVSDK_DIR environment variable (run setup.sh)" << std::endl;
+        return ValuesIn(modelsNames);  // empty list
+    }
+
+    cv::utils::fs::glob_relative(path, "", modelsNames, false, true);
+
+    modelsNames.erase(
+        std::remove_if(modelsNames.begin(), modelsNames.end(),
+                       [&](const String& dir){ return !utils::fs::isDirectory(utils::fs::join(path, dir)); }),
+        modelsNames.end()
+    );
+    CV_Assert(!modelsNames.empty());
+
+    return ValuesIn(modelsNames);
+}
+
+INSTANTIATE_TEST_CASE_P(/**/, DNNTestOpenVINO, Combine(
+    Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16), intelModels()
+));
+
+}}
+#endif  // HAVE_INF_ENGINE

modules/ts/include/opencv2/ts.hpp

@@ -103,6 +103,7 @@ using std::pair;
 using std::make_pair;
 using testing::TestWithParam;
 using testing::Values;
+using testing::ValuesIn;
 using testing::Combine;
 
 using cv::Mat;
@@ -654,6 +655,11 @@ void addDataSearchSubDirectory(const std::string& subdir);
  */
 std::string findDataFile(const std::string& relative_path, bool required = true);
 
+/*! @brief Try to find requested data directory
+@sa findDataFile
+ */
+std::string findDataDirectory(const std::string& relative_path, bool required = true);
+
 
 #ifndef __CV_TEST_EXEC_ARGS
 #if defined(_MSC_VER) && (_MSC_VER <= 1400)

modules/ts/src/ts.cpp

@@ -772,17 +772,25 @@ void addDataSearchSubDirectory(const std::string& subdir)
     TS::ptr()->data_search_subdir.push_back(subdir);
 }
 
-std::string findDataFile(const std::string& relative_path, bool required)
+static std::string findData(const std::string& relative_path, bool required, bool findDirectory)
 {
 #define TEST_TRY_FILE_WITH_PREFIX(prefix) \
 { \
     std::string path = path_join(prefix, relative_path); \
     /*printf("Trying %s\n", path.c_str());*/ \
+    if (findDirectory) \
+    { \
+        if (isDirectory(path)) \
+            return path; \
+    } \
+    else \
+    { \
         FILE* f = fopen(path.c_str(), "rb"); \
         if(f) { \
             fclose(f); \
             return path; \
         } \
+    } \
 }
 
     const std::vector<std::string>& search_path = TS::ptr()->data_search_path;
@@ -842,11 +850,21 @@ std::string findDataFile(const std::string& relative_path, bool required)
     }
 #endif
 #endif
+    const char* type = findDirectory ? "directory" : "data file";
     if (required)
-        CV_Error(cv::Error::StsError, cv::format("OpenCV tests: Can't find required data file: %s", relative_path.c_str()));
+        CV_Error(cv::Error::StsError, cv::format("OpenCV tests: Can't find required %s: %s", type, relative_path.c_str()));
-    throw SkipTestException(cv::format("OpenCV tests: Can't find data file: %s", relative_path.c_str()));
+    throw SkipTestException(cv::format("OpenCV tests: Can't find %s: %s", type, relative_path.c_str()));
+}
+
+std::string findDataFile(const std::string& relative_path, bool required)
+{
+    return findData(relative_path, required, false);
 }
 
+std::string findDataDirectory(const std::string& relative_path, bool required)
+{
+    return findData(relative_path, required, true);
+}
 
 } //namespace cvtest