diff --git a/CMakeLists.txt b/CMakeLists.txt index b81a77b0b9..add618bfba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,7 @@ OCV_OPTION(WITH_GTK "Include GTK support" ON OCV_OPTION(WITH_GTK_2_X "Use GTK version 2" OFF IF (UNIX AND NOT APPLE AND NOT ANDROID) ) OCV_OPTION(WITH_IPP "Include Intel IPP support" (NOT MINGW AND NOT CV_DISABLE_OPTIMIZATION) IF (X86_64 OR X86) AND NOT WINRT AND NOT IOS ) OCV_OPTION(WITH_HALIDE "Include Halide support" OFF) +OCV_OPTION(WITH_INF_ENGINE "Include Intel Inference Engine support" OFF) OCV_OPTION(WITH_JASPER "Include JPEG2K support" ON IF (NOT IOS) ) OCV_OPTION(WITH_JPEG "Include JPEG support" ON) OCV_OPTION(WITH_WEBP "Include WebP support" ON IF (NOT WINRT) ) @@ -669,6 +670,11 @@ if(WITH_HALIDE) include(cmake/OpenCVDetectHalide.cmake) endif() +# --- Inference Engine --- +if(WITH_INF_ENGINE) + include(cmake/OpenCVDetectInferenceEngine.cmake) +endif() + # --- DirectX --- if(WITH_DIRECTX) include(cmake/OpenCVDetectDirectX.cmake) @@ -1353,6 +1359,10 @@ if(WITH_HALIDE OR HAVE_HALIDE) status(" Halide:" HAVE_HALIDE THEN "YES (${HALIDE_LIBRARIES} ${HALIDE_INCLUDE_DIRS})" ELSE NO) endif() +if(WITH_INF_ENGINE OR HAVE_INF_ENGINE) + status(" Inference Engine:" HAVE_INF_ENGINE THEN "YES (${INF_ENGINE_LIBRARIES} ${INF_ENGINE_INCLUDE_DIRS})" ELSE NO) +endif() + if(WITH_EIGEN OR HAVE_EIGEN) status(" Eigen:" HAVE_EIGEN THEN "YES (ver ${EIGEN_WORLD_VERSION}.${EIGEN_MAJOR_VERSION}.${EIGEN_MINOR_VERSION})" ELSE NO) endif() diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake new file mode 100644 index 0000000000..f1bccb47e5 --- /dev/null +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -0,0 +1,59 @@ +# The script detects Intel(R) Inference Engine installation +# +# Parameters: +# INTEL_CVSDK_DIR - Path to Inference Engine root folder +# IE_PLUGINS_PATH - Path to folder with Inference Engine plugins +# +# On return this will define: +# +# HAVE_INF_ENGINE - True if Intel Inference Engine was found +# INF_ENGINE_INCLUDE_DIRS - Inference Engine include folder +# INF_ENGINE_LIBRARIES - Inference Engine libraries and it's dependencies +# +macro(ie_fail) + set(HAVE_INF_ENGINE FALSE) + return() +endmacro() + +if(NOT INF_ENGINE_ROOT_DIR OR NOT EXISTS "${INF_ENGINE_ROOT_DIR}/inference_engine/include/inference_engine.hpp") + set(ie_root_paths "${INF_ENGINE_ROOT_DIR}") + if(DEFINED ENV{INTEL_CVSDK_DIR}) + list(APPEND ie_root_paths "$ENV{INTEL_CVSDK_DIR}") + endif() + + if(WITH_INF_ENGINE AND NOT ie_root_paths) + list(APPEND ie_root_paths "/opt/intel/deeplearning_deploymenttoolkit/deployment_tools") + endif() + + find_path(INF_ENGINE_ROOT_DIR inference_engine/include/inference_engine.hpp PATHS ${ie_root_paths}) +endif() + +set(INF_ENGINE_INCLUDE_DIRS "${INF_ENGINE_ROOT_DIR}/inference_engine/include" CACHE PATH "Path to Inference Engine include directory") + +if(NOT INF_ENGINE_ROOT_DIR + OR NOT EXISTS "${INF_ENGINE_ROOT_DIR}" + OR NOT EXISTS "${INF_ENGINE_INCLUDE_DIRS}" + OR NOT EXISTS "${INF_ENGINE_INCLUDE_DIRS}/inference_engine.hpp" +) + ie_fail() +endif() + +set(INF_ENGINE_LIBRARIES "") +foreach(lib inference_engine mklml_intel iomp5) + find_library(${lib} + NAMES ${lib} + HINTS ${IE_PLUGINS_PATH} + HINTS "$ENV{IE_PLUGINS_PATH}" + HINTS ${INF_ENGINE_ROOT_DIR}/external/mklml_lnx/lib + ) + if(NOT ${lib}) + ie_fail() + endif() + list(APPEND INF_ENGINE_LIBRARIES ${${lib}}) +endforeach() + +set(HAVE_INF_ENGINE TRUE) + +include_directories(${INF_ENGINE_INCLUDE_DIRS}) +list(APPEND OPENCV_LINKER_LIBS ${INF_ENGINE_LIBRARIES}) +add_definitions(-DHAVE_INF_ENGINE) diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index adb2013bb0..4fb12f3c47 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -59,7 +59,7 @@ ocv_create_module(${extra_libs}) ocv_target_link_libraries(${the_module} LINK_PRIVATE "${ZLIB_LIBRARIES}" "${OPENCL_LIBRARIES}" "${VA_LIBRARIES}" - "${LAPACK_LIBRARIES}" "${CPUFEATURES_LIBRARIES}" "${HALIDE_LIBRARIES}" + "${LAPACK_LIBRARIES}" "${CPUFEATURES_LIBRARIES}" "${HALIDE_LIBRARIES}" "${INF_ENGINE_LIBRARIES}" "${ITT_LIBRARIES}" "${OPENCV_HAL_LINKER_LIBS}" ) diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 944574e3e5..1baa3ab81f 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -27,6 +27,7 @@ else() -Wunused-parameter -Wunused-local-typedefs -Wsign-compare -Wsign-promo -Wundef -Wtautological-undefined-compare -Wignored-qualifiers -Wextra -Wunused-function -Wunused-const-variable -Wdeprecated-declarations + -Werror=non-virtual-dtor ) endif() diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 81d0f8da8d..a31730c57f 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -70,7 +70,8 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN enum Backend { DNN_BACKEND_DEFAULT, - DNN_BACKEND_HALIDE + DNN_BACKEND_HALIDE, + DNN_BACKEND_INFERENCE_ENGINE }; /** @@ -242,6 +243,8 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN */ virtual Ptr initHalide(const std::vector > &inputs); + virtual Ptr initInfEngine(const std::vector > &inputs); + /** * @brief Automatic Halide scheduling based on layer hyper-parameters. * @param[in] node Backend node with Halide functions. diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index 219c702a8a..e2df19c173 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -13,14 +13,7 @@ namespace { -#ifdef HAVE_HALIDE -#define TEST_DNN_BACKEND DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE -#else -#define TEST_DNN_BACKEND DNN_BACKEND_DEFAULT -#endif -#define TEST_DNN_TARGET DNN_TARGET_CPU, DNN_TARGET_OPENCL - -CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE) +CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE, DNN_BACKEND_INFERENCE_ENGINE) CV_ENUM(DNNTarget, DNN_TARGET_CPU, DNN_TARGET_OPENCL) class DNNTestNetwork : public ::perf::TestBaseWithParam< tuple > @@ -31,13 +24,16 @@ public: dnn::Net net; - void processNet(std::string weights, std::string proto, std::string halide_scheduler, - const Mat& input, const std::string& outputLayer, - const std::string& framework) + DNNTestNetwork() { backend = (dnn::Backend)(int)get<0>(GetParam()); target = (dnn::Target)(int)get<1>(GetParam()); + } + void processNet(std::string weights, std::string proto, std::string halide_scheduler, + const Mat& input, const std::string& outputLayer, + const std::string& framework) + { if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL) { #if defined(HAVE_OPENCL) @@ -47,6 +43,8 @@ public: throw ::SkipTestException("OpenCL is not available/disabled in OpenCV"); } } + if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL) + throw SkipTestException("Skip OpenCL target of Inference Engine backend"); randu(input, 0.0f, 1.0f); @@ -117,7 +115,7 @@ PERF_TEST_P_(DNNTestNetwork, GoogLeNet) "", Mat(cv::Size(224, 224), CV_32FC3), "prob", "caffe"); } -PERF_TEST_P_(DNNTestNetwork, ResNet50) +PERF_TEST_P_(DNNTestNetwork, ResNet_50) { processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", "resnet_50.yml", Mat(cv::Size(224, 224), CV_32FC3), "prob", "caffe"); @@ -131,6 +129,7 @@ PERF_TEST_P_(DNNTestNetwork, SqueezeNet_v1_1) PERF_TEST_P_(DNNTestNetwork, Inception_5h) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/tensorflow_inception_graph.pb", "", "inception_5h.yml", Mat(cv::Size(224, 224), CV_32FC3), "softmax2", "tensorflow"); @@ -138,12 +137,14 @@ PERF_TEST_P_(DNNTestNetwork, Inception_5h) PERF_TEST_P_(DNNTestNetwork, ENet) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/Enet-model-best.net", "", "enet.yml", Mat(cv::Size(512, 256), CV_32FC3), "l367_Deconvolution", "torch"); } PERF_TEST_P_(DNNTestNetwork, SSD) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", "disabled", Mat(cv::Size(300, 300), CV_32FC3), "detection_out", "caffe"); } @@ -162,15 +163,53 @@ PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_Caffe) PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_TensorFlow) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/ssd_mobilenet_v1_coco.pb", "ssd_mobilenet_v1_coco.pbtxt", "", Mat(cv::Size(300, 300), CV_32FC3), "", "tensorflow"); } -INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, - testing::Combine( - ::testing::Values(TEST_DNN_BACKEND), - DNNTarget::all() - ) -); +PERF_TEST_P_(DNNTestNetwork, DenseNet_121) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", "", + Mat(cv::Size(224, 224), CV_32FC3), "", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_coco) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", "", + Mat(cv::Size(368, 368), CV_32FC3), "", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", "", + Mat(cv::Size(368, 368), CV_32FC3), "", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + // The same .caffemodel but modified .prototxt + // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp + processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", "", + Mat(cv::Size(368, 368), CV_32FC3), "", "caffe"); +} + +const tuple testCases[] = { +#ifdef HAVE_HALIDE + tuple(DNN_BACKEND_HALIDE, DNN_TARGET_CPU), + tuple(DNN_BACKEND_HALIDE, DNN_TARGET_OPENCL), +#endif +#ifdef HAVE_INF_ENGINE + tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_CPU), +#endif + tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_CPU), + tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL) +}; + +INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, testing::ValuesIn(testCases)); } // namespace diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 26ff469e18..0da3e2cbdc 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -41,6 +41,7 @@ #include "precomp.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "halide_scheduler.hpp" #include #include @@ -471,9 +472,9 @@ public: } } - void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst) + void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, bool forceCreate) { - if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS) + if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS && !forceCreate) { Mat bestBlob; LayerPin bestBlobPin; @@ -518,7 +519,8 @@ public: } void allocateBlobsForLayer(LayerData &ld, const LayerShapes& layerShapes, - std::vector& pinsForInternalBlobs) + std::vector& pinsForInternalBlobs, + bool forceCreate = false) { CV_TRACE_FUNCTION(); @@ -589,7 +591,7 @@ public: reuse(ld.inputBlobsId[0], blobPin); } else - reuseOrCreate(shapes[index], blobPin, *blobs[index]); + reuseOrCreate(shapes[index], blobPin, *blobs[index], forceCreate); } } } @@ -638,6 +640,13 @@ static Ptr wrapMat(int backendId, int targetId, cv::Mat& m) #ifdef HAVE_HALIDE return Ptr(new HalideBackendWrapper(targetId, m)); #endif // HAVE_HALIDE + } + else if (backendId == DNN_BACKEND_INFERENCE_ENGINE) + { + CV_Assert(haveInfEngine()); +#ifdef HAVE_INF_ENGINE + return Ptr(new InfEngineBackendWrapper(targetId, m)); +#endif // HAVE_INF_ENGINE } else CV_Error(Error::StsNotImplemented, "Unknown backend identifier"); @@ -710,6 +719,10 @@ struct Net::Impl return Ptr(new HalideBackendWrapper(baseBuffer, shape)); #endif // HAVE_HALIDE } + else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE) + { + return wrapMat(preferableBackend, preferableTarget, host); + } else CV_Error(Error::StsNotImplemented, "Unknown backend identifier"); } @@ -999,12 +1012,20 @@ struct Net::Impl void initBackend() { CV_TRACE_FUNCTION(); - if (preferableBackend == DNN_BACKEND_DEFAULT) - { CV_Assert(preferableTarget == DNN_TARGET_CPU || preferableTarget == DNN_TARGET_OPENCL); - return; - } + else if (preferableBackend == DNN_BACKEND_HALIDE) + initHalideBackend(); + else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE) + initInfEngineBackend(); + else + CV_Error(Error::StsNotImplemented, "Unknown backend identifier"); + } + + void initHalideBackend() + { + CV_TRACE_FUNCTION(); + CV_Assert(preferableBackend == DNN_BACKEND_HALIDE, haveHalide()); // Iterator to current layer. MapIdToLayerData::iterator it = layers.begin(); @@ -1050,17 +1071,115 @@ struct Net::Impl } // No layers fusion. ldTop.skip = false; - if (preferableBackend == DNN_BACKEND_HALIDE) + ldTop.backendNodes[DNN_BACKEND_HALIDE] = + layerTop->initHalide(ldTop.inputBlobsWrappers); + baseIt = it; + } + } + + void initInfEngineBackend() + { + // Build Inference Engine networks from sets of layers that support this + // backend. If an internal layer isn't supported we'll use default + // implementation of it but build a new network after it. + CV_TRACE_FUNCTION(); + CV_Assert(preferableBackend == DNN_BACKEND_INFERENCE_ENGINE, haveInfEngine()); +#ifdef HAVE_INF_ENGINE + MapIdToLayerData::iterator it; + Ptr net; + for (it = layers.begin(); it != layers.end(); ++it) + { + LayerData &ld = it->second; + ld.skip = true; + Ptr layer = ld.layerInstance; + + if (!layer->supportBackend(preferableBackend)) { - ldTop.backendNodes[DNN_BACKEND_HALIDE] = - layerTop->initHalide(ldTop.inputBlobsWrappers); - baseIt = it; + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + auto dataPtr = infEngineDataNode(ld.outputBlobsWrappers[i]); + dataPtr->name = ld.name; + } + ld.skip = false; + net = Ptr(); + continue; + } + + // Check what all inputs are from the same network or from default backend. + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) + { + Ptr ieInpNode = inpNode.dynamicCast(); + CV_Assert(!ieInpNode.empty(), net.empty() || net == ieInpNode->net); + } + } + + bool fused = false; + Ptr node; + if (!net.empty()) + { + // Try to fuse. + bool inPlace = ld.inputBlobsId.size() == 1 && ld.outputBlobs.size() == 1 && + ld.inputBlobs[0]->data == ld.outputBlobs[0].data; + if (inPlace) + { + node = layer->tryAttach(layers[ld.inputBlobsId[0].lid].backendNodes[preferableBackend]); + fused = !node.empty(); + if (fused) + ld.inputBlobsWrappers = layers[ld.inputBlobsId[0].lid].inputBlobsWrappers; + } } else + net = Ptr(new InfEngineBackendNet()); + + if (!fused) { - CV_Error(Error::StsNotImplemented, "Unknown backend identifier"); + node = layer->initInfEngine(ld.inputBlobsWrappers); } + + CV_Assert(!node.empty()); + ld.backendNodes[preferableBackend] = node; + + Ptr ieNode = node.dynamicCast(); + CV_Assert(!ieNode.empty()); + ieNode->net = net; + + ieNode->connect(ld.inputBlobsWrappers, ld.outputBlobsWrappers); + net->addBlobs(ld.inputBlobsWrappers); + net->addBlobs(ld.outputBlobsWrappers); + + if (!fused) + net->addLayer(ieNode->layer); } + + // Initialize all networks. + std::set initializedNets; + for (MapIdToLayerData::reverse_iterator it = layers.rbegin(); it != layers.rend(); ++it) + { + LayerData &ld = it->second; + if (ld.backendNodes.find(preferableBackend) == ld.backendNodes.end()) + continue; + + Ptr node = ld.backendNodes[preferableBackend]; + if (node.empty()) + continue; + + Ptr ieNode = node.dynamicCast(); + if (ieNode.empty()) + continue; + + CV_Assert(!ieNode->net.empty()); + + if (!ieNode->net->isInitialized()) + { + ieNode->net->initEngine(); + ld.skip = false; + } + } +#endif // HAVE_INF_ENGINE } void allocateLayer(int lid, const LayersShapesMap& layersShapes) @@ -1117,7 +1236,8 @@ struct Net::Impl CV_Assert(layerShapesIt != layersShapes.end()); std::vector pinsForInternalBlobs; - blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs); + blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs, + preferableBackend == DNN_BACKEND_INFERENCE_ENGINE); ld.outputBlobsWrappers.resize(ld.outputBlobs.size()); for (int i = 0; i < ld.outputBlobs.size(); ++i) { @@ -1564,6 +1684,10 @@ struct Net::Impl { forwardHalide(ld.outputBlobsWrappers, node); } + else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE) + { + forwardInfEngine(node); + } else { CV_Error(Error::StsNotImplemented, "Unknown backend identifier"); @@ -2329,6 +2453,13 @@ Ptr Layer::initHalide(const std::vector > &) return Ptr(); } +Ptr Layer::initInfEngine(const std::vector > &) +{ + CV_Error(Error::StsNotImplemented, "Inference Engine pipeline of " + type + + " layers is not defined."); + return Ptr(); +} + void Layer::applyHalideScheduler(Ptr& node, const std::vector &inputs, const std::vector &outputs, int targetId) const { diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index e3827d1360..d5f0df6010 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -11,6 +11,7 @@ Implementation of Batch Normalization layer. #include "../precomp.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include #include "opencl_kernels_dnn.hpp" @@ -103,7 +104,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide(); + backendId == DNN_BACKEND_HALIDE && haveHalide() || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); } #ifdef HAVE_OPENCL @@ -226,6 +228,19 @@ public: #endif // HAVE_HALIDE break; } + case DNN_BACKEND_INFERENCE_ENGINE: + { +#ifdef HAVE_INF_ENGINE + auto base = node.dynamicCast(); + auto conv = std::dynamic_pointer_cast(base->layer); + if (conv) + { + fuseConvWeights(conv, weights_, bias_); + return base; + } +#endif // HAVE_INF_ENGINE + break; + } } return Ptr(); } @@ -257,6 +272,25 @@ public: } #endif // HAVE_HALIDE + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "BatchNormalization"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::BatchNormalizationLayer(lp)); + + size_t numChannels = weights_.total(); + ieLayer->epsilon = epsilon; + ieLayer->_weights = wrapToInfEngineBlob(blobs[1], {numChannels}); + ieLayer->_biases = wrapToInfEngineBlob(blobs[0], {numChannels}); + + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index e49f22db2c..8466285e5d 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencl_kernels_dnn.hpp" namespace cv @@ -100,7 +101,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding; // By channels + backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding || // By channels + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && !padding; } class ChannelConcatInvoker : public ParallelLoopBody @@ -295,6 +297,20 @@ public: #endif // HAVE_HALIDE return Ptr(); } + + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Concat"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::ConcatLayer(lp)); + ieLayer->_axis = axis; + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } }; Ptr ConcatLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 4635c8f259..64c2212f59 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencv2/core/hal/hal.hpp" #include "opencv2/core/hal/intrin.hpp" #include @@ -65,7 +66,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide(); + backendId == DNN_BACKEND_HALIDE && haveHalide() || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); } void finalize(const std::vector &inputs, std::vector &outputs) @@ -335,6 +337,42 @@ public: return Ptr(); } + virtual Ptr initInfEngine(const std::vector > &inputs) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::DataPtr input = infEngineDataNode(inputs[0]); + CV_Assert(input->dims.size() == 4); + + const int inpCn = input->dims[2]; // NOTE: input->dims are reversed (whcn) + const int outCn = blobs[0].size[0]; + const int inpGroupCn = blobs[0].size[1]; + const int group = inpCn / inpGroupCn; + + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Convolution"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::ConvolutionLayer(lp)); + + ieLayer->_kernel_x = kernel.width; + ieLayer->_kernel_y = kernel.height; + ieLayer->_stride_x = stride.width; + ieLayer->_stride_y = stride.height; + ieLayer->_out_depth = outCn; + ieLayer->_padding_x = pad.width; + ieLayer->_padding_y = pad.height; + ieLayer->_dilation_x = dilation.width; + ieLayer->_dilation_y = dilation.height; + ieLayer->_group = group; + + ieLayer->_weights = wrapToInfEngineBlob(blobs[0]); + if (hasBias()) + ieLayer->_biases = wrapToInfEngineBlob(blobs[1]); + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + class ParallelConv : public cv::ParallelLoopBody { public: diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index e1ca59b96e..5b04bd6a4a 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -42,6 +42,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" +#include "op_inf_engine.hpp" #include #include #include "../nms.inl.hpp" @@ -189,6 +190,12 @@ public: setParamsFrom(params); } + virtual bool supportBackend(int backendId) + { + return backendId == DNN_BACKEND_DEFAULT || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -203,10 +210,10 @@ public: // num() and channels() are 1. // Since the number of bboxes to be kept is unknown before nms, we manually - // set it to (fake) 1. + // set it to maximal number of detections, [keep_top_k] parameter. // Each row is a 7 dimension std::vector, which stores // [image_id, label, confidence, xmin, ymin, xmax, ymax] - outputs.resize(1, shape(1, 1, 1, 7)); + outputs.resize(1, shape(1, 1, _keepTopK, 7)); return false; } @@ -850,6 +857,29 @@ public: return 0.; } } + + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "DetectionOutput"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::CNNLayer(lp)); + + ieLayer->params["num_classes"] = format("%d", _numClasses); + ieLayer->params["share_location"] = _shareLocation ? "1" : "0"; + ieLayer->params["background_label_id"] = format("%d", _backgroundLabelId); + ieLayer->params["nms_threshold"] = format("%f", _nmsThreshold); + ieLayer->params["top_k"] = format("%d", _topK); + ieLayer->params["keep_top_k"] = format("%d", _keepTopK); + ieLayer->params["confidence_threshold"] = format("%f", _confidenceThreshold); + ieLayer->params["variance_encoded_in_target"] = _varianceEncodedInTarget ? "1" : "0"; + ieLayer->params["code_type"] = "caffe.PriorBoxParameter." + _codeType; + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } }; float util::caffe_box_overlap(const util::NormalizedBBox& a, const util::NormalizedBBox& b) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index cc66fb00fe..86ff5a5055 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencv2/imgproc.hpp" #include #include "opencl_kernels_dnn.hpp" @@ -112,7 +113,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide(); + backendId == DNN_BACKEND_HALIDE && haveHalide() || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); } virtual Ptr tryAttach(const Ptr& node) @@ -147,6 +149,17 @@ public: return Ptr(); } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = this->name; + lp.precision = InferenceEngine::Precision::FP32; + return Ptr(new InfEngineBackendNode(func.initInfEngine(lp))); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -308,6 +321,15 @@ struct ReLUFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + lp.type = "ReLU"; + std::shared_ptr ieLayer(new InferenceEngine::ReLULayer(lp)); + return ieLayer; + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 1; } }; @@ -372,6 +394,14 @@ struct ReLU6Functor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "ReLU6"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 2; } }; @@ -427,6 +457,14 @@ struct TanHFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "TanH"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 1; } }; @@ -462,6 +500,14 @@ struct SigmoidFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "Sigmoid"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 3; } }; @@ -499,6 +545,14 @@ struct ELUFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "ELU"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 2; } }; @@ -534,6 +588,14 @@ struct AbsValFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "Abs"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 1; } }; @@ -569,6 +631,14 @@ struct BNLLFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "BNLL"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 5; } }; @@ -658,6 +728,18 @@ struct PowerFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + lp.type = "Power"; + std::shared_ptr ieLayer(new InferenceEngine::PowerLayer(lp)); + ieLayer->power = power; + ieLayer->scale = scale; + ieLayer->offset = shift; + return ieLayer; + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; } }; @@ -750,6 +832,14 @@ struct ChannelsPReLUFunctor } #endif // HAVE_HALIDE +#ifdef HAVE_INF_ENGINE + InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp) + { + CV_Error(Error::StsNotImplemented, "PReLU"); + return InferenceEngine::CNNLayerPtr(); + } +#endif // HAVE_INF_ENGINE + int64 getFLOPSPerElement() const { return 1; } }; diff --git a/modules/dnn/src/layers/eltwise_layer.cpp b/modules/dnn/src/layers/eltwise_layer.cpp index 27bd7eedd2..0e09202909 100644 --- a/modules/dnn/src/layers/eltwise_layer.cpp +++ b/modules/dnn/src/layers/eltwise_layer.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencl_kernels_dnn.hpp" namespace cv @@ -93,7 +94,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide(); + backendId == DNN_BACKEND_HALIDE && haveHalide() || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); } bool getMemoryShapes(const std::vector &inputs, @@ -402,6 +404,27 @@ public: return Ptr(); } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Eltwise"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::EltwiseLayer(lp)); + if (op == SUM) + ieLayer->_operation = InferenceEngine::EltwiseLayer::Sum; + else if (op == PROD) + ieLayer->_operation = InferenceEngine::EltwiseLayer::Prod; + else if (op == MAX) + ieLayer->_operation = InferenceEngine::EltwiseLayer::Max; + else + CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index 039720c038..9d12d05c19 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -42,6 +42,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" +#include "op_inf_engine.hpp" #include #include #include @@ -61,6 +62,12 @@ public: setParamsFrom(params); } + virtual bool supportBackend(int backendId) + { + return backendId == DNN_BACKEND_DEFAULT || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -153,6 +160,21 @@ public: } } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Flatten"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::CNNLayer(lp)); + ieLayer->params["axis"] = format("%d", _startAxis); + ieLayer->params["end_axis"] = format("%d", _endAxis); + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + int _startAxis; int _endAxis; }; diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index a64a895a7f..ccc8dec96a 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencl_kernels_dnn.hpp" #include @@ -127,7 +128,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1; + backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && axis == 1; } virtual bool setActivation(const Ptr& layer) @@ -395,6 +397,24 @@ public: return Ptr(); } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "FullyConnected"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::FullyConnectedLayer(lp)); + + ieLayer->_out_num = blobs[0].size[0]; + ieLayer->_weights = wrapToInfEngineBlob(blobs[0]); + if (blobs.size() > 1) + ieLayer->_biases = wrapToInfEngineBlob(blobs[1]); + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/layers/lrn_layer.cpp b/modules/dnn/src/layers/lrn_layer.cpp index 0f869015bf..a01d5f0a8d 100644 --- a/modules/dnn/src/layers/lrn_layer.cpp +++ b/modules/dnn/src/layers/lrn_layer.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencv2/imgproc.hpp" #include "opencv2/dnn/shape_utils.hpp" #include "opencv2/core/hal/hal.hpp" @@ -90,7 +91,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide(); + backendId == DNN_BACKEND_HALIDE && haveHalide() || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); } #ifdef HAVE_OPENCL @@ -369,6 +371,25 @@ public: #endif // HAVE_HALIDE } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Norm"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::NormLayer(lp)); + + ieLayer->_size = size; + ieLayer->_k = (int)bias; + ieLayer->_beta = beta; + ieLayer->_alpha = alpha; + ieLayer->_isAcrossMaps = (type == CHANNEL_NRM); + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index 664c24e475..bdc47fb737 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -42,6 +42,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" +#include "op_inf_engine.hpp" #include #include #include "opencl_kernels_dnn.hpp" @@ -112,6 +113,12 @@ public: checkNeedForPermutation(); } + virtual bool supportBackend(int backendId) + { + return backendId == DNN_BACKEND_DEFAULT || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -370,6 +377,25 @@ public: } } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Permute"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::CNNLayer(lp)); + + CV_Assert(!_order.empty()); + ieLayer->params["order"] = format("%d", _order[0]); + for (int i = 1; i < _order.size(); ++i) + ieLayer->params["order"] += format(",%d", _order[i]); + + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + size_t _count; std::vector _order; diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index b451b4b931..aee2d63e3c 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -44,6 +44,7 @@ #include "layers_common.hpp" #include "opencv2/core/hal/intrin.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencl_kernels_dnn.hpp" #include #include @@ -130,7 +131,8 @@ public: { return backendId == DNN_BACKEND_DEFAULT || backendId == DNN_BACKEND_HALIDE && haveHalide() && - (type == MAX || type == AVE && !pad.width && !pad.height); + (type == MAX || type == AVE && !pad.width && !pad.height) || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && (type == MAX || type == AVE); } #ifdef HAVE_OPENCL @@ -222,6 +224,35 @@ public: return Ptr(); } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Pooling"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::PoolingLayer(lp)); + + ieLayer->_kernel_x = kernel.width; + ieLayer->_kernel_y = kernel.height; + ieLayer->_stride_x = stride.width; + ieLayer->_stride_y = stride.height; + ieLayer->_padding_x = pad.width; + ieLayer->_padding_y = pad.height; + ieLayer->_exclude_pad = false; + ieLayer->params["rounding-type"] = ceilMode ? "ceil" : "floor"; + if (type == MAX) + ieLayer->_type = InferenceEngine::PoolingLayer::PoolType::MAX; + else if (type == AVE) + ieLayer->_type = InferenceEngine::PoolingLayer::PoolType::AVG; + else + CV_Error(Error::StsNotImplemented, "Unsupported pooling type"); + + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + class PoolingInvoker : public ParallelLoopBody { public: diff --git a/modules/dnn/src/layers/prior_box_layer.cpp b/modules/dnn/src/layers/prior_box_layer.cpp index 872a934660..cacc27921c 100644 --- a/modules/dnn/src/layers/prior_box_layer.cpp +++ b/modules/dnn/src/layers/prior_box_layer.cpp @@ -42,6 +42,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" +#include "op_inf_engine.hpp" #include #include #include @@ -248,6 +249,12 @@ public: } } + virtual bool supportBackend(int backendId) + { + return backendId == DNN_BACKEND_DEFAULT || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -518,6 +525,43 @@ public: } } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "PriorBox"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::CNNLayer(lp)); + + ieLayer->params["min_size"] = format("%f", _minSize); + ieLayer->params["max_size"] = _maxSize > 0 ? format("%f", _maxSize) : ""; + + CV_Assert(!_aspectRatios.empty()); + ieLayer->params["aspect_ratio"] = format("%f", _aspectRatios[0]); + for (int i = 1; i < _aspectRatios.size(); ++i) + ieLayer->params["aspect_ratio"] += format(",%f", _aspectRatios[i]); + + ieLayer->params["flip"] = _flip ? "1" : "0"; + ieLayer->params["clip"] = _clip ? "1" : "0"; + + CV_Assert(!_variance.empty()); + ieLayer->params["variance"] = format("%f", _variance[0]); + for (int i = 1; i < _variance.size(); ++i) + ieLayer->params["variance"] += format(",%f", _variance[i]); + + ieLayer->params["step"] = "0"; + ieLayer->params["step_h"] = _stepY; + ieLayer->params["step_w"] = _stepX; + + CV_Assert(_offsetsX.size() == 1, _offsetsY.size() == 1, _offsetsX[0] == _offsetsY[0]); + ieLayer->params["offset"] = format("%f", _offsetsX[0]);; + + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index 3183a3f898..7530ffc59c 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -42,6 +42,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" +#include "op_inf_engine.hpp" #include namespace cv @@ -165,6 +166,12 @@ public: } } + virtual bool supportBackend(int backendId) + { + return backendId == DNN_BACKEND_DEFAULT || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); + } + bool getMemoryShapes(const std::vector &inputs, const int requiredOutputs, std::vector &outputs, @@ -231,6 +238,20 @@ public: srcBlob.reshape(1, shape(outputs[i])).copyTo(outputs[i]); } } + + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "Reshape"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::ReshapeLayer(lp)); + ieLayer->shape = newShapeDesc; + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } }; Ptr ReshapeLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index 7912f47619..66faa15822 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -12,6 +12,7 @@ Implementation of Scale layer. #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include namespace cv @@ -42,7 +43,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide(); + backendId == DNN_BACKEND_HALIDE && haveHalide() || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine(); } void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) @@ -130,6 +132,20 @@ public: #endif // HAVE_HALIDE break; } + case DNN_BACKEND_INFERENCE_ENGINE: + { +#ifdef HAVE_INF_ENGINE + auto base = node.dynamicCast(); + auto conv = std::dynamic_pointer_cast(base->layer); + if (conv) + { + Mat bias = hasBias ? blobs[1] : Mat(); + fuseConvWeights(conv, blobs[0], bias); + return base; + } +#endif // HAVE_INF_ENGINE + break; + } } return Ptr(); } @@ -167,6 +183,24 @@ public: } #endif // HAVE_HALIDE + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "ScaleShift"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::ScaleShiftLayer(lp)); + + ieLayer->_weights = wrapToInfEngineBlob(blobs[0]); + if (hasBias) + ieLayer->_biases = wrapToInfEngineBlob(blobs[1]); + + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index 1ab9ed2b79..49807cd8ac 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -43,6 +43,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" #include "op_halide.hpp" +#include "op_inf_engine.hpp" #include "opencl_kernels_dnn.hpp" #include #include @@ -63,7 +64,7 @@ public: SoftMaxLayerImpl(const LayerParams& params) { axisRaw = params.get("axis", 1); - logSoftMax = params.get("log_softmax", false); + logSoftMax = params.get("log_softmax", false); setParamsFrom(params); } @@ -87,7 +88,8 @@ public: virtual bool supportBackend(int backendId) { return backendId == DNN_BACKEND_DEFAULT || - backendId == DNN_BACKEND_HALIDE && haveHalide() && axisRaw == 1; + backendId == DNN_BACKEND_HALIDE && haveHalide() && axisRaw == 1 || + backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && !logSoftMax; } #ifdef HAVE_OPENCL @@ -307,6 +309,20 @@ public: return Ptr(); } + virtual Ptr initInfEngine(const std::vector >&) + { +#ifdef HAVE_INF_ENGINE + InferenceEngine::LayerParams lp; + lp.name = name; + lp.type = "SoftMax"; + lp.precision = InferenceEngine::Precision::FP32; + std::shared_ptr ieLayer(new InferenceEngine::SoftMaxLayer(lp)); + ieLayer->axis = axisRaw; + return Ptr(new InfEngineBackendNode(ieLayer)); +#endif // HAVE_INF_ENGINE + return Ptr(); + } + int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const { diff --git a/modules/dnn/src/op_inf_engine.cpp b/modules/dnn/src/op_inf_engine.cpp new file mode 100644 index 0000000000..7bda0b81f9 --- /dev/null +++ b/modules/dnn/src/op_inf_engine.cpp @@ -0,0 +1,360 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2018, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "precomp.hpp" +#include "op_inf_engine.hpp" +#include + +namespace cv { namespace dnn { + +#ifdef HAVE_INF_ENGINE + +InfEngineBackendNode::InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& _layer) + : BackendNode(DNN_BACKEND_INFERENCE_ENGINE), layer(_layer) {} + +void InfEngineBackendNode::connect(std::vector >& inputs, + std::vector >& outputs) +{ + layer->insData.resize(inputs.size()); + for (int i = 0; i < inputs.size(); ++i) + { + InferenceEngine::DataPtr dataPtr = infEngineDataNode(inputs[i]); + layer->insData[i] = InferenceEngine::DataWeakPtr(dataPtr); + dataPtr->inputTo[layer->name] = layer; + } + + CV_Assert(!outputs.empty()); + + layer->outData.resize(1); + InferenceEngine::DataPtr dataPtr = infEngineDataNode(outputs[0]); + dataPtr->name = layer->name; + layer->outData[0] = dataPtr; + dataPtr->creatorLayer = InferenceEngine::CNNLayerWeakPtr(layer); +} + +static std::vector > +infEngineWrappers(const std::vector >& ptrs) +{ + std::vector > wrappers(ptrs.size()); + for (int i = 0; i < ptrs.size(); ++i) + { + CV_Assert(!ptrs[i].empty()); + wrappers[i] = ptrs[i].dynamicCast(); + CV_Assert(!wrappers[i].empty()); + } + return wrappers; +} + +static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "") +{ + std::vector reversedShape(&m.size[0], &m.size[0] + m.dims); + std::reverse(reversedShape.begin(), reversedShape.end()); + return InferenceEngine::DataPtr( + new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32) + ); +} + +InferenceEngine::TBlob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector& shape) +{ + return InferenceEngine::make_shared_blob(InferenceEngine::Precision::FP32, + shape, (float*)m.data); +} + +InferenceEngine::TBlob::Ptr wrapToInfEngineBlob(const Mat& m) +{ + std::vector reversedShape(&m.size[0], &m.size[0] + m.dims); + std::reverse(reversedShape.begin(), reversedShape.end()); + return wrapToInfEngineBlob(m, reversedShape); +} + +InferenceEngine::DataPtr infEngineDataNode(const Ptr& ptr) +{ + CV_Assert(!ptr.empty()); + Ptr p = ptr.dynamicCast(); + CV_Assert(!p.empty()); + return p->dataPtr; +} + +InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m) + : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, targetId) +{ + dataPtr = wrapToInfEngineDataNode(m); + blob = wrapToInfEngineBlob(m); +} + +InfEngineBackendWrapper::~InfEngineBackendWrapper() +{ + +} + +void InfEngineBackendWrapper::copyToHost() +{ + +} + +void InfEngineBackendWrapper::setHostDirty() +{ + +} + +void InfEngineBackendNet::Release() noexcept +{ + layers.clear(); + inputs.clear(); + outputs.clear(); +} + +InferenceEngine::Precision InfEngineBackendNet::getPrecision() noexcept +{ + return InferenceEngine::Precision::FP32; +} + +// Assume that outputs of network is unconnected blobs. +void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) noexcept +{ + if (outputs.empty()) + { + for (const auto& l : layers) + { + // Add all outputs. + for (const InferenceEngine::DataPtr& out : l->outData) + { + // TODO: Replace to uniquness assertion. + if (outputs.find(out->name) == outputs.end()) + outputs[out->name] = out; + } + // Remove internally connected outputs. + for (const InferenceEngine::DataWeakPtr& inp : l->insData) + { + outputs.erase(InferenceEngine::DataPtr(inp)->name); + } + } + CV_Assert(layers.empty() || !outputs.empty()); + } + outBlobs.clear(); + for (const auto& it : outputs) + { + CV_Assert(allBlobs.find(it.first) != allBlobs.end()); + outBlobs[it.first] = allBlobs[it.first]; + } + outputs_ = outputs; +} + +// Returns input references that aren't connected to internal outputs. +void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) noexcept +{ + if (inputs.empty()) + { + std::map internalOutputs; + for (const auto& l : layers) + { + for (const InferenceEngine::DataWeakPtr& ptr : l->insData) + { + InferenceEngine::DataPtr inp(ptr); + if (internalOutputs.find(inp->name) == internalOutputs.end()) + { + InferenceEngine::InputInfo::Ptr inpInfo(new InferenceEngine::InputInfo()); + inpInfo->setInputData(inp); + if (inputs.find(inp->name) == inputs.end()) + inputs[inp->name] = inpInfo; + } + } + for (const InferenceEngine::DataPtr& out : l->outData) + { + // TODO: Replace to uniquness assertion. + if (internalOutputs.find(out->name) == internalOutputs.end()) + internalOutputs[out->name] = out; + } + } + CV_Assert(layers.empty() || !inputs.empty()); + } + inpBlobs.clear(); + for (const auto& it : inputs) + { + CV_Assert(allBlobs.find(it.first) != allBlobs.end()); + inpBlobs[it.first] = allBlobs[it.first]; + } + inputs_ = inputs; +} + +InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) noexcept +{ + getInputsInfo(inputs); + const auto& it = inputs.find(inputName); + CV_Assert(it != inputs.end()); + return it->second; +} + +void InfEngineBackendNet::getName(char *pName, size_t len) noexcept +{ + CV_Error(Error::StsNotImplemented, ""); +} + +size_t InfEngineBackendNet::layerCount() noexcept +{ + return layers.size(); +} + +InferenceEngine::DataPtr& InfEngineBackendNet::getData(const char *dname) noexcept +{ + CV_Error(Error::StsNotImplemented, ""); + return outputs.begin()->second; // Just return something. +} + +void InfEngineBackendNet::addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept +{ + layers.push_back(layer); + inputs.clear(); + outputs.clear(); +} + +InferenceEngine::StatusCode +InfEngineBackendNet::addOutput(const std::string &layerName, size_t outputIndex, + InferenceEngine::ResponseDesc *resp) noexcept +{ + CV_Error(Error::StsNotImplemented, ""); + return InferenceEngine::StatusCode::OK; +} + +InferenceEngine::StatusCode +InfEngineBackendNet::getLayerByName(const char *layerName, InferenceEngine::CNNLayerPtr &out, + InferenceEngine::ResponseDesc *resp) noexcept +{ + CV_Error(Error::StsNotImplemented, ""); + return InferenceEngine::StatusCode::OK; +} + +void InfEngineBackendNet::setTargetDevice(InferenceEngine::TargetDevice device) noexcept +{ + if (device != InferenceEngine::TargetDevice::eCPU) + CV_Error(Error::StsNotImplemented, ""); +} + +InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() noexcept +{ + return InferenceEngine::TargetDevice::eCPU; +} + +InferenceEngine::StatusCode InfEngineBackendNet::setBatchSize(const size_t size) noexcept +{ + CV_Error(Error::StsNotImplemented, ""); + return InferenceEngine::StatusCode::OK; +} + +size_t InfEngineBackendNet::getBatchSize() const noexcept +{ + CV_Error(Error::StsNotImplemented, ""); + return 0; +} + +void InfEngineBackendNet::initEngine() +{ + CV_Assert(!isInitialized()); + engine = InferenceEngine::InferenceEnginePluginPtr("libMKLDNNPlugin.so"); + InferenceEngine::ResponseDesc resp; + InferenceEngine::StatusCode status = engine->LoadNetwork(*this, &resp); + if (status != InferenceEngine::StatusCode::OK) + CV_Error(Error::StsAssert, resp.msg); +} + +bool InfEngineBackendNet::isInitialized() +{ + return (bool)engine; +} + +void InfEngineBackendNet::addBlobs(const std::vector >& ptrs) +{ + auto wrappers = infEngineWrappers(ptrs); + for (const auto& wrapper : wrappers) + { + allBlobs[wrapper->dataPtr->name] = wrapper->blob; + } +} + +void InfEngineBackendNet::forward() +{ + InferenceEngine::ResponseDesc resp; + InferenceEngine::StatusCode status = engine->Infer(inpBlobs, outBlobs, &resp); + if (status != InferenceEngine::StatusCode::OK) + CV_Error(Error::StsAssert, resp.msg); +} + +static inline Mat infEngineBlobToMat(const InferenceEngine::Blob::Ptr& blob) +{ + // NOTE: Inference Engine sizes are reversed. + std::vector size(blob->dims().begin(), blob->dims().end()); + std::reverse(size.begin(), size.end()); + return Mat(size, CV_32F, (void*)blob->buffer()); +} + +void fuseConvWeights(const std::shared_ptr& conv, + const Mat& w, const Mat& b) +{ + // Get convolution's weights. Clone the data because Inference Engine can host it + // and conv->_weights->allocate() below will deallocate it. + Mat originWeights = infEngineBlobToMat(conv->_weights).clone(); + + // Create new weights blob. + conv->_weights = InferenceEngine::make_shared_blob( + InferenceEngine::Precision::FP32, conv->_weights->dims()); + conv->_weights->allocate(); + + // Convolution weights have OIHW data layout. + // (conv(I) + b1 ) * w + b2 + // w*conv(I) + b1 * w + b2 + Mat fusedWeights = infEngineBlobToMat(conv->_weights); + + const int numChannels = fusedWeights.size[0]; + // Mat weights = blobs[0].reshape(1, 1); + // Mat bias = hasBias ? blobs[1].reshape(1, 1) : Mat(); + CV_Assert(numChannels == w.total()); + CV_Assert(b.empty() || numChannels == b.total()); + for (int i = 0; i < numChannels; ++i) + { + cv::multiply(slice(originWeights, i), w.at(i), slice(fusedWeights, i)); + } + if (conv->_biases) + { + // The same for biases. + Mat originBiases = infEngineBlobToMat(conv->_biases).clone(); + + conv->_biases = InferenceEngine::make_shared_blob( + InferenceEngine::Precision::FP32, conv->_biases->dims()); + conv->_biases->allocate(); + Mat fusedBiases = infEngineBlobToMat(conv->_biases); + + cv::multiply(w.reshape(1, fusedBiases.dims, &fusedBiases.size[0]), originBiases, fusedBiases); + if (!b.empty()) + cv::add(fusedBiases, b.reshape(1, fusedBiases.dims, &fusedBiases.size[0]), fusedBiases); + } + else + conv->_biases = wrapToInfEngineBlob(b); +} + +#endif // HAVE_INF_ENGINE + +bool haveInfEngine() +{ +#ifdef HAVE_INF_ENGINE + return true; +#else + return false; +#endif // HAVE_INF_ENGINE +} + +void forwardInfEngine(Ptr& node) +{ + CV_Assert(haveInfEngine()); +#ifdef HAVE_INF_ENGINE + CV_Assert(!node.empty()); + Ptr ieNode = node.dynamicCast(); + CV_Assert(!ieNode.empty()); + ieNode->net->forward(); +#endif // HAVE_INF_ENGINE +} + +}} // namespace dnn, namespace cv diff --git a/modules/dnn/src/op_inf_engine.hpp b/modules/dnn/src/op_inf_engine.hpp new file mode 100644 index 0000000000..7a622e0acf --- /dev/null +++ b/modules/dnn/src/op_inf_engine.hpp @@ -0,0 +1,122 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2018, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#ifndef __OPENCV_DNN_OP_INF_ENGINE_HPP__ +#define __OPENCV_DNN_OP_INF_ENGINE_HPP__ + +#include "precomp.hpp" + +#ifdef HAVE_INF_ENGINE +#include +#endif // HAVE_INF_ENGINE + +namespace cv { namespace dnn { + +#ifdef HAVE_INF_ENGINE + +class InfEngineBackendNet : public InferenceEngine::ICNNNetwork +{ +public: + virtual void Release() noexcept; + + virtual InferenceEngine::Precision getPrecision() noexcept; + + virtual void getOutputsInfo(InferenceEngine::OutputsDataMap &out) noexcept; + + virtual void getInputsInfo(InferenceEngine::InputsDataMap &inputs) noexcept; + + virtual InferenceEngine::InputInfo::Ptr getInput(const std::string &inputName) noexcept; + + virtual void getName(char *pName, size_t len) noexcept; + + virtual size_t layerCount() noexcept; + + virtual InferenceEngine::DataPtr& getData(const char *dname) noexcept; + + virtual void addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept; + + virtual InferenceEngine::StatusCode addOutput(const std::string &layerName, + size_t outputIndex = 0, + InferenceEngine::ResponseDesc *resp = nullptr) noexcept; + + virtual InferenceEngine::StatusCode getLayerByName(const char *layerName, + InferenceEngine::CNNLayerPtr &out, + InferenceEngine::ResponseDesc *resp) noexcept; + + virtual void setTargetDevice(InferenceEngine::TargetDevice device) noexcept; + + virtual InferenceEngine::TargetDevice getTargetDevice() noexcept; + + virtual InferenceEngine::StatusCode setBatchSize(const size_t size) noexcept; + + virtual size_t getBatchSize() const noexcept; + + void initEngine(); + + void addBlobs(const std::vector >& wrappers); + + void forward(); + + bool isInitialized(); + +private: + std::vector layers; + InferenceEngine::InputsDataMap inputs; + InferenceEngine::OutputsDataMap outputs; + InferenceEngine::BlobMap inpBlobs; + InferenceEngine::BlobMap outBlobs; + InferenceEngine::BlobMap allBlobs; + InferenceEngine::InferenceEnginePluginPtr engine; +}; + +class InfEngineBackendNode : public BackendNode +{ +public: + InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& layer); + + void connect(std::vector >& inputs, + std::vector >& outputs); + + InferenceEngine::CNNLayerPtr layer; + // Inference Engine network object that allows to obtain the outputs of this layer. + Ptr net; +}; + +class InfEngineBackendWrapper : public BackendWrapper +{ +public: + InfEngineBackendWrapper(int targetId, const Mat& m); + + ~InfEngineBackendWrapper(); + + virtual void copyToHost(); + + virtual void setHostDirty(); + + InferenceEngine::DataPtr dataPtr; + InferenceEngine::TBlob::Ptr blob; +}; + +InferenceEngine::TBlob::Ptr wrapToInfEngineBlob(const Mat& m); + +InferenceEngine::TBlob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector& shape); + +InferenceEngine::DataPtr infEngineDataNode(const Ptr& ptr); + +// Fuses convolution weights and biases with channel-wise scales and shifts. +void fuseConvWeights(const std::shared_ptr& conv, + const Mat& w, const Mat& b = Mat()); + +#endif // HAVE_INF_ENGINE + +bool haveInfEngine(); + +void forwardInfEngine(Ptr& node); + +}} // namespace dnn, namespace cv + +#endif // __OPENCV_DNN_OP_INF_ENGINE_HPP__ diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 684de09e3f..36765d14a3 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -14,7 +14,7 @@ using namespace cv; using namespace dnn; using namespace testing; -CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE) +CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE, DNN_BACKEND_INFERENCE_ENGINE) CV_ENUM(DNNTarget, DNN_TARGET_CPU, DNN_TARGET_OPENCL) static void loadNet(const std::string& weights, const std::string& proto, @@ -151,6 +151,7 @@ TEST_P(DNNTestNetwork, GoogLeNet) TEST_P(DNNTestNetwork, Inception_5h) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/tensorflow_inception_graph.pb", "", Size(224, 224), "softmax2", "tensorflow", target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_inception_5h.yml" : "dnn/halide_scheduler_inception_5h.yml"); @@ -158,6 +159,7 @@ TEST_P(DNNTestNetwork, Inception_5h) TEST_P(DNNTestNetwork, ENet) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/Enet-model-best.net", "", Size(512, 512), "l367_Deconvolution", "torch", target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_enet.yml" : "dnn/halide_scheduler_enet.yml", @@ -176,16 +178,48 @@ TEST_P(DNNTestNetwork, MobileNetSSD) TEST_P(DNNTestNetwork, SSD_VGG16) { if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL || - backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU) + backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU || + backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException(""); processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", Size(300, 300), "detection_out", "caffe"); } +TEST_P(DNNTestNetwork, OpenPose_pose_coco) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", + Size(368, 368), "", "caffe"); +} + +TEST_P(DNNTestNetwork, OpenPose_pose_mpi) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", + Size(368, 368), "", "caffe"); +} + +TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) +{ + if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); + // The same .caffemodel but modified .prototxt + // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp + processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", + Size(368, 368), "", "caffe"); +} + +TEST_P(DNNTestNetwork, OpenFace) +{ + processNet("dnn/openface_nn4.small2.v1.t7", "", Size(96, 96), "", "torch"); +} + const tuple testCases[] = { #ifdef HAVE_HALIDE tuple(DNN_BACKEND_HALIDE, DNN_TARGET_CPU), tuple(DNN_BACKEND_HALIDE, DNN_TARGET_OPENCL), +#endif +#ifdef HAVE_INF_ENGINE + tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_CPU), #endif tuple(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL) }; diff --git a/samples/dnn/openpose.py b/samples/dnn/openpose.py new file mode 100644 index 0000000000..9b48af5101 --- /dev/null +++ b/samples/dnn/openpose.py @@ -0,0 +1,105 @@ +# To use Inference Engine backend, specify location of plugins: +# export LD_LIBRARY_PATH=/opt/intel/deeplearning_deploymenttoolkit/deployment_tools/external/mklml_lnx/lib:$LD_LIBRARY_PATH +import cv2 as cv +import numpy as np +import argparse + +parser = argparse.ArgumentParser( + description='This script is used to demonstrate OpenPose human pose estimation network ' + 'from https://github.com/CMU-Perceptual-Computing-Lab/openpose project using OpenCV. ' + 'The sample and model are simplified and could be used for a single person on the frame.') +parser.add_argument('--input', help='Path to image or video. Skip to capture frames from camera') +parser.add_argument('--proto', help='Path to .prototxt') +parser.add_argument('--model', help='Path to .caffemodel') +parser.add_argument('--dataset', help='Specify what kind of model was trained. ' + 'It could be (COCO, MPI) depends on dataset.') +parser.add_argument('--thr', default=0.1, type=float, help='Threshold value for pose parts heat map') +parser.add_argument('--width', default=368, type=int, help='Resize input to specific width.') +parser.add_argument('--height', default=368, type=int, help='Resize input to specific height.') +parser.add_argument('--inf_engine', action='store_true', + help='Enable Intel Inference Engine computational backend. ' + 'Check that plugins folder is in LD_LIBRARY_PATH environment variable') + +args = parser.parse_args() + +if args.dataset == 'COCO': + BODY_PARTS = { "Nose": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4, + "LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9, + "RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "REye": 14, + "LEye": 15, "REar": 16, "LEar": 17, "Background": 18 } + + POSE_PAIRS = [ ["Neck", "RShoulder"], ["Neck", "LShoulder"], ["RShoulder", "RElbow"], + ["RElbow", "RWrist"], ["LShoulder", "LElbow"], ["LElbow", "LWrist"], + ["Neck", "RHip"], ["RHip", "RKnee"], ["RKnee", "RAnkle"], ["Neck", "LHip"], + ["LHip", "LKnee"], ["LKnee", "LAnkle"], ["Neck", "Nose"], ["Nose", "REye"], + ["REye", "REar"], ["Nose", "LEye"], ["LEye", "LEar"] ] +else: + assert(args.dataset == 'MPI') + BODY_PARTS = { "Head": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4, + "LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9, + "RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "Chest": 14, + "Background": 15 } + + POSE_PAIRS = [ ["Head", "Neck"], ["Neck", "RShoulder"], ["RShoulder", "RElbow"], + ["RElbow", "RWrist"], ["Neck", "LShoulder"], ["LShoulder", "LElbow"], + ["LElbow", "LWrist"], ["Neck", "Chest"], ["Chest", "RHip"], ["RHip", "RKnee"], + ["RKnee", "RAnkle"], ["Chest", "LHip"], ["LHip", "LKnee"], ["LKnee", "LAnkle"] ] + +inWidth = args.width +inHeight = args.height + +net = cv.dnn.readNetFromCaffe(args.proto, args.model) +if args.inf_engine: + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + +cap = cv.VideoCapture(args.input if args.input else 0) + +while cv.waitKey(1) < 0: + hasFrame, frame = cap.read() + if not hasFrame: + cv.waitKey() + break + + frameWidth = frame.shape[1] + frameHeight = frame.shape[0] + inp = cv.dnn.blobFromImage(frame, 1.0 / 255, (inWidth, inHeight), + (0, 0, 0), swapRB=False, crop=False) + net.setInput(inp) + out = net.forward() + + assert(len(BODY_PARTS) == out.shape[1]) + + points = [] + for i in range(len(BODY_PARTS)): + # Slice heatmap of corresponging body's part. + heatMap = out[0, i, :, :] + + # Originally, we try to find all the local maximums. To simplify a sample + # we just find a global one. However only a single pose at the same time + # could be detected this way. + _, conf, _, point = cv.minMaxLoc(heatMap) + x = (frameWidth * point[0]) / out.shape[3] + y = (frameHeight * point[1]) / out.shape[2] + + # Add a point if it's confidence is higher than threshold. + points.append((x, y) if conf > args.thr else None) + + for pair in POSE_PAIRS: + partFrom = pair[0] + partTo = pair[1] + assert(partFrom in BODY_PARTS) + assert(partTo in BODY_PARTS) + + idFrom = BODY_PARTS[partFrom] + idTo = BODY_PARTS[partTo] + + if points[idFrom] and points[idTo]: + cv.line(frame, points[idFrom], points[idTo], (0, 255, 0), 3) + cv.ellipse(frame, points[idFrom], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED) + cv.ellipse(frame, points[idTo], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED) + + t, _ = net.getPerfProfile() + freq = cv.getTickFrequency() / 1000 + cv.putText(frame, '%.2fms' % (t / freq), (10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0)) + + cv.imshow('OpenPose using OpenCV', frame)