From 0d117312c99759ef842f5cc4d2b4891f446bcbb7 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 16 Nov 2018 17:09:54 +0300 Subject: [PATCH] DNN_TARGET_FPGA using Intel's Inference Engine --- modules/dnn/include/opencv2/dnn/dnn.hpp | 5 ++- modules/dnn/perf/perf_net.cpp | 2 +- modules/dnn/src/dnn.cpp | 7 ++- modules/dnn/src/layers/convolution_layer.cpp | 7 ++- modules/dnn/src/op_inf_engine.cpp | 20 +++++++-- modules/dnn/test/test_common.hpp | 9 ++-- modules/dnn/test/test_ie_models.cpp | 45 +++++++++++--------- modules/dnn/test/test_layers.cpp | 2 +- modules/dnn/test/test_misc.cpp | 2 +- 9 files changed, 63 insertions(+), 36 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 20a953b895..3b26bb2421 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -88,7 +88,9 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16, - DNN_TARGET_MYRIAD + DNN_TARGET_MYRIAD, + //! FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin. + DNN_TARGET_FPGA }; /** @brief This class provides all data needed to initialize layer. @@ -501,6 +503,7 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN * | DNN_TARGET_OPENCL | + | + | + | * | DNN_TARGET_OPENCL_FP16 | + | + | | * | DNN_TARGET_MYRIAD | | + | | + * | DNN_TARGET_FPGA | | + | | */ CV_WRAP void setPreferableTarget(int targetId); diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index 1647db3b31..03d7a233f3 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -42,7 +42,7 @@ public: } if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD) { - if (!checkMyriadTarget()) + if (!checkIETarget(DNN_TARGET_MYRIAD)) { throw SkipTestException("Myriad is not available/disabled in OpenCV"); } diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 087769ac3f..036726da28 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -1077,7 +1077,8 @@ struct Net::Impl preferableTarget == DNN_TARGET_CPU || preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16 || - preferableTarget == DNN_TARGET_MYRIAD); + preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_FPGA); if (!netWasAllocated || this->blobsToKeep != blobsToKeep_) { if (preferableBackend == DNN_BACKEND_OPENCV && IS_DNN_OPENCL_TARGET(preferableTarget)) @@ -1512,7 +1513,9 @@ struct Net::Impl ieNode->net = net; auto weightableLayer = std::dynamic_pointer_cast(ieNode->layer); - if ((preferableTarget == DNN_TARGET_OPENCL_FP16 || preferableTarget == DNN_TARGET_MYRIAD) && !fused) + if ((preferableTarget == DNN_TARGET_OPENCL_FP16 || + preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_FPGA) && !fused) { ieNode->layer->precision = InferenceEngine::Precision::FP16; if (weightableLayer) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index bc61e39517..9daceb5c13 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -219,9 +219,14 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { +#ifdef HAVE_INF_ENGINE if (backendId == DNN_BACKEND_INFERENCE_ENGINE) - return preferableTarget != DNN_TARGET_MYRIAD || dilation.width == dilation.height; + { + return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2018R4) || + (preferableTarget != DNN_TARGET_MYRIAD || dilation.width == dilation.height); + } else +#endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } diff --git a/modules/dnn/src/op_inf_engine.cpp b/modules/dnn/src/op_inf_engine.cpp index 4d75f8f2e9..144369063e 100644 --- a/modules/dnn/src/op_inf_engine.cpp +++ b/modules/dnn/src/op_inf_engine.cpp @@ -302,7 +302,8 @@ void InfEngineBackendNet::setTargetDevice(InferenceEngine::TargetDevice device) { if (device != InferenceEngine::TargetDevice::eCPU && device != InferenceEngine::TargetDevice::eGPU && - device != InferenceEngine::TargetDevice::eMYRIAD) + device != InferenceEngine::TargetDevice::eMYRIAD && + device != InferenceEngine::TargetDevice::eFPGA) CV_Error(Error::StsNotImplemented, ""); targetDevice = device; } @@ -314,7 +315,8 @@ InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() noexcept InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() const noexcept { - return targetDevice; + return targetDevice == InferenceEngine::TargetDevice::eFPGA ? + InferenceEngine::TargetDevice::eHETERO : targetDevice; } InferenceEngine::StatusCode InfEngineBackendNet::setBatchSize(const size_t) noexcept @@ -466,6 +468,11 @@ void InfEngineBackendNet::init(int targetId) setPrecision(InferenceEngine::Precision::FP16); setTargetDevice(InferenceEngine::TargetDevice::eMYRIAD); break; } + case DNN_TARGET_FPGA: + { + setPrecision(InferenceEngine::Precision::FP16); + setTargetDevice(InferenceEngine::TargetDevice::eFPGA); break; + } default: CV_Error(Error::StsError, format("Unknown target identifier: %d", targetId)); } @@ -489,10 +496,15 @@ void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net) } else { - enginePtr = InferenceEngine::PluginDispatcher({""}).getSuitablePlugin(targetDevice); + auto dispatcher = InferenceEngine::PluginDispatcher({""}); + if (targetDevice == InferenceEngine::TargetDevice::eFPGA) + enginePtr = dispatcher.getPluginByDevice("HETERO:FPGA,CPU"); + else + enginePtr = dispatcher.getSuitablePlugin(targetDevice); sharedPlugins[targetDevice] = enginePtr; - if (targetDevice == InferenceEngine::TargetDevice::eCPU) + if (targetDevice == InferenceEngine::TargetDevice::eCPU || + targetDevice == InferenceEngine::TargetDevice::eFPGA) { std::string suffixes[] = {"_avx2", "_sse4", ""}; bool haveFeature[] = { diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index bee1f222a3..9e9b25ef95 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -66,6 +66,7 @@ static inline void PrintTo(const cv::dnn::Target& v, std::ostream* os) case DNN_TARGET_OPENCL: *os << "OCL"; return; case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return; case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return; + case DNN_TARGET_FPGA: *os << "FPGA"; return; } // don't use "default:" to emit compiler warnings *os << "DNN_TARGET_UNKNOWN(" << (int)v << ")"; } @@ -188,7 +189,7 @@ static inline void normAssertDetections(cv::Mat ref, cv::Mat out, const char *co testBoxes, comment, confThreshold, scores_diff, boxes_iou_diff); } -static inline bool checkMyriadTarget() +static inline bool checkIETarget(int target) { #ifndef HAVE_INF_ENGINE return false; @@ -197,7 +198,7 @@ static inline bool checkMyriadTarget() cv::dnn::LayerParams lp; net.addLayerToPrev("testLayer", "Identity", lp); net.setPreferableBackend(cv::dnn::DNN_BACKEND_INFERENCE_ENGINE); - net.setPreferableTarget(cv::dnn::DNN_TARGET_MYRIAD); + net.setPreferableTarget(target); static int inpDims[] = {1, 2, 3, 4}; net.setInput(cv::Mat(4, &inpDims[0], CV_32FC1, cv::Scalar(0))); try @@ -264,7 +265,7 @@ testing::internal::ParamGenerator > dnnBackendsAndTargets targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL_FP16)); } #endif - if (checkMyriadTarget()) + if (checkIETarget(DNN_TARGET_MYRIAD)) targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_MYRIAD)); } #endif @@ -344,7 +345,7 @@ public: } if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD) { - if (!checkMyriadTarget()) + if (!checkIETarget(DNN_TARGET_MYRIAD)) { throw SkipTestException("Myriad is not available/disabled in OpenCV"); } diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index a50fed8d58..a8404e09a2 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -57,28 +57,29 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath InferencePlugin plugin; ExecutableNetwork netExec; InferRequest infRequest; - TargetDevice targetDevice; - switch (target) - { - case DNN_TARGET_CPU: - targetDevice = TargetDevice::eCPU; - break; - case DNN_TARGET_OPENCL: - case DNN_TARGET_OPENCL_FP16: - targetDevice = TargetDevice::eGPU; - break; - case DNN_TARGET_MYRIAD: - targetDevice = TargetDevice::eMYRIAD; - break; - default: - CV_Error(Error::StsNotImplemented, "Unknown target"); - }; - try { - enginePtr = PluginDispatcher({""}).getSuitablePlugin(targetDevice); - - if (targetDevice == TargetDevice::eCPU) + auto dispatcher = InferenceEngine::PluginDispatcher({""}); + switch (target) + { + case DNN_TARGET_CPU: + enginePtr = dispatcher.getSuitablePlugin(TargetDevice::eCPU); + break; + case DNN_TARGET_OPENCL: + case DNN_TARGET_OPENCL_FP16: + enginePtr = dispatcher.getSuitablePlugin(TargetDevice::eGPU); + break; + case DNN_TARGET_MYRIAD: + enginePtr = dispatcher.getSuitablePlugin(TargetDevice::eMYRIAD); + break; + case DNN_TARGET_FPGA: + enginePtr = dispatcher.getPluginByDevice("HETERO:FPGA,CPU"); + break; + default: + CV_Error(Error::StsNotImplemented, "Unknown target"); + }; + + if (target == DNN_TARGET_CPU || target == DNN_TARGET_FPGA) { std::string suffixes[] = {"_avx2", "_sse4", ""}; bool haveFeature[] = { @@ -255,8 +256,10 @@ static testing::internal::ParamGenerator dnnDLIETargets() targets.push_back(DNN_TARGET_OPENCL_FP16); } #endif - if (checkMyriadTarget()) + if (checkIETarget(DNN_TARGET_MYRIAD)) targets.push_back(DNN_TARGET_MYRIAD); + if (checkIETarget(DNN_TARGET_FPGA)) + targets.push_back(DNN_TARGET_FPGA); return testing::ValuesIn(targets); } diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index cf94fad701..2b2148573b 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -351,7 +351,7 @@ TEST_P(Test_Caffe_layers, Conv_Elu) { if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD) { - if (!checkMyriadTarget()) + if (!checkIETarget(DNN_TARGET_MYRIAD)) throw SkipTestException("Myriad is not available/disabled in OpenCV"); } diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index 73163f73d6..327de6f27d 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -157,7 +157,7 @@ TEST_P(setInput, normalization) const int target = get<1>(get<3>(GetParam())); const bool kSwapRB = true; - if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD && !checkMyriadTarget()) + if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD && !checkIETarget(DNN_TARGET_MYRIAD)) throw SkipTestException("Myriad is not available/disabled in OpenCV"); if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16 && dtype != CV_32F) throw SkipTestException("");