diff --git a/modules/dnn/include/opencv2/dnn/layer.hpp b/modules/dnn/include/opencv2/dnn/layer.hpp index 8500599371..a4d167564d 100644 --- a/modules/dnn/include/opencv2/dnn/layer.hpp +++ b/modules/dnn/include/opencv2/dnn/layer.hpp @@ -66,6 +66,9 @@ public: //! Unregisters registered layer with specified type name. Thread-safe. static void unregisterLayer(const String &type); + //! Check if layer is registered. + static bool isLayerRegistered(const std::string& type); + /** @brief Creates instance of registered layer. * @param type type name of creating layer. * @param params parameters which will be used for layer initialization. diff --git a/modules/dnn/src/debug_utils.cpp b/modules/dnn/src/debug_utils.cpp index d951205bd8..0e1ba10236 100644 --- a/modules/dnn/src/debug_utils.cpp +++ b/modules/dnn/src/debug_utils.cpp @@ -37,11 +37,8 @@ void skipModelImport(bool skip) void detail::LayerHandler::addMissing(const std::string& name, const std::string& type) { - cv::AutoLock lock(getLayerFactoryMutex()); - auto& registeredLayers = getLayerFactoryImpl(); - // If we didn't add it, but can create it, it's custom and not missing. - if (layers.find(type) == layers.end() && registeredLayers.find(type) != registeredLayers.end()) + if (!contains(type) && LayerFactory::isLayerRegistered(type)) { return; } @@ -51,17 +48,17 @@ void detail::LayerHandler::addMissing(const std::string& name, const std::string bool detail::LayerHandler::contains(const std::string& type) const { - return layers.find(type) != layers.end(); + return layers.count(type) != 0; } -void detail::LayerHandler::printMissing() +void detail::LayerHandler::printMissing() const { if (layers.empty()) { return; } - std::stringstream ss; + std::ostringstream ss; ss << "DNN: Not supported types:\n"; for (const auto& type_names : layers) { diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 67312dba78..b58a097e67 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -6092,6 +6092,13 @@ void LayerFactory::unregisterLayer(const String &type) } } +bool LayerFactory::isLayerRegistered(const std::string& type) +{ + cv::AutoLock lock(getLayerFactoryMutex()); + auto& registeredLayers = getLayerFactoryImpl(); + return registeredLayers.find(type) != registeredLayers.end(); +} + Ptr LayerFactory::createLayerInstance(const String &type, LayerParams& params) { CV_TRACE_FUNCTION(); diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index ffeb3bfda1..99628a8322 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -5,8 +5,8 @@ #ifndef __OPENCV_DNN_COMMON_HPP__ #define __OPENCV_DNN_COMMON_HPP__ -#include #include +#include #include @@ -59,7 +59,7 @@ class LayerHandler public: void addMissing(const std::string& name, const std::string& type); bool contains(const std::string& type) const; - void printMissing(); + void printMissing() const; protected: LayerParams getNotImplementedParams(const std::string& name, const std::string& op); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 0a691913a8..c1ed6e90c3 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -48,6 +48,8 @@ CV__DNN_INLINE_NS_BEGIN extern bool DNN_DIAGNOSTICS_RUN; +class ONNXLayerHandler; + class ONNXImporter { opencv_onnx::ModelProto model_proto; @@ -80,7 +82,7 @@ public: void populateNet(); protected: - std::unique_ptr missingLayerHandler; + std::unique_ptr layerHandler; Net& dstNet; opencv_onnx::GraphProto graph_proto; @@ -98,11 +100,14 @@ protected: void handleNode(const opencv_onnx::NodeProto& node_proto); private: + friend class ONNXLayerHandler; typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); typedef std::map DispatchMap; typedef std::map DomainDispatchMap; DomainDispatchMap domain_dispatch_map; + std::string getLayerTypeDomain(const opencv_onnx::NodeProto& node_proto); + const DispatchMap& getDispatchMap(const opencv_onnx::NodeProto& node_proto); void buildDispatchMap_ONNX_AI(int opset_version); void buildDispatchMap_COM_MICROSOFT(int opset_version); @@ -156,6 +161,7 @@ private: void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSimpleLayers (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); // Domain: com.microsoft // URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md @@ -178,9 +184,38 @@ private: const std::string str_domain_ai_onnx = "ai.onnx"; }; +class ONNXLayerHandler : public detail::LayerHandler +{ +public: + explicit ONNXLayerHandler(ONNXImporter* importer_); + + void fillRegistry(const opencv_onnx::GraphProto& net); + +protected: + ONNXImporter* importer; +}; + +ONNXLayerHandler::ONNXLayerHandler(ONNXImporter* importer_) : importer(importer_){} + +void ONNXLayerHandler::fillRegistry(const opencv_onnx::GraphProto &net) +{ + int layersSize = net.node_size(); + for (int li = 0; li < layersSize; li++) { + const opencv_onnx::NodeProto &node_proto = net.node(li); + const std::string& name = node_proto.output(0); + const std::string& type = node_proto.op_type(); + const std::string& layer_type_domain = importer->getLayerTypeDomain(node_proto); + const auto& dispatch = importer->getDispatchMap(node_proto); + if (dispatch.find(type) == dispatch.end()) + { + addMissing(name, cv::format("%s.%s", layer_type_domain.c_str(), type.c_str())); + } + } + printMissing(); +} ONNXImporter::ONNXImporter(Net& net, const char *onnxFile) - : missingLayerHandler(DNN_DIAGNOSTICS_RUN ? new detail::LayerHandler() : nullptr) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr) , dstNet(net) , onnx_opset(0) { @@ -203,7 +238,7 @@ ONNXImporter::ONNXImporter(Net& net, const char *onnxFile) } ONNXImporter::ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : missingLayerHandler(DNN_DIAGNOSTICS_RUN ? new detail::LayerHandler() : nullptr) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr) , dstNet(net) , onnx_opset(0) { @@ -795,6 +830,7 @@ void ONNXImporter::populateNet() if (DNN_DIAGNOSTICS_RUN) { CV_LOG_INFO(NULL, "DNN/ONNX: start diagnostic run!"); + layerHandler->fillRegistry(graph_proto); } for(int li = 0; li < layersSize; li++) @@ -806,54 +842,48 @@ void ONNXImporter::populateNet() CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN ? "DNN/ONNX: diagnostic run completed!" : "DNN/ONNX: import completed!")); } +std::string ONNXImporter::getLayerTypeDomain(const opencv_onnx::NodeProto& node_proto) +{ + if (!node_proto.has_domain()) + return str_domain_ai_onnx; + const std::string& domain = node_proto.domain(); + if (domain.empty()) + return str_domain_ai_onnx; + return domain; +} + +const ONNXImporter::DispatchMap& ONNXImporter::getDispatchMap(const opencv_onnx::NodeProto& node_proto) +{ + static DispatchMap empty_map; + const std::string& layer_type_domain = getLayerTypeDomain(node_proto); + auto it = domain_dispatch_map.find(layer_type_domain); + if (it == domain_dispatch_map.end()) + { + return empty_map; + } + + return it->second; +} + void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) { CV_Assert(node_proto.output_size() >= 1); const std::string& name = node_proto.output(0); const std::string& layer_type = node_proto.op_type(); - const std::string& layer_type_domain = [&]() - { - if (!node_proto.has_domain()) - return str_domain_ai_onnx; - const std::string& domain = node_proto.domain(); - if (domain.empty()) - return str_domain_ai_onnx; - return domain; - }(); - const auto& dispatch = [&]() + const std::string& layer_type_domain = getLayerTypeDomain(node_proto); + const auto& dispatch = getDispatchMap(node_proto); + + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " + << node_proto.output_size() << " outputs: " + << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) + << cv::format(" from %sdomain='", onnx_opset_map.count(layer_type_domain) == 1 ? "" : "undeclared ") + << layer_type_domain << "'" + ); + + if (dispatch.empty()) { - if (layer_type_domain != str_domain_ai_onnx) - { - if (onnx_opset_map.find(layer_type_domain) == onnx_opset_map.end()) - { - CV_LOG_WARNING(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " - << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) - << " from undeclared domain='" << layer_type_domain << "'" - ); - } - else - { - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " - << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) - << " from domain='" << layer_type_domain << "'" - ); - } - auto it = domain_dispatch_map.find(layer_type_domain); - if (it == domain_dispatch_map.end()) - { - CV_LOG_WARNING(NULL, "DNN/ONNX: missing dispatch map for domain='" << layer_type_domain << "'"); - return DispatchMap(); - } - return it->second; - } - else - { - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " - << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) - ); - return domain_dispatch_map[str_domain_ai_onnx]; - } - }(); + CV_LOG_WARNING(NULL, "DNN/ONNX: missing dispatch map for domain='" << layer_type_domain << "'"); + } LayerParams layerParams; try @@ -2882,6 +2912,15 @@ void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } +void ONNXImporter::parseSimpleLayers(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int j = 0; j < node_proto.input_size(); j++) { + if (layer_id.find(node_proto.input(j)) == layer_id.end()) + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { const std::string& name = layerParams.name; @@ -2897,20 +2936,11 @@ void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx: } } - CV_LOG_INFO(NULL, "DNN/ONNX: unknown node type, try using custom handler for node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + CV_LOG_IF_INFO(NULL, !LayerFactory::isLayerRegistered(layer_type), "DNN/ONNX: unknown node type, try using custom handler for node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); - if (missingLayerHandler) - { - missingLayerHandler->addMissing(layerParams.name, layerParams.type); - } - - for (int j = 0; j < node_proto.input_size(); j++) { - if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - addLayer(layerParams, node_proto); + parseSimpleLayers(layerParams, node_proto); } void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) @@ -3360,6 +3390,15 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version) dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; + std::vector simpleLayers{"Acos", "Acosh", "Asin", "Asinh", "Atan", "Atanh", "Ceil", "Celu", "Cos", + "Cosh", "Dropout", "Erf", "Exp", "Floor", "HardSigmoid", "HardSwish", + "Identity", "Log", "Round", "Selu", "Sigmoid", "Sin", "Sinh", "Softmax", + "Softplus", "Softsign", "Sqrt", "Tan", "ThresholdedRelu"}; + for (const auto& name : simpleLayers) + { + dispatch[name] = &ONNXImporter::parseSimpleLayers; + } + // ai.onnx: opset 10+ dispatch["QuantizeLinear"] = dispatch["DequantizeLinear"] = &ONNXImporter::parseQuantDequant; dispatch["QLinearConv"] = &ONNXImporter::parseQConv; diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index efaedfaab1..fa190005ac 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -3090,10 +3090,8 @@ void TFImporter::populateNet() { const tensorflow::NodeDef& layer = net.node(li); - const std::string name = layer.name(); - const std::string type = layer.op(); - const int ninputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: (" << li << "/" << layersSize << ") Parse layer " << name << " @ " << type << " with " << ninputs << " inputs"); + CV_LOG_DEBUG(NULL, "DNN/TF: processing node (" << li << "/" << layersSize << ") with " << layer.input_size() << " inputs: " + << cv::format("[%s]:(%s)", layer.op().c_str(), layer.name().c_str())); parseNode(layer); }