diff --git a/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown b/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown index a2d4b2a306..ce95234f88 100644 --- a/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown +++ b/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown @@ -24,7 +24,9 @@ model, but the methodology applies to other supported models. @note Currently, OpenCV supports the following YOLO models: - [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX/blob/main), -- [YoloNas](https://github.com/Deci-AI/super-gradients/tree/master), +- [YOLONas](https://github.com/Deci-AI/super-gradients/tree/master), +- [YOLOv10](https://github.com/THU-MIG/yolov10/tree/main), +- [YOLOv9](https://github.com/WongKinYiu/yolov9), - [YOLOv8](https://github.com/ultralytics/ultralytics/tree/main), - [YOLOv7](https://github.com/WongKinYiu/yolov7/tree/main), - [YOLOv6](https://github.com/meituan/YOLOv6/blob/main), @@ -79,7 +81,7 @@ the ONNX graph, a process that we will detail further in the subsequent sections Now that we know know the parameters of the pre-precessing we can go on and export the model from Pytorch to ONNX graph. Since in this tutorial we are using YOLOX as our sample model, lets use its -export for demonstration purposes (the process is identical for the rest of the YOLO detectors). +export for demonstration purposes (the process is identical for the rest of the YOLO detectors except `YOLOv10` model, see details on how to export it later in the post). To exporting YOLOX we can just use [export script](https://github.com/Megvii-BaseDetection/YOLOX/blob/ac58e0a5e68e57454b7b9ac822aced493b553c53/tools/export_onnx.py). Particularly we need following commands: @code{.bash} @@ -125,6 +127,20 @@ than YOLOX) in case it is needed. However, usually each YOLO repository has pred onnx.save(model_simp, args.output_name) @endcode +#### Exporting YOLOv10 model + +In oder to run YOLOv10 one needs to cut off postporcessing with dynamic shapes from torch and then convert it to ONNX. If someone is looking for on how to cut off the postprocessing, there is this [forked branch](https://github.com/Abdurrahheem/yolov10/tree/ash/opencv-export) from official YOLOv10. The forked branch cuts of the postprocessing by [returning output](https://github.com/Abdurrahheem/yolov10/blob/4fdaafd912c8891642bfbe85751ea66ec20f05ad/ultralytics/nn/modules/head.py#L522) of the model before postprocessing procedure itself. To convert torch model to ONNX follow this proceduce. + +@code{.bash} +git clone git@github.com:Abdurrahheem/yolov10.git +conda create -n yolov10 python=3.9 +conda activate yolov10 +pip install -r requirements.txt +python export_opencv.py --model= --imgsz= +@endcode + +By default `--model="yolov10s"` and `--imgsz=(480,640)`. This will generate file `yolov10s.onnx`, which can be use for inference in OpenCV + ### Running Yolo ONNX detector with OpenCV Sample Once we have our ONNX graph of the model, we just simply can run with OpenCV's sample. To that we need to make sure: @@ -144,24 +160,25 @@ Once we have our ONNX graph of the model, we just simply can run with OpenCV's s --padvalue= \ --paddingmode= \ --backend= \ - --target= + --target= \ + --width= \ + --height= \ @endcode -VIDEO DEMO: -@youtube{NHtRlndE2cg} - - --input: File path to your input image or video. If omitted, it will capture frames from a camera. - --classes: File path to a text file containing class names for object detection. - --thr: Confidence threshold for detection (e.g., 0.5). - --nms: Non-maximum suppression threshold (e.g., 0.4). - --mean: Mean normalization value (e.g., 0.0 for no mean normalization). -- --scale: Scale factor for input normalization (e.g., 1.0). +- --scale: Scale factor for input normalization (e.g., 1.0, 1/255.0, etc). - --yolo: YOLO model version (e.g., YOLOv3, YOLOv4, etc.). - --padvalue: Padding value used in pre-processing (e.g., 114.0). - --paddingmode: Method for handling image resizing and padding. Options: 0 (resize without extra processing), 1 (crop after resize), 2 (resize with aspect ratio preservation). - --backend: Selection of computation backend (0 for automatic, 1 for Halide, 2 for OpenVINO, etc.). - --target: Selection of target computation device (0 for CPU, 1 for OpenCL, etc.). - --device: Camera device number (0 for default camera). If `--input` is not provided camera with index 0 will used by default. +- --width: Model input width. Not to be confused with the image width. (e.g., 416, 480, 640, 1280, etc). +- --height: Model input height. Not to be confused with the image height. (e.g., 416, 480, 640, 1280, etc). Here `mean`, `scale`, `padvalue`, `paddingmode` should exactly match those that we discussed in pre-processing section in order for the model to match result in PyTorch @@ -183,7 +200,8 @@ cd ./bin/example_dnn_yolo_detector @endcode -This will execute the YOLOX detector with your camera. For YOLOv8 (for instance), follow these additional steps: +This will execute the YOLOX detector with your camera. +For YOLOv8 (for instance), follow these additional steps: @code{.sh} cd opencv_extra/testdata/dnn @@ -195,6 +213,23 @@ cd ./bin/example_dnn_yolo_detector --model=onnx/models/yolov8n.onnx --yolo=yolov8 --mean=0.0 --scale=0.003921568627 --paddingmode=2 --padvalue=144.0 --thr=0.5 --nms=0.4 --rgb=0 @endcode +For YOLOv10, follow these steps: + +@code{.sh} +cd opencv_extra/testdata/dnn +python download_models.py yolov10 +cd .. +export OPENCV_TEST_DATA_PATH=$(pwd) +cd + +./bin/example_dnn_yolo_detector --model=onnx/models/yolov8n.onnx --yolo=yolov10 --width=640 --height=480 --scale=0.003921568627 --padvalue=114 +@endcode + +This will run `YOLOv10` detector on first camera found on your system. If you want to run it on a image/video file, you can use `--input` option to specify the path to the file. + + +VIDEO DEMO: +@youtube{NHtRlndE2cg} ### Building a Custom Pipeline diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 82b10fb1ba..35ac0be56a 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -19,7 +19,8 @@ void yoloPostProcessing( std::vector& keep_boxes, float conf_threshold, float iou_threshold, - const std::string& test_name); + const std::string& model_name, + const int nc=80); template static std::string _tf(TString filename, bool required = true) @@ -2670,7 +2671,8 @@ void yoloPostProcessing( std::vector& keep_boxes, float conf_threshold, float iou_threshold, - const std::string& test_name + const std::string& model_name, + const int nc ){ // Retrieve @@ -2678,11 +2680,13 @@ void yoloPostProcessing( std::vector confidences; std::vector boxes; - if (test_name == "yolov8"){ + if (model_name == "yolov8" || model_name == "yolov10" || + model_name == "yolov9") + { cv::transposeND(outs[0], {0, 2, 1}, outs[0]); } - if (test_name == "yolonas"){ + if (model_name == "yolonas"){ // outs contains 2 elemets of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84] Mat concat_out; // squeeze the first dimension @@ -2696,22 +2700,27 @@ void yoloPostProcessing( outs[0] = outs[0].reshape(0, std::vector{1, 8400, 84}); } + // assert if last dim is 85 or 84 + CV_CheckEQ(outs[0].dims, 3, "Invalid output shape. The shape should be [1, #anchors, 85 or 84]"); + CV_CheckEQ((outs[0].size[2] == nc + 5 || outs[0].size[2] == 80 + 4), true, "Invalid output shape: "); + for (auto preds : outs){ preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85] for (int i = 0; i < preds.rows; ++i) { // filter out non object - float obj_conf = (test_name == "yolov8" || test_name == "yolonas") ? 1.0f : preds.at(i, 4) ; + float obj_conf = (model_name == "yolov8" || model_name == "yolonas" || + model_name == "yolov9" || model_name == "yolov10") ? 1.0f : preds.at(i, 4) ; if (obj_conf < conf_threshold) continue; - Mat scores = preds.row(i).colRange((test_name == "yolov8" || test_name == "yolonas") ? 4 : 5, preds.cols); + Mat scores = preds.row(i).colRange((model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? 4 : 5, preds.cols); double conf; Point maxLoc; minMaxLoc(scores, 0, &conf, 0, &maxLoc); - conf = (test_name == "yolov8" || test_name == "yolonas") ? conf : conf * obj_conf; + conf = (model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? conf : conf * obj_conf; if (conf < conf_threshold) continue; @@ -2722,15 +2731,14 @@ void yoloPostProcessing( double w = det[2]; double h = det[3]; - // std::cout << "cx: " << cx << " cy: " << cy << " w: " << w << " h: " << h << " conf: " << conf << " idx: " << maxLoc.x << std::endl; // [x1, y1, x2, y2] - if (test_name == "yolonas"){ + if (model_name == "yolonas" || model_name == "yolov10"){ boxes.push_back(Rect2d(cx, cy, w, h)); } else { boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h, cx + 0.5 * w, cy + 0.5 * h)); } - classIds.push_back(maxLoc.x); + classIds.push_back(maxLoc.x); confidences.push_back(conf); } } @@ -2747,7 +2755,75 @@ void yoloPostProcessing( } } +TEST_P(Test_ONNX_nets, YOLOv10) +{ + + std::string weightPath = _tf("models/yolov10s.onnx", false); + + Size targetSize{640, 480}; + float conf_threshold = 0.50; + float iou_threshold = 0.50; + + std::vector refClassIds{1, 16, 7}; + std::vector refScores{0.9510f, 0.9454f, 0.8404f}; + + std::vector refBoxes{ + Rect2d(105.5014, 112.8838, 472.9274, 350.0603), + Rect2d(109.8231, 185.7994, 258.5916, 452.9302), + Rect2d(388.5018, 62.1034, 576.6399, 143.3986) + }; + + Image2BlobParams imgParams( + Scalar::all(1 / 255.0), + targetSize, + Scalar::all(0), + true, + CV_32F, + DNN_LAYOUT_NCHW, + DNN_PMODE_LETTERBOX, + Scalar::all(114) + ); + testYOLO( + weightPath, refClassIds, refScores, refBoxes, + imgParams, conf_threshold, iou_threshold, + 1.0e-4, 1.0e-4, "yolov10"); +} + +TEST_P(Test_ONNX_nets, YOLOv9) +{ + + std::string weightPath = _tf("models/yolov9t.onnx", false); + + Size targetSize{640, 480}; + float conf_threshold = 0.50; + float iou_threshold = 0.50; + + std::vector refClassIds{1, 16, 2}; // wrong class mapping for yolov9 + std::vector refScores{0.959274f, 0.901125, 0.559396f}; + + std::vector refBoxes{ + Rect2d(106.255, 107.927, 472.497, 350.309), + Rect2d(108.633, 185.256, 259.287, 450.672), + Rect2d(390.701, 62.1454, 576.928, 141.795) + }; + + Image2BlobParams imgParams( + Scalar::all(1 / 255.0), + targetSize, + Scalar::all(0), + true, + CV_32F, + DNN_LAYOUT_NCHW, + DNN_PMODE_LETTERBOX, + Scalar::all(114) + ); + + testYOLO( + weightPath, refClassIds, refScores, refBoxes, + imgParams, conf_threshold, iou_threshold, + 1.0e-4, 1.0e-4, "yolov9"); +} TEST_P(Test_ONNX_nets, YOLOX) { applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); diff --git a/samples/dnn/yolo_detector.cpp b/samples/dnn/yolo_detector.cpp index b439b0d4bc..bd82acff4a 100644 --- a/samples/dnn/yolo_detector.cpp +++ b/samples/dnn/yolo_detector.cpp @@ -27,7 +27,8 @@ void yoloPostProcessing( std::vector& keep_boxes, float conf_threshold, float iou_threshold, - const std::string& test_name + const std::string& model_name, + const int nc ); std::vector classes; @@ -40,6 +41,7 @@ std::string keys = "{ yolo | yolox | yolo model version. }" "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera. }" "{ classes | | Optional path to a text file with names of classes to label detected objects. }" + "{ nc | 80 | Number of classes. Default is 80 (coming from COCO dataset). }" "{ thr | .5 | Confidence threshold. }" "{ nms | .4 | Non-maximum suppression threshold. }" "{ mean | 0.0 | Normalization constant. }" @@ -107,19 +109,21 @@ void yoloPostProcessing( std::vector& keep_boxes, float conf_threshold, float iou_threshold, - const std::string& test_name) + const std::string& model_name, + const int nc=80) { // Retrieve std::vector classIds; std::vector confidences; std::vector boxes; - if (test_name == "yolov8") + if (model_name == "yolov8" || model_name == "yolov10" || + model_name == "yolov9") { cv::transposeND(outs[0], {0, 2, 1}, outs[0]); } - if (test_name == "yolonas") + if (model_name == "yolonas") { // outs contains 2 elemets of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84] Mat concat_out; @@ -131,25 +135,30 @@ void yoloPostProcessing( // remove the second element outs.pop_back(); // unsqueeze the first dimension - outs[0] = outs[0].reshape(0, std::vector{1, 8400, 84}); + outs[0] = outs[0].reshape(0, std::vector{1, 8400, nc + 4}); } + // assert if last dim is 85 or 84 + CV_CheckEQ(outs[0].dims, 3, "Invalid output shape. The shape should be [1, #anchors, 85 or 84]"); + CV_CheckEQ((outs[0].size[2] == nc + 5 || outs[0].size[2] == 80 + 4), true, "Invalid output shape: "); + for (auto preds : outs) { preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85] for (int i = 0; i < preds.rows; ++i) { // filter out non object - float obj_conf = (test_name == "yolov8" || test_name == "yolonas") ? 1.0f : preds.at(i, 4) ; + float obj_conf = (model_name == "yolov8" || model_name == "yolonas" || + model_name == "yolov9" || model_name == "yolov10") ? 1.0f : preds.at(i, 4) ; if (obj_conf < conf_threshold) continue; - Mat scores = preds.row(i).colRange((test_name == "yolov8" || test_name == "yolonas") ? 4 : 5, preds.cols); + Mat scores = preds.row(i).colRange((model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? 4 : 5, preds.cols); double conf; Point maxLoc; minMaxLoc(scores, 0, &conf, 0, &maxLoc); - conf = (test_name == "yolov8" || test_name == "yolonas") ? conf : conf * obj_conf; + conf = (model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? conf : conf * obj_conf; if (conf < conf_threshold) continue; @@ -161,7 +170,7 @@ void yoloPostProcessing( double h = det[3]; // [x1, y1, x2, y2] - if (test_name == "yolonas"){ + if (model_name == "yolonas" || model_name == "yolov10"){ boxes.push_back(Rect2d(cx, cy, w, h)); } else { boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h, @@ -203,6 +212,7 @@ int main(int argc, char** argv) // if model is default, use findFile to get the full path otherwise use the given path std::string weightPath = findFile(parser.get("model")); std::string yolo_model = parser.get("yolo"); + int nc = parser.get("nc"); float confThreshold = parser.get("thr"); float nmsThreshold = parser.get("nms"); @@ -219,6 +229,7 @@ int main(int argc, char** argv) // check if yolo model is valid if (yolo_model != "yolov5" && yolo_model != "yolov6" && yolo_model != "yolov7" && yolo_model != "yolov8" + && yolo_model != "yolov10" && yolo_model !="yolov9" && yolo_model != "yolox" && yolo_model != "yolonas") CV_Error(Error::StsError, "Invalid yolo model: " + yolo_model); @@ -331,7 +342,8 @@ int main(int argc, char** argv) yoloPostProcessing( outs, keep_classIds, keep_confidences, keep_boxes, confThreshold, nmsThreshold, - yolo_model); + yolo_model, + nc); //![postprocess] // covert Rect2d to Rect