From 9fa014edcde69bdfdad126d309813294d3432bff Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Fri, 2 Jun 2023 16:18:24 +0300 Subject: [PATCH] Merge pull request #23264 from AleksandrPanov:add_detect_qr_with_aruco Add detect qr with aruco #23264 Using Aruco to detect finder patterns to search QR codes. TODO (in next PR): - add single QR detect (update `detect()` and `detectAndDecode()`) - need reduce full enumeration of finder patterns - need add finder pattern info to `decode` step - need to merge the pipeline of the old and new algorithm [Current results:](https://docs.google.com/spreadsheets/d/1ufKyR-Zs-IGXwvqPgftssmTlceVjiQX364sbrjr2QU8/edit#gid=1192415584) +20% total detect, +8% total decode in OpenCV [QR benchmark](https://github.com/opencv/opencv_benchmarks/tree/develop/python_benchmarks/qr_codes) ![res1](https://user-images.githubusercontent.com/22337800/231228556-191d3eae-a318-44e1-af99-e7d420bf6248.png) 78.4% detect, 58.7% decode vs 58.5 detect, 50.5% decode in default [main.py.txt](https://github.com/opencv/opencv/files/10762369/main.py.txt) ![res2](https://user-images.githubusercontent.com/22337800/231229123-ed7f1eda-159a-444b-a3ff-f107d8eb4a20.png) add new info to [google docs](https://docs.google.com/spreadsheets/d/1ufKyR-Zs-IGXwvqPgftssmTlceVjiQX364sbrjr2QU8/edit?usp=sharing) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../objdetect/include/opencv2/objdetect.hpp | 146 ++-- modules/objdetect/misc/objc/gen_dict.json | 7 + .../objdetect/perf/perf_qrcode_pipeline.cpp | 84 +-- modules/objdetect/src/qrcode.cpp | 696 ++++++++++++++++-- modules/objdetect/test/test_qr_utils.hpp | 78 ++ modules/objdetect/test/test_qrcode.cpp | 264 ++----- samples/cpp/qrcode.cpp | 16 +- 7 files changed, 929 insertions(+), 362 deletions(-) create mode 100644 modules/objdetect/misc/objc/gen_dict.json create mode 100644 modules/objdetect/test/test_qr_utils.hpp diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index c398f3a0cd..70c787a389 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -45,6 +45,7 @@ #define OPENCV_OBJDETECT_HPP #include "opencv2/core.hpp" +#include "opencv2/objdetect/aruco_detector.hpp" /** @defgroup objdetect Object Detection @@ -763,28 +764,15 @@ public: }; -class CV_EXPORTS_W QRCodeDetector -{ +class CV_EXPORTS_W_SIMPLE QRCodeDetectorBase { public: - CV_WRAP QRCodeDetector(); - ~QRCodeDetector(); + CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first) + QRCodeDetectorBase(); - /** @brief sets the epsilon used during the horizontal scan of QR code stop marker detection. - @param epsX Epsilon neighborhood, which allows you to determine the horizontal pattern - of the scheme 1:1:3:1:1 according to QR code standard. - */ - CV_WRAP void setEpsX(double epsX); - /** @brief sets the epsilon used during the vertical scan of QR code stop marker detection. - @param epsY Epsilon neighborhood, which allows you to determine the vertical pattern - of the scheme 1:1:3:1:1 according to QR code standard. - */ - CV_WRAP void setEpsY(double epsY); - - /** @brief use markers to improve the position of the corners of the QR code - * - * alignmentMarkers using by default - */ - CV_WRAP void setUseAlignmentMarkers(bool useAlignmentMarkers); + QRCodeDetectorBase(const QRCodeDetectorBase&) = default; + QRCodeDetectorBase(QRCodeDetectorBase&&) = default; + QRCodeDetectorBase& operator=(const QRCodeDetectorBase&) = default; + QRCodeDetectorBase& operator=(QRCodeDetectorBase&&) = default; /** @brief Detects QR code in image and returns the quadrangle containing the code. @param img grayscale or color (BGR) image containing (or not) QR code. @@ -799,16 +787,7 @@ public: @param points Quadrangle vertices found by detect() method (or some other algorithm). @param straight_qrcode The optional output image containing rectified and binarized QR code */ - CV_WRAP std::string decode(InputArray img, InputArray points, OutputArray straight_qrcode = noArray()); - - /** @brief Decodes QR code on a curved surface in image once it's found by the detect() method. - - Returns UTF8-encoded output string or empty string if the code cannot be decoded. - @param img grayscale or color (BGR) image containing QR code. - @param points Quadrangle vertices found by detect() method (or some other algorithm). - @param straight_qrcode The optional output image containing rectified and binarized QR code - */ - CV_WRAP cv::String decodeCurved(InputArray img, InputArray points, OutputArray straight_qrcode = noArray()); + CV_WRAP std::string decode(InputArray img, InputArray points, OutputArray straight_qrcode = noArray()) const; /** @brief Both detects and decodes QR code @@ -817,16 +796,8 @@ public: @param straight_qrcode The optional output image containing rectified and binarized QR code */ CV_WRAP std::string detectAndDecode(InputArray img, OutputArray points=noArray(), - OutputArray straight_qrcode = noArray()); - - /** @brief Both detects and decodes QR code on a curved surface + OutputArray straight_qrcode = noArray()) const; - @param img grayscale or color (BGR) image containing QR code. - @param points optional output array of vertices of the found QR code quadrangle. Will be empty if not found. - @param straight_qrcode The optional output image containing rectified and binarized QR code - */ - CV_WRAP std::string detectAndDecodeCurved(InputArray img, OutputArray points=noArray(), - OutputArray straight_qrcode = noArray()); /** @brief Detects QR codes in image and returns the vector of the quadrangles containing the codes. @param img grayscale or color (BGR) image containing (or not) QR codes. @@ -860,18 +831,109 @@ public: OutputArray points = noArray(), OutputArrayOfArrays straight_qrcode = noArray() ) const; - -protected: struct Impl; +protected: Ptr p; }; +class CV_EXPORTS_W_SIMPLE QRCodeDetector : public QRCodeDetectorBase +{ +public: + CV_WRAP QRCodeDetector(); + + /** @brief sets the epsilon used during the horizontal scan of QR code stop marker detection. + @param epsX Epsilon neighborhood, which allows you to determine the horizontal pattern + of the scheme 1:1:3:1:1 according to QR code standard. + */ + CV_WRAP QRCodeDetector& setEpsX(double epsX); + /** @brief sets the epsilon used during the vertical scan of QR code stop marker detection. + @param epsY Epsilon neighborhood, which allows you to determine the vertical pattern + of the scheme 1:1:3:1:1 according to QR code standard. + */ + CV_WRAP QRCodeDetector& setEpsY(double epsY); + + /** @brief use markers to improve the position of the corners of the QR code + * + * alignmentMarkers using by default + */ + CV_WRAP QRCodeDetector& setUseAlignmentMarkers(bool useAlignmentMarkers); + + /** @brief Decodes QR code on a curved surface in image once it's found by the detect() method. + + Returns UTF8-encoded output string or empty string if the code cannot be decoded. + @param img grayscale or color (BGR) image containing QR code. + @param points Quadrangle vertices found by detect() method (or some other algorithm). + @param straight_qrcode The optional output image containing rectified and binarized QR code + */ + CV_WRAP cv::String decodeCurved(InputArray img, InputArray points, OutputArray straight_qrcode = noArray()); + + /** @brief Both detects and decodes QR code on a curved surface + + @param img grayscale or color (BGR) image containing QR code. + @param points optional output array of vertices of the found QR code quadrangle. Will be empty if not found. + @param straight_qrcode The optional output image containing rectified and binarized QR code + */ + CV_WRAP std::string detectAndDecodeCurved(InputArray img, OutputArray points=noArray(), + OutputArray straight_qrcode = noArray()); +}; + +class CV_EXPORTS_W_SIMPLE QRCodeDetectorAruco : public QRCodeDetectorBase { +public: + CV_WRAP QRCodeDetectorAruco(); + + struct CV_EXPORTS_W_SIMPLE Params { + CV_WRAP Params(); + + /** @brief The minimum allowed pixel size of a QR module in the smallest image in the image pyramid, default 4.f */ + CV_PROP_RW float minModuleSizeInPyramid; + + /** @brief The maximum allowed relative rotation for finder patterns in the same QR code, default pi/12 */ + CV_PROP_RW float maxRotation; + + /** @brief The maximum allowed relative mismatch in module sizes for finder patterns in the same QR code, default 1.75f */ + CV_PROP_RW float maxModuleSizeMismatch; + + /** @brief The maximum allowed module relative mismatch for timing pattern module, default 2.f + * + * If relative mismatch of timing pattern module more this value, penalty points will be added. + * If a lot of penalty points are added, QR code will be rejected. */ + CV_PROP_RW float maxTimingPatternMismatch; + + /** @brief The maximum allowed percentage of penalty points out of total pins in timing pattern, default 0.4f */ + CV_PROP_RW float maxPenalties; + + /** @brief The maximum allowed relative color mismatch in the timing pattern, default 0.2f*/ + CV_PROP_RW float maxColorsMismatch; + + /** @brief The algorithm find QR codes with almost minimum timing pattern score and minimum size, default 0.9f + * + * The QR code with the minimum "timing pattern score" and minimum "size" is selected as the best QR code. + * If for the current QR code "timing pattern score" * scaleTimingPatternScore < "previous timing pattern score" and "size" < "previous size", then + * current QR code set as the best QR code. */ + CV_PROP_RW float scaleTimingPatternScore; + }; + + /** @brief QR code detector constructor for Aruco-based algorithm. See cv::QRCodeDetectorAruco::Params */ + CV_WRAP explicit QRCodeDetectorAruco(const QRCodeDetectorAruco::Params& params); + + /** @brief Detector parameters getter. See cv::QRCodeDetectorAruco::Params */ + CV_WRAP const QRCodeDetectorAruco::Params& getDetectorParameters() const; + + /** @brief Detector parameters setter. See cv::QRCodeDetectorAruco::Params */ + CV_WRAP QRCodeDetectorAruco& setDetectorParameters(const QRCodeDetectorAruco::Params& params); + + /** @brief Aruco detector parameters are used to search for the finder patterns. */ + CV_WRAP aruco::DetectorParameters getArucoParameters(); + + /** @brief Aruco detector parameters are used to search for the finder patterns. */ + CV_WRAP void setArucoParameters(const aruco::DetectorParameters& params); +}; + //! @} } #include "opencv2/objdetect/detection_based_tracker.hpp" #include "opencv2/objdetect/face.hpp" -#include "opencv2/objdetect/aruco_detector.hpp" #include "opencv2/objdetect/charuco_detector.hpp" #endif diff --git a/modules/objdetect/misc/objc/gen_dict.json b/modules/objdetect/misc/objc/gen_dict.json new file mode 100644 index 0000000000..0311e1e5d6 --- /dev/null +++ b/modules/objdetect/misc/objc/gen_dict.json @@ -0,0 +1,7 @@ +{ + "ManualFuncs" : { + "QRCodeDetectorAruco": { + "getDetectorParameters": { "declaration" : [""], "implementation" : [""] } + } + } +} diff --git a/modules/objdetect/perf/perf_qrcode_pipeline.cpp b/modules/objdetect/perf/perf_qrcode_pipeline.cpp index 6722978b9a..9eac2352e3 100644 --- a/modules/objdetect/perf/perf_qrcode_pipeline.cpp +++ b/modules/objdetect/perf/perf_qrcode_pipeline.cpp @@ -3,6 +3,7 @@ // of this distribution and at http://opencv.org/license.html. #include "perf_precomp.hpp" +#include "../test/test_qr_utils.hpp" namespace opencv_test { @@ -23,7 +24,9 @@ PERF_TEST_P_(Perf_Objdetect_QRCode, detect) std::vector< Point > corners; QRCodeDetector qrcode; TEST_CYCLE() ASSERT_TRUE(qrcode.detect(src, corners)); - SANITY_CHECK(corners); + const int pixels_error = 3; + check_qr(root, name_current_image, "test_images", corners, {}, pixels_error); + SANITY_CHECK_NOTHING(); } #ifdef HAVE_QUIRC @@ -45,48 +48,52 @@ PERF_TEST_P_(Perf_Objdetect_QRCode, decode) decoded_info = qrcode.decode(src, corners, straight_barcode); ASSERT_FALSE(decoded_info.empty()); } - - std::vector decoded_info_uint8_t(decoded_info.begin(), decoded_info.end()); - SANITY_CHECK(decoded_info_uint8_t); - SANITY_CHECK(straight_barcode); - + const int pixels_error = 3; + check_qr(root, name_current_image, "test_images", corners, {decoded_info}, pixels_error); + SANITY_CHECK_NOTHING(); } #endif -typedef ::perf::TestBaseWithParam< std::string > Perf_Objdetect_QRCode_Multi; +typedef ::perf::TestBaseWithParam> Perf_Objdetect_QRCode_Multi; -static inline bool compareCorners(const Point2f& corner1, const Point2f& corner2) { - return corner1.x == corner2.x ? corner1.y < corner2.y : corner1.x < corner2.x; -} +static std::set> disabled_samples = {{"5_qrcodes.png", "aruco_based"}}; PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, detectMulti) { - const std::string name_current_image = GetParam(); + const std::string name_current_image = get<0>(GetParam()); + const std::string method = get<1>(GetParam()); const std::string root = "cv/qrcode/multiple/"; std::string image_path = findDataFile(root + name_current_image); Mat src = imread(image_path); ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; - std::vector corners; - QRCodeDetector qrcode; + std::vector corners; + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (method == "aruco_based") { + qrcode = QRCodeDetectorAruco(); + } TEST_CYCLE() ASSERT_TRUE(qrcode.detectMulti(src, corners)); - sort(corners.begin(), corners.end(), compareCorners); - SANITY_CHECK(corners); -} - -static inline bool compareQR(const pair& v1, const pair& v2) { - return v1.first < v2.first; + const int pixels_error = 7; + check_qr(root, name_current_image, "multiple_images", corners, {}, pixels_error, true); + SANITY_CHECK_NOTHING(); } #ifdef HAVE_QUIRC PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, decodeMulti) { - const std::string name_current_image = GetParam(); + const std::string name_current_image = get<0>(GetParam()); + std::string method = get<1>(GetParam()); const std::string root = "cv/qrcode/multiple/"; std::string image_path = findDataFile(root + name_current_image); Mat src = imread(image_path); ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; - QRCodeDetector qrcode; + if (disabled_samples.find({name_current_image, method}) != disabled_samples.end()) { + throw SkipTestException(name_current_image + " is disabled sample for method " + method); + } + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (method == "aruco_based") { + qrcode = QRCodeDetectorAruco(); + } std::vector corners; ASSERT_TRUE(qrcode.detectMulti(src, corners)); std::vector straight_barcode; @@ -94,26 +101,20 @@ PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, decodeMulti) TEST_CYCLE() { ASSERT_TRUE(qrcode.decodeMulti(src, corners, decoded_info, straight_barcode)); - for(size_t i = 0; i < decoded_info.size(); i++) - { - ASSERT_FALSE(decoded_info[i].empty()); - } + } + ASSERT_TRUE(decoded_info.size() > 0ull); + for(size_t i = 0; i < decoded_info.size(); i++) { + ASSERT_FALSE(decoded_info[i].empty()); } ASSERT_EQ(decoded_info.size(), straight_barcode.size()); - vector > result; - for (size_t i = 0ull; i < decoded_info.size(); i++) { - result.push_back(make_pair(decoded_info[i], straight_barcode[i])); + vector corners_result(corners.size()); + for (size_t i = 0ull; i < corners_result.size(); i++) { + corners_result[i] = corners[i]; } - sort(result.begin(), result.end(), compareQR); - vector > decoded_info_sort; - vector straight_barcode_sort; - for (size_t i = 0ull; i < result.size(); i++) { - vector tmp(result[i].first.begin(), result[i].first.end()); - decoded_info_sort.push_back(tmp); - straight_barcode_sort.push_back(result[i].second); - } - SANITY_CHECK(decoded_info_sort); + const int pixels_error = 7; + check_qr(root, name_current_image, "multiple_images", corners_result, decoded_info, pixels_error, true); + SANITY_CHECK_NOTHING(); } #endif @@ -127,11 +128,10 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode, // version_5_right.jpg DISABLED after tile fix, PR #22025 INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode_Multi, - ::testing::Values( - "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png", - "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png" - ) -); + testing::Combine(testing::Values("2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png", + "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png"), + testing::Values("contours_based", "aruco_based"))); + typedef ::perf::TestBaseWithParam< tuple< std::string, Size > > Perf_Objdetect_Not_QRCode; diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index db619fb441..18b2650d99 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -950,34 +950,99 @@ vector QRDetect::getQuadrilateral(vector angle_list) return result_angle_list; } +struct QRCodeDetectorBase::Impl { + virtual ~Impl() {} + virtual bool detect(InputArray img, OutputArray points) const = 0; + virtual std::string decode(InputArray img, InputArray points, OutputArray straight_qrcode) const = 0; + virtual std::string detectAndDecode(InputArray img, OutputArray points, OutputArray straight_qrcode) const = 0; + virtual bool detectMulti(InputArray img, OutputArray points) const = 0; + virtual bool decodeMulti(InputArray img, InputArray points, std::vector& decoded_info, + OutputArrayOfArrays straight_qrcode) const = 0; + virtual bool detectAndDecodeMulti(InputArray img, std::vector& decoded_info, OutputArray points, + OutputArrayOfArrays straight_qrcode) const = 0; +}; + +QRCodeDetectorBase::QRCodeDetectorBase() {} + +bool QRCodeDetectorBase::detect(InputArray img, OutputArray points) const { + CV_Assert(p); + return p->detect(img, points); +} + +std::string QRCodeDetectorBase::decode(InputArray img, InputArray points, OutputArray straight_qrcode) const { + CV_Assert(p); + return p->decode(img, points, straight_qrcode); +} -struct QRCodeDetector::Impl +std::string QRCodeDetectorBase::detectAndDecode(InputArray img, OutputArray points, OutputArray straight_qrcode) const { + CV_Assert(p); + return p->detectAndDecode(img, points, straight_qrcode); +} + +bool QRCodeDetectorBase::detectMulti(InputArray img, OutputArray points) const { + CV_Assert(p); + return p->detectMulti(img, points); +} + +bool QRCodeDetectorBase::decodeMulti(InputArray img, InputArray points, std::vector& decoded_info, + OutputArrayOfArrays straight_qrcode) const { + CV_Assert(p); + return p->decodeMulti(img, points, decoded_info, straight_qrcode); +} + +bool QRCodeDetectorBase::detectAndDecodeMulti(InputArray img, std::vector& decoded_info, + OutputArray points, OutputArrayOfArrays straight_qrcode) const { + CV_Assert(p); + return p->detectAndDecodeMulti(img, decoded_info, points, straight_qrcode); +} + +struct ImplContour : public QRCodeDetectorBase::Impl { public: - Impl() { epsX = 0.2; epsY = 0.1; } - ~Impl() {} + ImplContour(): epsX(0.2), epsY(0.1) {} double epsX, epsY; - vector> alignmentMarkers; - vector updateQrCorners; + mutable vector> alignmentMarkers; + mutable vector updateQrCorners; bool useAlignmentMarkers = true; + + bool detect(InputArray in, OutputArray points) const override; + std::string decode(InputArray img, InputArray points, OutputArray straight_qrcode) const override; + std::string detectAndDecode(InputArray img, OutputArray points, OutputArray straight_qrcode) const override; + + bool detectMulti(InputArray img, OutputArray points) const override; + bool decodeMulti(InputArray img, InputArray points, std::vector& decoded_info, + OutputArrayOfArrays straight_qrcode) const override; + bool detectAndDecodeMulti(InputArray img, std::vector& decoded_info, OutputArray points, + OutputArrayOfArrays straight_qrcode) const override; + + String decodeCurved(InputArray in, InputArray points, OutputArray straight_qrcode); + + std::string detectAndDecodeCurved(InputArray in, OutputArray points, OutputArray straight_qrcode); }; -QRCodeDetector::QRCodeDetector() : p(new Impl) {} +QRCodeDetector::QRCodeDetector() { + p = makePtr(); +} -QRCodeDetector::~QRCodeDetector() {} +QRCodeDetector& QRCodeDetector::setEpsX(double epsX) { + std::dynamic_pointer_cast(p)->epsX = epsX; + return *this; +} -void QRCodeDetector::setEpsX(double epsX) { p->epsX = epsX; } -void QRCodeDetector::setEpsY(double epsY) { p->epsY = epsY; } +QRCodeDetector& QRCodeDetector::setEpsY(double epsY) { + std::dynamic_pointer_cast(p)->epsY = epsY; + return *this; +} -bool QRCodeDetector::detect(InputArray in, OutputArray points) const +bool ImplContour::detect(InputArray in, OutputArray points) const { Mat inarr; if (!checkQRInputImage(in, inarr)) return false; QRDetect qrdet; - qrdet.init(inarr, p->epsX, p->epsY); + qrdet.init(inarr, epsX, epsY); if (!qrdet.localization()) { return false; } if (!qrdet.computeTransformationPoints()) { return false; } vector pnts2f = qrdet.getTransformationPoints(); @@ -2789,9 +2854,7 @@ QRDecode::QRDecode(bool _useAlignmentMarkers): test_perspective_size(0.f) {} -std::string QRCodeDetector::decode(InputArray in, InputArray points, - OutputArray straight_qrcode) -{ +std::string ImplContour::decode(InputArray in, InputArray points, OutputArray straight_qrcode) const { Mat inarr; if (!checkQRInputImage(in, inarr)) return std::string(); @@ -2801,7 +2864,7 @@ std::string QRCodeDetector::decode(InputArray in, InputArray points, CV_Assert(src_points.size() == 4); CV_CheckGT(contourArea(src_points), 0.0, "Invalid QR code source points"); - QRDecode qrdec(p->useAlignmentMarkers); + QRDecode qrdec(useAlignmentMarkers); qrdec.init(inarr, src_points); bool ok = qrdec.straightDecodingProcess(); @@ -2815,14 +2878,18 @@ std::string QRCodeDetector::decode(InputArray in, InputArray points, qrdec.getStraightBarcode().convertTo(straight_qrcode, CV_8UC1); } if (ok && !decoded_info.empty()) { - p->alignmentMarkers = {qrdec.alignment_coords}; - p->updateQrCorners = qrdec.getOriginalPoints(); + alignmentMarkers = {qrdec.alignment_coords}; + updateQrCorners = qrdec.getOriginalPoints(); } return ok ? decoded_info : std::string(); } -cv::String QRCodeDetector::decodeCurved(InputArray in, InputArray points, - OutputArray straight_qrcode) +String QRCodeDetector::decodeCurved(InputArray in, InputArray points, OutputArray straight_qrcode) { + CV_Assert(p); + return std::dynamic_pointer_cast(p)->decodeCurved(in, points, straight_qrcode); +} + +String ImplContour::decodeCurved(InputArray in, InputArray points, OutputArray straight_qrcode) { Mat inarr; if (!checkQRInputImage(in, inarr)) @@ -2833,7 +2900,7 @@ cv::String QRCodeDetector::decodeCurved(InputArray in, InputArray points, CV_Assert(src_points.size() == 4); CV_CheckGT(contourArea(src_points), 0.0, "Invalid QR code source points"); - QRDecode qrdec(p->useAlignmentMarkers); + QRDecode qrdec(useAlignmentMarkers); qrdec.init(inarr, src_points); bool ok = qrdec.curvedDecodingProcess(); @@ -2851,10 +2918,7 @@ cv::String QRCodeDetector::decodeCurved(InputArray in, InputArray points, return ok ? decoded_info : std::string(); } -std::string QRCodeDetector::detectAndDecode(InputArray in, - OutputArray points_, - OutputArray straight_qrcode) -{ +std::string ImplContour::detectAndDecode(InputArray in, OutputArray points_, OutputArray straight_qrcode) const { Mat inarr; if (!checkQRInputImage(in, inarr)) { @@ -2874,9 +2938,14 @@ std::string QRCodeDetector::detectAndDecode(InputArray in, return decoded_info; } -std::string QRCodeDetector::detectAndDecodeCurved(InputArray in, - OutputArray points_, - OutputArray straight_qrcode) +std::string QRCodeDetector::detectAndDecodeCurved(InputArray in, OutputArray points, + OutputArray straight_qrcode) { + CV_Assert(p); + return std::dynamic_pointer_cast(p)->detectAndDecodeCurved(in, points, straight_qrcode); +} + +std::string ImplContour::detectAndDecodeCurved(InputArray in, OutputArray points_, + OutputArray straight_qrcode) { Mat inarr; if (!checkQRInputImage(in, inarr)) @@ -3817,31 +3886,28 @@ bool QRDetectMulti::computeTransformationPoints(const size_t cur_ind) return true; } -bool QRCodeDetector::detectMulti(InputArray in, OutputArray points) const -{ - Mat inarr; - if (!checkQRInputImage(in, inarr)) - { +bool ImplContour::detectMulti(InputArray in, OutputArray points) const { + Mat gray; + if (!checkQRInputImage(in, gray)) { points.release(); return false; } - + vector result; QRDetectMulti qrdet; - qrdet.init(inarr, p->epsX, p->epsY); - if (!qrdet.localization()) - { + qrdet.init(gray, epsX, epsY); + if (!qrdet.localization()) { points.release(); return false; } - vector< vector< Point2f > > pnts2f = qrdet.getTransformationPoints(); - vector trans_points; + vector > pnts2f = qrdet.getTransformationPoints(); for(size_t i = 0; i < pnts2f.size(); i++) for(size_t j = 0; j < pnts2f[i].size(); j++) - trans_points.push_back(pnts2f[i][j]); - - updatePointsResult(points, trans_points); - - return true; + result.push_back(pnts2f[i][j]); + if (result.size() >= 4) { + updatePointsResult(points, result); + return true; + } + return false; } class ParallelDecodeProcess : public ParallelLoopBody @@ -3902,7 +3968,7 @@ private: }; -bool QRCodeDetector::decodeMulti( +bool ImplContour::decodeMulti( InputArray img, InputArray points, CV_OUT std::vector& decoded_info, @@ -3926,7 +3992,7 @@ bool QRCodeDetector::decodeMulti( } } CV_Assert(src_points.size() > 0); - vector qrdec(src_points.size(), p->useAlignmentMarkers); + vector qrdec(src_points.size(), useAlignmentMarkers); vector straight_barcode(src_points.size()); vector info(src_points.size()); ParallelDecodeProcess parallelDecodeProcess(inarr, qrdec, info, straight_barcode, src_points); @@ -3957,12 +4023,12 @@ bool QRCodeDetector::decodeMulti( { decoded_info.push_back(info[i]); } - p->alignmentMarkers.resize(src_points.size()); - p->updateQrCorners.resize(src_points.size()*4ull); + alignmentMarkers.resize(src_points.size()); + updateQrCorners.resize(src_points.size()*4ull); for (size_t i = 0ull; i < src_points.size(); i++) { - p->alignmentMarkers[i] = qrdec[i].alignment_coords; + alignmentMarkers[i] = qrdec[i].alignment_coords; for (size_t j = 0ull; j < 4ull; j++) - p->updateQrCorners[i*4ull+j] = qrdec[i].getOriginalPoints()[j] * qrdec[i].coeff_expansion; + updateQrCorners[i*4ull+j] = qrdec[i].getOriginalPoints()[j] * qrdec[i].coeff_expansion; } if (!decoded_info.empty()) return true; @@ -3970,7 +4036,7 @@ bool QRCodeDetector::decodeMulti( return false; } -bool QRCodeDetector::detectAndDecodeMulti( +bool ImplContour::detectAndDecodeMulti( InputArray img, CV_OUT std::vector& decoded_info, OutputArray points_, @@ -3994,13 +4060,537 @@ bool QRCodeDetector::detectAndDecodeMulti( updatePointsResult(points_, points); decoded_info.clear(); ok = decodeMulti(inarr, points, decoded_info, straight_qrcode); - updatePointsResult(points_, p->updateQrCorners); + updatePointsResult(points_, updateQrCorners); return ok; } -void QRCodeDetector::setUseAlignmentMarkers(bool useAlignmentMarkers) { - p->useAlignmentMarkers = useAlignmentMarkers; +QRCodeDetector& QRCodeDetector::setUseAlignmentMarkers(bool useAlignmentMarkers) { + (std::dynamic_pointer_cast)(p)->useAlignmentMarkers = useAlignmentMarkers; + return *this; +} + +QRCodeDetectorAruco::Params::Params() { + minModuleSizeInPyramid = 4.f; + maxRotation = (float)CV_PI/12.f; + maxModuleSizeMismatch = 1.75f; + maxTimingPatternMismatch = 2.f; + maxPenalties = 0.4f; + maxColorsMismatch = 0.2f; + scaleTimingPatternScore = 0.9f; +} + +namespace { + +struct FinderPatternInfo { + FinderPatternInfo() {} + + FinderPatternInfo(const vector& patternPoints): points(patternPoints) { + float minSin = 1.f; + for (int i = 0; i < 4; i++) { + center += points[i]; + const Point2f side = points[i]-points[(i+1) % 4]; + const float lenSide = sqrt(normL2Sqr(side)); + minSin = min(minSin, abs(side.y) / lenSide); + moduleSize += lenSide; + } + moduleSize /= (4.f * 7.f); // 4 sides, 7 modules in one side + center /= 4.f; + minQrAngle = asin(minSin); + } + + enum TypePattern { + CENTER, + RIGHT, + BOTTOM, + NONE + }; + + void setType(const TypePattern& _typePattern, const Point2f& centerQR) { + typePattern = _typePattern; + float bestLen = normL2Sqr(centerQR - points[0]); + int id = 0; + for (int i = 1; i < 4; i++) { + float len = normL2Sqr(centerQR - points[i]); + if (len < bestLen) { + bestLen = len; + id = i; + } + } + innerCornerId = id; + } + + Point2f getDirectionTo(const TypePattern& other) const { + Point2f res = points[innerCornerId]; + if (typePattern == TypePattern::CENTER) { + if (other == TypePattern::RIGHT) { + res -= points[(innerCornerId + 1) % 4]; + res = 0.5f*(res + points[(innerCornerId + 3) % 4] - points[(innerCornerId + 2) % 4]); + } + else if (other == TypePattern::BOTTOM) { + res -= points[(innerCornerId + 3) % 4]; + res = 0.5f*(res + points[(innerCornerId + 1) % 4] - points[(innerCornerId + 2) % 4]); + } + } + else if (typePattern == TypePattern::RIGHT && other == TypePattern::CENTER) { + res = res - points[(innerCornerId + 3) % 4]; + res = 0.5f*(res + points[(innerCornerId + 1) % 4] - points[(innerCornerId + 2) % 4]); + } + else if (typePattern == TypePattern::BOTTOM && other == TypePattern::CENTER) { + res = res - points[(innerCornerId + 1) % 4]; + res = 0.5f*(res + points[(innerCornerId + 3) % 4] - points[(innerCornerId + 2) % 4]); + } + return res; + } + + bool checkTriangleAngle(const FinderPatternInfo& patternRight, const FinderPatternInfo& patternBottom, const float length2Vec) { + // check the triangle angle btw right & center & bootom sides of QR code + // the triangle angle shoud be between 30 and 150 degrees + // abs(pi/2 - triangle_angle) should be less 60 degrees + const float angle = abs((float)CV_PI/2.f - acos((center - patternRight.center).dot((center - patternBottom.center)) / length2Vec)); + + const float maxTriangleDeltaAngle = (float)CV_PI / 3.f; + if (angle > maxTriangleDeltaAngle) { + return false; + } + return true; + } + + bool checkAngle(const FinderPatternInfo& other, const float maxRotation) { + Point2f toOther = getDirectionTo(other.typePattern); + Point2f toThis = other.getDirectionTo(typePattern); + const float cosAngle = getCosAngle(toOther, toThis); + if (cosAngle < 0.f && (CV_PI - acos(cosAngle)) / 2.f < maxRotation) { + const float angleCenter = max(acos(getCosAngle(toOther, other.center - center)), acos(getCosAngle(toThis, center - other.center))); + if (angleCenter < maxRotation) + return true; + } + return false; + } + + static float getCosAngle(const Point2f& vec1, const Point2f& vec2) { + float cosAngle = vec1.dot(vec2) / (sqrt(normL2Sqr(vec1)) * sqrt(normL2Sqr(vec2))); + cosAngle = std::max(-1.f, cosAngle); + cosAngle = std::min(1.f, cosAngle); + return cosAngle; + } + + pair getQRCorner() const { + if (typePattern == TypePattern::CENTER) { + int id = (innerCornerId + 2) % 4; + return std::make_pair(id, points[id]); + } + else if (typePattern != TypePattern::NONE) { + int id = (innerCornerId + 2) % 4; + return std::make_pair(id, points[id]); + } + return std::make_pair(-1, Point2f()); + } + + pair getCornerForIntersection() const { + if (typePattern == TypePattern::RIGHT) { + int id = (innerCornerId + 3) % 4; + return std::make_pair(id, points[id]); + } + else if (typePattern == TypePattern::BOTTOM) { + int id = (innerCornerId + 1) % 4; + return std::make_pair(id, points[id]); + } + return std::make_pair(-1, Point2f()); + } + + Point2f getTimingStart(TypePattern direction) const { + const float timingStartPosition = .5f; + const float patternLength = 7.f; + Point2f start = points[innerCornerId]*((patternLength - timingStartPosition)/patternLength); + if (typePattern == TypePattern::CENTER && direction == TypePattern::RIGHT) { + start += points[(innerCornerId + 3) % 4]*(timingStartPosition/patternLength); + } + else if (typePattern == TypePattern::CENTER && direction == TypePattern::BOTTOM) { + start += points[(innerCornerId + 1) % 4]*(timingStartPosition/patternLength); + } + else if (typePattern == TypePattern::RIGHT && direction == TypePattern::CENTER) { + start += points[(innerCornerId + 1) % 4]*(timingStartPosition/patternLength); + } + else if (typePattern == TypePattern::BOTTOM && direction == TypePattern::CENTER) { + start += points[(innerCornerId + 3) % 4]*(timingStartPosition/patternLength); + } + return start + getDirectionTo(direction)/(patternLength*2.f); + } + + // return total white+black modules in timing pattern, total white modules, penaltyPoints + Point3i getTimingPatternScore(const Point2f& start, const Point2f& end, Mat &img, const float maxTimingPatternMismatch) const { + Rect imageRect(Point(), img.size()); + int penaltyPoints = 0; + int colorCounters[2] = {0, 0}; + if (imageRect.contains(Point(cvRound(end.x), cvRound(end.y)))) { + LineIterator lineIterator(start, end); + uint8_t prevValue = img.at(lineIterator.pos()); + + vector vec = {lineIterator.pos()}; + + // the starting position in the timing pattern is the white module white module next to the finder pattern. + bool whiteColor = true; + lineIterator++; + colorCounters[whiteColor]++; + + for(int i = 1; i < lineIterator.count; i++, ++lineIterator) { + const uint8_t value = img.at(lineIterator.pos()); + if (prevValue != value) { + const float dist = sqrt(normL2Sqr((Point2f)(vec.back()-lineIterator.pos()))); + // check long and short lines in timing pattern + const float relativeDiff = max(moduleSize, dist)/min(moduleSize, dist); + if (relativeDiff > maxTimingPatternMismatch) { + if (dist < moduleSize || relativeDiff < maxTimingPatternMismatch*8.f) + penaltyPoints++; + else + penaltyPoints += cvRound(relativeDiff); + } + vec.push_back(lineIterator.pos()); + prevValue = value; + whiteColor ^= true; + colorCounters[whiteColor]++; + } + } + } + return Point3i(colorCounters[0] + colorCounters[1], colorCounters[1], penaltyPoints); + } + + FinderPatternInfo& operator*=(const float scale) { + moduleSize *= scale; + center *= scale; + for (auto& point: points) + point *= scale; + return *this; + } + + float moduleSize = 0.f; + + // Index of inner QR corner. + // The inner corner is the corner closest to the center of the QR code. + int innerCornerId = 0; + + float minQrAngle = 0.f; + TypePattern typePattern = NONE; + + Point2f center; + vector points; +}; + +struct QRCode { + QRCode() {} + + QRCode(const FinderPatternInfo& _centerPattern, const FinderPatternInfo& _rightPattern, const FinderPatternInfo& _bottomPattern, + Point2f _center, float dist): centerPattern(_centerPattern), rightPattern(_rightPattern), bottomPattern(_bottomPattern), + center(_center), distance(dist) { + moduleSize = (centerPattern.moduleSize + rightPattern.moduleSize + bottomPattern.moduleSize) / 3.f; + } + + vector getQRCorners() const { + Point2f a1 = rightPattern.getQRCorner().second; + Point2f a2 = rightPattern.getCornerForIntersection().second; + + Point2f b1 = bottomPattern.getQRCorner().second; + Point2f b2 = bottomPattern.getCornerForIntersection().second; + + Point2f rightBottom = intersectionLines(a1, a2, b1, b2); + + return {centerPattern.getQRCorner().second, rightPattern.getQRCorner().second, rightBottom, bottomPattern.getQRCorner().second}; + } + + static QRCode checkCompatibilityPattern(const FinderPatternInfo &_pattern1, const FinderPatternInfo& _pattern2, const FinderPatternInfo& _pattern3, + Point3i& index, const QRCodeDetectorAruco::Params& qrDetectorParameters) { + FinderPatternInfo pattern1 = _pattern1, pattern2 = _pattern2, pattern3 = _pattern3; + Point2f centerQR; + float distance = std::numeric_limits::max(); + + if (abs(pattern1.minQrAngle - pattern2.minQrAngle) > qrDetectorParameters.maxRotation || + abs(pattern1.minQrAngle - pattern3.minQrAngle) > qrDetectorParameters.maxRotation) // check maxRotation + return QRCode(pattern1, pattern2, pattern3, centerQR, distance); + if (max(pattern1.moduleSize, pattern2.moduleSize) / min(pattern1.moduleSize, pattern2.moduleSize) > qrDetectorParameters.maxModuleSizeMismatch || + max(pattern1.moduleSize, pattern3.moduleSize) / min(pattern1.moduleSize, pattern3.moduleSize) > qrDetectorParameters.maxModuleSizeMismatch) + return QRCode(pattern1, pattern2, pattern3, centerQR, distance); + // QR code: + // center right + // 1 ________ 2 + // |_| |_| + // | / | + // | / | + // | / | + // |_ / | + // |_|______| + // 4 + // bottom + + // sides length check + const float side1 = sqrt(normL2Sqr(pattern1.center - pattern2.center)); + const float side2 = sqrt(normL2Sqr(pattern1.center - pattern3.center)); + const float side3 = sqrt(normL2Sqr(pattern2.center - pattern3.center)); + std::array sides = {side1, side2, side3}; + std::sort(sides.begin(), sides.end()); + // check sides diff + if (sides[1] / sides[0] < qrDetectorParameters.maxModuleSizeMismatch) { + // find center pattern + if (side1 > side2 && side1 > side3) { // centerPattern is pattern3 + std::swap(pattern3, pattern1); // now pattern1 is centerPattern + std::swap(index.x, index.z); + } + else if (side2 > side1 && side2 > side3) { // centerPattern is pattern2 + std::swap(pattern2, pattern1); // now pattern1 is centerPattern + std::swap(index.x, index.y); + } + // now pattern1 is centerPattern + centerQR = (pattern2.center + pattern3.center) / 2.f; + pattern1.setType(FinderPatternInfo::TypePattern::CENTER, centerQR); + // check triangle angle + if (pattern1.checkTriangleAngle(pattern2, pattern3, sides[0]*sides[1]) == false) + return QRCode(pattern1, pattern2, pattern3, centerQR, distance); + // check that pattern2 is right + pattern2.setType(FinderPatternInfo::TypePattern::RIGHT, centerQR); + bool ok = pattern1.checkAngle(pattern2, qrDetectorParameters.maxRotation); + if (!ok) { + // check that pattern3 is right + pattern3.setType(FinderPatternInfo::TypePattern::RIGHT, centerQR); + ok = pattern1.checkAngle(pattern3, qrDetectorParameters.maxRotation); + if (ok) { + std::swap(pattern3, pattern2); // now pattern2 is rightPattern + std::swap(index.y, index.z); + } + } + if (ok) { + // check that pattern3 is bottom + pattern3.setType(FinderPatternInfo::TypePattern::BOTTOM, centerQR); + ok = pattern1.checkAngle(pattern3, qrDetectorParameters.maxRotation); + if (ok) { + // intersection check + Point2f c1 = intersectionLines(pattern1.getQRCorner().second, pattern1.points[pattern1.innerCornerId], + pattern2.getQRCorner().second, pattern2.points[pattern2.innerCornerId]); + Point2f c2 = intersectionLines(pattern1.getQRCorner().second, pattern1.points[pattern1.innerCornerId], + pattern3.getQRCorner().second, pattern3.points[pattern3.innerCornerId]); + const float centerDistance = sqrt(normL2Sqr(c1 - c2)); + distance = (sides[0] + sides[1] + centerDistance)*(sides[1] / sides[0]); + } + } + } + QRCode qrcode(pattern1, pattern2, pattern3, centerQR, distance); + return qrcode; + } + + int calculateScoreByTimingPattern(Mat &img, const QRCodeDetectorAruco::Params& params) { + const int minModulesInTimingPattern = 4; + + const Point3i v1 = centerPattern.getTimingPatternScore(rightPattern.getTimingStart(FinderPatternInfo::CENTER), + centerPattern.getTimingStart(FinderPatternInfo::RIGHT), img, + params.maxTimingPatternMismatch); + + if ((float)v1.z > params.maxPenalties*v1.x || v1.x <= minModulesInTimingPattern || abs(v1.y / (float)v1.x - 0.5f) > params.maxColorsMismatch) + return std::numeric_limits::max(); + + const Point3i v2 = centerPattern.getTimingPatternScore(bottomPattern.getTimingStart(FinderPatternInfo::CENTER), + centerPattern.getTimingStart(FinderPatternInfo::BOTTOM), img, + params.maxTimingPatternMismatch); + + + if ((float)v2.z > params.maxPenalties*v2.x || v2.x <= minModulesInTimingPattern || abs(v2.y / (float)v2.x - 0.5f) > params.maxColorsMismatch) + return std::numeric_limits::max(); + + // TODO: add v1, v2 check, add "y" checks + float numModules = (sqrt(normL2Sqr((centerPattern.getQRCorner().second - rightPattern.getQRCorner().second)))*0.5f + + sqrt(normL2Sqr((centerPattern.getQRCorner().second - bottomPattern.getQRCorner().second))*0.5f)) / moduleSize; + + const int sizeDelta = abs(cvRound(numModules) - (14 + v1.z < v2.z ? v1.x : v2.x)); + const int colorDelta = abs(v1.x - v1.y - v1.y) + abs(v2.x - v2.y - v2.y); + const int score = v1.z + v2.z + sizeDelta + colorDelta; + return score; + } + + QRCode& operator*=(const float scale) { + centerPattern *= scale; + rightPattern *= scale; + bottomPattern *= scale; + center *= scale; + moduleSize *= scale; + return *this; + } + + FinderPatternInfo centerPattern; + FinderPatternInfo rightPattern; + FinderPatternInfo bottomPattern; + Point2f center; + float distance = std::numeric_limits::max(); + int timingPatternScore = std::numeric_limits::max(); + float moduleSize = 0.f; +}; + +} // namespace + +static +vector analyzeFinderPatterns(const vector > &corners, const Mat& img, + const QRCodeDetectorAruco::Params& qrDetectorParameters) { + vector qrCodes; + vector patterns; + if (img.empty()) + return qrCodes; + float maxModuleSize = 0.f; + for (size_t i = 0ull; i < corners.size(); i++) { + FinderPatternInfo pattern = FinderPatternInfo(corners[i]); + // TODO: improve thinning Aruco markers + bool isUniq = true; + for (const FinderPatternInfo& tmp : patterns) { + Point2f dist = pattern.center - tmp.center; + if (max(abs(dist.x), abs(dist.y)) < 3.f * tmp.moduleSize) { + isUniq = false; + break; + } + } + if (isUniq) { + patterns.push_back(pattern); + maxModuleSize = max(maxModuleSize, patterns.back().moduleSize); + } + } + const int threshold = cvRound(qrDetectorParameters.minModuleSizeInPyramid * 12.5f) + + (cvRound(qrDetectorParameters.minModuleSizeInPyramid * 12.5f) % 2 ? 0 : 1); + int maxLevelPyramid = 0; + while (maxModuleSize / 2.f > qrDetectorParameters.minModuleSizeInPyramid) { + maxLevelPyramid++; + maxModuleSize /= 2.f; + } + vector pyramid; + buildPyramid(img, pyramid, maxLevelPyramid); + // TODO: ADAPTIVE_THRESH_GAUSSIAN_C vs ADAPTIVE_THRESH_MEAN_C + for (Mat& pyr: pyramid) { + adaptiveThreshold(pyr, pyr, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, threshold, -1); + } + + for (size_t i = 0ull; i < patterns.size(); i++) { + QRCode qrCode; + int indexes[3] = {0}; + for (size_t j = i + 1ull; j < patterns.size(); j++) { + for (size_t k = j + 1ull; k < patterns.size(); k++) { + Point3i index((int)i, (int)j, (int)k); + QRCode tmp = QRCode::checkCompatibilityPattern(patterns[i], patterns[j], patterns[k], index, qrDetectorParameters); + if (tmp.distance != std::numeric_limits::max()) { + int levelPyramid = 0; + QRCode qrCopy = tmp; + while (tmp.moduleSize / 2.f > qrDetectorParameters.minModuleSizeInPyramid) { + tmp *= 0.5f; + levelPyramid++; + } + qrCopy.timingPatternScore = tmp.calculateScoreByTimingPattern(pyramid[levelPyramid], qrDetectorParameters); + if (qrCopy.timingPatternScore != std::numeric_limits::max() && + qrCopy.timingPatternScore * qrDetectorParameters.scaleTimingPatternScore < (float)qrCode.timingPatternScore + && qrCopy.distance < qrCode.distance) + { + qrCode = qrCopy; + indexes[0] = (int)i; + indexes[1] = (int)j; + indexes[2] = (int)k; + } + } + } + } + if (qrCode.distance != std::numeric_limits::max()) { + qrCodes.push_back(qrCode); + std::swap(patterns[indexes[2]], patterns.back()); + patterns.pop_back(); + std::swap(patterns[indexes[1]], patterns.back()); + patterns.pop_back(); + std::swap(patterns[indexes[0]], patterns.back()); + patterns.pop_back(); + i--; + } + } + return qrCodes; +} + +struct PimplQRAruco : public ImplContour { + QRCodeDetectorAruco::Params qrParams; + aruco::ArucoDetector arucoDetector; + aruco::DetectorParameters arucoParams; + + PimplQRAruco() { + Mat bits = Mat::ones(Size(5, 5), CV_8UC1); + Mat(bits, Rect(1, 1, 3, 3)).setTo(Scalar(0)); + Mat byteList = aruco::Dictionary::getByteListFromBits(bits); + aruco::Dictionary dictionary = aruco::Dictionary(byteList, 5, 4); + arucoParams.minMarkerPerimeterRate = 0.02; + arucoDetector = aruco::ArucoDetector(dictionary, arucoParams); + } + + bool detectMulti(InputArray in, OutputArray points) const override { + Mat gray; + if (!checkQRInputImage(in, gray)) { + points.release(); + return false; + } + vector result; + vector > corners; + vector ids; + arucoDetector.detectMarkers(gray, corners, ids); + if (corners.size() >= 3ull) { + vector qrCodes = analyzeFinderPatterns(corners, gray.clone(), qrParams); + if (qrCodes.size() == 0ull) + return false; + for (auto& qr : qrCodes) { + for (Point2f& corner : qr.getQRCorners()) { + result.push_back(corner); + } + } + } + if (result.size() >= 4) { + updatePointsResult(points, result); + return true; + } + return false; + } + + bool detect(InputArray img, OutputArray points) const override { + vector corners, result; + bool flag = detectMulti(img, corners); + CV_Assert((int)corners.size() % 4 == 0); + + Point2f imageCenter(((float)img.cols())/2.f, ((float)img.rows())/2.f); + size_t minQrId = 0ull; + float minDist = std::numeric_limits::max(); + for (size_t i = 0ull; i < corners.size(); i += 4ull) { + Point2f qrCenter((corners[i] + corners[i+1ull] + corners[i+2ull] + corners[i+3ull]) / 4.f); + float dist = sqrt(normL2Sqr(qrCenter - imageCenter)); + if (dist < minDist) { + minQrId = i; + minDist = dist; + } + } + if (flag) { + result = {corners[minQrId], corners[minQrId+1ull], corners[minQrId+2ull], corners[minQrId+3ull]}; + updatePointsResult(points, result); + } + return flag; + } +}; + +QRCodeDetectorAruco::QRCodeDetectorAruco() { + p = makePtr(); +} + +QRCodeDetectorAruco::QRCodeDetectorAruco(const QRCodeDetectorAruco::Params& params) { + p = makePtr(); + std::dynamic_pointer_cast(p)->qrParams = params; +} + +const QRCodeDetectorAruco::Params& QRCodeDetectorAruco::getDetectorParameters() const { + return std::dynamic_pointer_cast(p)->qrParams; } +QRCodeDetectorAruco& QRCodeDetectorAruco::setDetectorParameters(const QRCodeDetectorAruco::Params& params) { + std::dynamic_pointer_cast(p)->qrParams = params; + return *this; +} + +aruco::DetectorParameters QRCodeDetectorAruco::getArucoParameters() { + return std::dynamic_pointer_cast(p)->arucoParams; +} + +void QRCodeDetectorAruco::setArucoParameters(const aruco::DetectorParameters& params) { + std::dynamic_pointer_cast(p)->arucoParams = params; +} } // namespace diff --git a/modules/objdetect/test/test_qr_utils.hpp b/modules/objdetect/test/test_qr_utils.hpp new file mode 100644 index 0000000000..cfbe1a5078 --- /dev/null +++ b/modules/objdetect/test/test_qr_utils.hpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +#include "test_precomp.hpp" + +namespace opencv_test { + +static inline +void check_qr(const string& root, const string& name_current_image, const string& config_name, + const std::vector& corners, + const std::vector& decoded_info, const int max_pixel_error, + bool isMulti = false) { + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + FileNode images_list = file_config[config_name]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; + for (size_t index = 0; index < images_count; index++) { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) { + if (isMulti) { + for(int j = 0; j < int(corners.size()); j += 4) { + bool ok = false; + for (int k = 0; k < int(corners.size() / 4); k++) { + int count_eq_points = 0; + for (int i = 0; i < 4; i++) { + int x = config["x"][k][i]; + int y = config["y"][k][i]; + if(((abs(corners[j + i].x - x)) <= max_pixel_error) && ((abs(corners[j + i].y - y)) <= max_pixel_error)) + count_eq_points++; + } + if (count_eq_points == 4) { + ok = true; + break; + } + } + EXPECT_TRUE(ok); + } + } + else { + for (int i = 0; i < (int)corners.size(); i++) { + int x = config["x"][i]; + int y = config["y"][i]; + EXPECT_NEAR(x, corners[i].x, max_pixel_error); + EXPECT_NEAR(y, corners[i].y, max_pixel_error); + } + } +#ifdef HAVE_QUIRC + if (decoded_info.size() == 0ull) + return; + if (isMulti) { + size_t count_eq_info = 0; + for(int i = 0; i < int(decoded_info.size()); i++) { + for(int j = 0; j < int(decoded_info.size()); j++) { + std::string original_info = config["info"][j]; + if(original_info == decoded_info[i]) { + count_eq_info++; + break; + } + } + } + EXPECT_EQ(decoded_info.size(), count_eq_info); + } + else { + std::string original_info = config["info"]; + EXPECT_EQ(decoded_info[0], original_info); + } +#endif + return; // done + } + } + FAIL() << "Not found results for '" << name_current_image << "' image in config file:" << dataset_config << + "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data.\n"; +} + +} diff --git a/modules/objdetect/test/test_qrcode.cpp b/modules/objdetect/test/test_qrcode.cpp index 0868932eab..609cc6dd2e 100644 --- a/modules/objdetect/test/test_qrcode.cpp +++ b/modules/objdetect/test/test_qrcode.cpp @@ -3,6 +3,7 @@ // of this distribution and at http://opencv.org/license.html. #include "test_precomp.hpp" +#include "test_qr_utils.hpp" #include "opencv2/imgproc.hpp" namespace opencv_test { namespace { @@ -33,6 +34,8 @@ std::string qrcode_images_multiple[] = { "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png" }; +static std::set> disabled_samples = {{"5_qrcodes.png", "aruco_based"}}; + //#define UPDATE_QRCODE_TEST_DATA #ifdef UPDATE_QRCODE_TEST_DATA @@ -262,43 +265,7 @@ TEST_P(Objdetect_QRCode, regression) #else ASSERT_TRUE(qrcode.detect(src, corners)); #endif - - const std::string dataset_config = findDataFile(root + "dataset_config.json"); - FileStorage file_config(dataset_config, FileStorage::READ); - ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; - { - FileNode images_list = file_config["test_images"]; - size_t images_count = static_cast(images_list.size()); - ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; - - for (size_t index = 0; index < images_count; index++) - { - FileNode config = images_list[(int)index]; - std::string name_test_image = config["image_name"]; - if (name_test_image == name_current_image) - { - for (int i = 0; i < 4; i++) - { - int x = config["x"][i]; - int y = config["y"][i]; - EXPECT_NEAR(x, corners[i].x, pixels_error); - EXPECT_NEAR(y, corners[i].y, pixels_error); - } - -#ifdef HAVE_QUIRC - std::string original_info = config["info"]; - EXPECT_EQ(decoded_info, original_info); -#endif - - return; // done - } - } - std::cerr - << "Not found results for '" << name_current_image - << "' image in config file:" << dataset_config << std::endl - << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." - << std::endl; - } + check_qr(root, name_current_image, "test_images", corners, {decoded_info}, pixels_error); } typedef testing::TestWithParam< std::string > Objdetect_QRCode_Close; @@ -329,43 +296,7 @@ TEST_P(Objdetect_QRCode_Close, regression) #else ASSERT_TRUE(qrcode.detect(barcode, corners)); #endif - - const std::string dataset_config = findDataFile(root + "dataset_config.json"); - FileStorage file_config(dataset_config, FileStorage::READ); - ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; - { - FileNode images_list = file_config["close_images"]; - size_t images_count = static_cast(images_list.size()); - ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; - - for (size_t index = 0; index < images_count; index++) - { - FileNode config = images_list[(int)index]; - std::string name_test_image = config["image_name"]; - if (name_test_image == name_current_image) - { - for (int i = 0; i < 4; i++) - { - int x = config["x"][i]; - int y = config["y"][i]; - EXPECT_NEAR(x, corners[i].x, pixels_error); - EXPECT_NEAR(y, corners[i].y, pixels_error); - } - -#ifdef HAVE_QUIRC - std::string original_info = config["info"]; - EXPECT_EQ(decoded_info, original_info); -#endif - - return; // done - } - } - std::cerr - << "Not found results for '" << name_current_image - << "' image in config file:" << dataset_config << std::endl - << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." - << std::endl; - } + check_qr(root, name_current_image, "close_images", corners, {decoded_info}, pixels_error); } typedef testing::TestWithParam< std::string > Objdetect_QRCode_Monitor; @@ -396,43 +327,7 @@ TEST_P(Objdetect_QRCode_Monitor, regression) #else ASSERT_TRUE(qrcode.detect(barcode, corners)); #endif - - const std::string dataset_config = findDataFile(root + "dataset_config.json"); - FileStorage file_config(dataset_config, FileStorage::READ); - ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; - { - FileNode images_list = file_config["monitor_images"]; - size_t images_count = static_cast(images_list.size()); - ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; - - for (size_t index = 0; index < images_count; index++) - { - FileNode config = images_list[(int)index]; - std::string name_test_image = config["image_name"]; - if (name_test_image == name_current_image) - { - for (int i = 0; i < 4; i++) - { - int x = config["x"][i]; - int y = config["y"][i]; - EXPECT_NEAR(x, corners[i].x, pixels_error); - EXPECT_NEAR(y, corners[i].y, pixels_error); - } - -#ifdef HAVE_QUIRC - std::string original_info = config["info"]; - EXPECT_EQ(decoded_info, original_info); -#endif - - return; // done - } - } - std::cerr - << "Not found results for '" << name_current_image - << "' image in config file:" << dataset_config << std::endl - << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." - << std::endl; - } + check_qr(root, name_current_image, "monitor_images", corners, {decoded_info}, pixels_error); } typedef testing::TestWithParam< std::string > Objdetect_QRCode_Curved; @@ -458,56 +353,26 @@ TEST_P(Objdetect_QRCode_Curved, regression) #else ASSERT_TRUE(qrcode.detect(src, corners)); #endif - - const std::string dataset_config = findDataFile(root + "dataset_config.json"); - FileStorage file_config(dataset_config, FileStorage::READ); - ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; - { - FileNode images_list = file_config["test_images"]; - size_t images_count = static_cast(images_list.size()); - ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; - - for (size_t index = 0; index < images_count; index++) - { - FileNode config = images_list[(int)index]; - std::string name_test_image = config["image_name"]; - if (name_test_image == name_current_image) - { - for (int i = 0; i < 4; i++) - { - int x = config["x"][i]; - int y = config["y"][i]; - EXPECT_NEAR(x, corners[i].x, pixels_error); - EXPECT_NEAR(y, corners[i].y, pixels_error); - } - -#ifdef HAVE_QUIRC - std::string original_info = config["info"]; - EXPECT_EQ(decoded_info, original_info); -#endif - - return; // done - } - } - std::cerr - << "Not found results for '" << name_current_image - << "' image in config file:" << dataset_config << std::endl - << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." - << std::endl; - } + check_qr(root, name_current_image, "test_images", corners, {decoded_info}, pixels_error); } -typedef testing::TestWithParam < std::string > Objdetect_QRCode_Multi; +typedef testing::TestWithParam> Objdetect_QRCode_Multi; TEST_P(Objdetect_QRCode_Multi, regression) { - const std::string name_current_image = GetParam(); + const std::string name_current_image = get<0>(GetParam()); const std::string root = "qrcode/multiple/"; + const std::string method = get<1>(GetParam()); const int pixels_error = 4; std::string image_path = findDataFile(root + name_current_image); Mat src = imread(image_path); ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; - QRCodeDetector qrcode; + if (disabled_samples.find({name_current_image, method}) != disabled_samples.end()) + throw SkipTestException(name_current_image + " is disabled sample for method " + method); + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (method == "aruco_based") { + qrcode = QRCodeDetectorAruco(); + } std::vector corners; #ifdef HAVE_QUIRC std::vector decoded_info; @@ -521,75 +386,15 @@ TEST_P(Objdetect_QRCode_Multi, regression) #else ASSERT_TRUE(qrcode.detectMulti(src, corners)); #endif - - const std::string dataset_config = findDataFile(root + "dataset_config.json"); - FileStorage file_config(dataset_config, FileStorage::READ); - ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; - { - FileNode images_list = file_config["multiple_images"]; - size_t images_count = static_cast(images_list.size()); - ASSERT_GT(images_count, 0u) << "Can't find validation data entries in 'test_images': " << dataset_config; - for (size_t index = 0; index < images_count; index++) - { - FileNode config = images_list[(int)index]; - std::string name_test_image = config["image_name"]; - if (name_test_image == name_current_image) - { - for(int j = 0; j < int(corners.size()); j += 4) - { - bool ok = false; - for (int k = 0; k < int(corners.size() / 4); k++) - { - int count_eq_points = 0; - for (int i = 0; i < 4; i++) - { - int x = config["x"][k][i]; - int y = config["y"][k][i]; - if(((abs(corners[j + i].x - x)) <= pixels_error) && ((abs(corners[j + i].y - y)) <= pixels_error)) - count_eq_points++; - } - if (count_eq_points == 4) - { - ok = true; - break; - } - } - EXPECT_TRUE(ok); - } - -#ifdef HAVE_QUIRC - size_t count_eq_info = 0; - for(int i = 0; i < int(decoded_info.size()); i++) - { - for(int j = 0; j < int(decoded_info.size()); j++) - { - std::string original_info = config["info"][j]; - if(original_info == decoded_info[i]) - { - count_eq_info++; - break; - } - } - } - EXPECT_EQ(decoded_info.size(), count_eq_info); -#endif - - return; // done - } - } - std::cerr - << "Not found results for '" << name_current_image - << "' image in config file:" << dataset_config << std::endl - << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." - << std::endl; - } + check_qr(root, name_current_image, "multiple_images", corners, decoded_info, pixels_error, true); } INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode, testing::ValuesIn(qrcode_images_name)); INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Close, testing::ValuesIn(qrcode_images_close)); INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Monitor, testing::ValuesIn(qrcode_images_monitor)); INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Curved, testing::ValuesIn(qrcode_images_curved)); -INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Multi, testing::ValuesIn(qrcode_images_multiple)); +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Multi, testing::Combine(testing::ValuesIn(qrcode_images_multiple), + testing::Values("contours_based", "aruco_based"))); TEST(Objdetect_QRCode_decodeMulti, decode_regression_16491) { @@ -611,8 +416,10 @@ TEST(Objdetect_QRCode_decodeMulti, decode_regression_16491) #endif } -TEST(Objdetect_QRCode_detectMulti, detect_regression_16961) +typedef testing::TestWithParam Objdetect_QRCode_detectMulti; +TEST_P(Objdetect_QRCode_detectMulti, detect_regression_16961) { + const std::string method = GetParam(); const std::string name_current_image = "9_qrcodes.jpg"; const std::string root = "qrcode/multiple/"; @@ -620,7 +427,10 @@ TEST(Objdetect_QRCode_detectMulti, detect_regression_16961) Mat src = imread(image_path); ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; - QRCodeDetector qrcode; + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (method == "aruco_based") { + qrcode = QRCodeDetectorAruco(); + } std::vector corners; EXPECT_TRUE(qrcode.detectMulti(src, corners)); ASSERT_FALSE(corners.empty()); @@ -628,21 +438,27 @@ TEST(Objdetect_QRCode_detectMulti, detect_regression_16961) EXPECT_EQ(corners.size(), expect_corners_size); } -TEST(Objdetect_QRCode_decodeMulti, check_output_parameters_type_19363) +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_detectMulti, testing::Values("contours_based", "aruco_based")); +typedef testing::TestWithParam Objdetect_QRCode_detectAndDecodeMulti; +TEST_P(Objdetect_QRCode_detectAndDecodeMulti, check_output_parameters_type_19363) { const std::string name_current_image = "9_qrcodes.jpg"; const std::string root = "qrcode/multiple/"; + const std::string method = GetParam(); std::string image_path = findDataFile(root + name_current_image); Mat src = imread(image_path); ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; #ifdef HAVE_QUIRC - QRCodeDetector qrcode; + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (method == "aruco_based") { + qrcode = QRCodeDetectorAruco(); + } std::vector corners; std::vector decoded_info; #if 0 // FIXIT: OutputArray::create() type check std::vector straight_barcode_nchannels; - EXPECT_ANY_THROW(qrcode.detectAndDecodeMulti(src, decoded_info, corners, straight_barcode_nchannels)); + EXPECT_ANY_THROW(qrcode->detectAndDecodeMulti(src, decoded_info, corners, straight_barcode_nchannels)); #endif int expected_barcode_type = CV_8UC1; @@ -653,6 +469,8 @@ TEST(Objdetect_QRCode_decodeMulti, check_output_parameters_type_19363) EXPECT_EQ(expected_barcode_type, straight_barcode[i].type()); #endif } +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_detectAndDecodeMulti, testing::Values("contours_based", "aruco_based")); + TEST(Objdetect_QRCode_detect, detect_regression_20882) { @@ -793,14 +611,18 @@ TEST(Objdetect_QRCode_decode, decode_regression_version_25) #endif } -TEST(Objdetect_QRCode_decodeMulti, decode_9_qrcodes_version7) +TEST_P(Objdetect_QRCode_detectAndDecodeMulti, decode_9_qrcodes_version7) { const std::string name_current_image = "9_qrcodes_version7.jpg"; const std::string root = "qrcode/multiple/"; std::string image_path = findDataFile(root + name_current_image); Mat src = imread(image_path); - QRCodeDetector qrcode; + const std::string method = GetParam(); + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (method == "aruco_based") { + qrcode = QRCodeDetectorAruco(); + } std::vector corners; std::vector decoded_info; diff --git a/samples/cpp/qrcode.cpp b/samples/cpp/qrcode.cpp index af332d307c..84ef621543 100644 --- a/samples/cpp/qrcode.cpp +++ b/samples/cpp/qrcode.cpp @@ -12,6 +12,7 @@ using namespace cv; static int liveQRCodeDetect(); static int imageQRCodeDetect(const string& in_file); +static bool g_useArucoBased = false; static bool g_modeMultiQR = false; static bool g_detectOnly = false; @@ -35,6 +36,7 @@ int main(int argc, char *argv[]) const string keys = "{h help ? | | print help messages }" "{i in | | input image path (also switches to image detection mode) }" + "{aruco_based | false | use Aruco-based QR code detector instead of contour-based }" "{detect | false | detect QR code only (skip decoding) }" "{m multi | | use detect for multiple qr-codes }" "{o out | qr_code.png | path to result file }" @@ -75,6 +77,7 @@ int main(int argc, char *argv[]) g_modeMultiQR = cmd_parser.has("multi") && cmd_parser.get("multi"); g_detectOnly = cmd_parser.has("detect") && cmd_parser.get("detect"); + g_useArucoBased = cmd_parser.has("aruco_based") && cmd_parser.get("aruco_based"); g_saveDetections = cmd_parser.has("save_detections") && cmd_parser.get("save_detections"); g_saveAll = cmd_parser.has("save_all") && cmd_parser.get("save_all"); @@ -157,7 +160,7 @@ void drawQRCodeResults(Mat& frame, const vector& corners, const vector& corners, vector& decode_info // +global: bool g_modeMultiQR, bool g_detectOnly ) @@ -191,7 +194,7 @@ void runQR( } static -double processQRCodeDetection(QRCodeDetector& qrcode, const Mat& input, Mat& result, vector& corners) +double processQRCodeDetection(const QRCodeDetectorBase& qrcode, const Mat& input, Mat& result, vector& corners) { if (input.channels() == 1) cvtColor(input, result, COLOR_GRAY2BGR); @@ -229,7 +232,9 @@ int liveQRCodeDetect() cout << "Press 'd' to switch between decoder and detector" << endl; cout << "Press ' ' (space) to save result into images" << endl; cout << "Press 'ESC' to exit" << endl; - QRCodeDetector qrcode; + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (g_useArucoBased) + qrcode = QRCodeDetectorAruco(); for (;;) { @@ -310,7 +315,10 @@ int imageQRCodeDetect(const string& in_file) << " on image: " << input.size() << " (" << typeToString(input.type()) << ")" << endl; - QRCodeDetector qrcode; + QRCodeDetectorBase qrcode = QRCodeDetector(); + if (g_useArucoBased) + qrcode = QRCodeDetectorAruco(); + vector corners; vector decode_info;