Merge pull request #20823 from AleksandrPanov:fix_orb_integer_overflow

Fix ORB integer overflow

* set size_t step to fix integer overflow in ptr0 offset

* added issue_537 test

* minor fix tags, points

* added size_t_step and offset to remove mixed unsigned and signed operations

* features2d: update ORB checks

Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>

modules/features2d/src/orb.cpp

@@ -131,12 +131,17 @@ static void
 HarrisResponses(const Mat& img, const std::vector<Rect>& layerinfo,
                 std::vector<KeyPoint>& pts, int blockSize, float harris_k)
 {
-    CV_Assert( img.type() == CV_8UC1 && blockSize*blockSize <= 2048 );
+    CV_CheckTypeEQ(img.type(), CV_8UC1, "");
+    CV_CheckGT(blockSize, 0, "");
+    CV_CheckLE(blockSize*blockSize, 2048, "");
 
     size_t ptidx, ptsize = pts.size();
 
     const uchar* ptr00 = img.ptr<uchar>();
-    int step = (int)(img.step/img.elemSize1());
+    size_t size_t_step = img.step;
+    CV_CheckLE(size_t_step * blockSize + blockSize + 1, (size_t)INT_MAX, "");  // ofs computation, step+1
+    int step = static_cast<int>(size_t_step);
+
     int r = blockSize/2;
 
     float scale = 1.f/((1 << 2) * blockSize * 255.f);
@@ -154,7 +159,7 @@ HarrisResponses(const Mat& img, const std::vector<Rect>& layerinfo,
         int y0 = cvRound(pts[ptidx].pt.y);
         int z = pts[ptidx].octave;
 
-        const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*step + x0 - r + layerinfo[z].x;
+        const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*size_t_step + (x0 - r + layerinfo[z].x);
         int a = 0, b = 0, c = 0;
 
         for( int k = 0; k < blockSize*blockSize; k++ )

modules/features2d/test/test_orb.cpp

@@ -141,5 +141,31 @@ TEST(Features2D_ORB, regression_16197)
     ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv));
 }
 
+// https://github.com/opencv/opencv-python/issues/537
+BIGDATA_TEST(Features2D_ORB, regression_opencv_python_537)  // memory usage: ~3 Gb
+{
+    applyTestTag(
+        CV_TEST_TAG_LONG,
+        CV_TEST_TAG_DEBUG_VERYLONG,
+        CV_TEST_TAG_MEMORY_6GB
+    );
+
+    const int width = 25000;
+    const int height = 25000;
+    Mat img(Size(width, height), CV_8UC1, Scalar::all(0));
+
+    const int border = 23, num_lines = 23;
+    for (int i = 0; i < num_lines; i++)
+    {
+        cv::Point2i point1(border + i * 100, border + i * 100);
+        cv::Point2i point2(width - border - i * 100, height - border * i * 100);
+        cv::line(img, point1, point2, 255, 1, LINE_AA);
+    }
+
+    Ptr<ORB> orbPtr = ORB::create(31);
+    std::vector<KeyPoint> kps;
+    Mat fv;
+    ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv));
+}
 
 }} // namespace