calib3d: fix cornerSubPix memory error

modules/calib3d/test/test_cornerssubpix.cpp

@@ -239,4 +239,18 @@ void CV_ChessboardSubpixelTest::generateIntrinsicParams()
 
 TEST(Calib3d_ChessboardSubPixDetector, accuracy) { CV_ChessboardSubpixelTest test; test.safe_run(); }
 
+TEST(Calib3d_CornerSubPix, regression_7204)
+{
+    cv::Mat image(cv::Size(70, 38), CV_8UC1, cv::Scalar::all(0));
+    image(cv::Rect(65, 26, 5, 5)).setTo(cv::Scalar::all(255));
+    image(cv::Rect(55, 31, 8, 1)).setTo(cv::Scalar::all(255));
+    image(cv::Rect(56, 35, 14, 2)).setTo(cv::Scalar::all(255));
+    image(cv::Rect(66, 24, 4, 2)).setTo(cv::Scalar::all(255));
+    image.at<uchar>(24, 69) = 0;
+    std::vector<cv::Point2f> corners;
+    corners.push_back(cv::Point2f(65, 30));
+    cv::cornerSubPix(image, corners, cv::Size(3, 3), cv::Size(-1, -1),
+        cv::TermCriteria(CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, 30, 0.1));
+}
+
 /* End of file. */

modules/imgproc/src/cornersubpix.cpp

@@ -234,6 +234,15 @@ cvFindCornerSubPix( const void* srcarr, CvPoint2D32f* corners,
 
             err = (cI2.x - cI.x) * (cI2.x - cI.x) + (cI2.y - cI.y) * (cI2.y - cI.y);
             cI = cI2;
+
+            /* if new point is too far from initial, it means poor convergence. */
+            if (fabs(cI.x - cT.x) > win.width || fabs(cI.y - cT.y) > win.height)
+            {
+                cI = cT;
+                break;
+            }
+            cI.x = std::max(0.0f, std::min((float)size.width, cI.x));
+            cI.y = std::max(0.0f, std::min((float)size.height, cI.y));
         }
         while( ++iter < max_iters && err > eps );
 

modules/imgproc/src/samplers.cpp

@@ -392,6 +392,8 @@ CvStatus CV_STDCALL icvGetRectSubPix_8u32f_C1R
     ip.x = cvFloor( center.x );
     ip.y = cvFloor( center.y );
 
+    CV_DbgAssert(fabs(center.x - ip.x) <= 1.0f && fabs(center.y - ip.y) <= 1.0f);
+
     if( win_size.width <= 0 || win_size.height <= 0 )
         return CV_BADRANGE_ERR;