Fixed getContinuousSize() to handle huge matrices properly. This should solve http://code.opencv.org/issues/3232

modules/core/src/precomp.hpp

@@ -151,39 +151,46 @@ template<typename T> struct OpMax
     T operator ()(const T a, const T b) const { return std::max(a, b); }
 };
 
+inline Size getContinuousSize_( int flags, int cols, int rows, int widthScale )
+{
+    int64 sz = (int64)cols * rows * widthScale;
+    return (flags & Mat::CONTINUOUS_FLAG) != 0 &&
+        (int)sz == sz ? Size((int)sz, 1) : Size(cols * widthScale, rows);
+}
+
 inline Size getContinuousSize( const Mat& m1, int widthScale=1 )
 {
-    return m1.isContinuous() ? Size(m1.cols*m1.rows*widthScale, 1) :
-        Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2, int widthScale=1 )
 {
-    return (m1.flags & m2.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2,
                                const Mat& m3, int widthScale=1 )
 {
-    return (m1.flags & m2.flags & m3.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2,
                                const Mat& m3, const Mat& m4,
                                int widthScale=1 )
 {
-    return (m1.flags & m2.flags & m3.flags & m4.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags & m4.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 inline Size getContinuousSize( const Mat& m1, const Mat& m2,
                                const Mat& m3, const Mat& m4,
                                const Mat& m5, int widthScale=1 )
 {
-    return (m1.flags & m2.flags & m3.flags & m4.flags & m5.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags & m4.flags & m5.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 struct NoVec