From 6b4ea68bc76379de8b1ebcf3a2ed7574e1758738 Mon Sep 17 00:00:00 2001
From: Vincent Rabaud <vrabaud@google.com>
Date: Wed, 17 Nov 2021 17:38:21 +0100
Subject: [PATCH] Solve Rect overflow issues.

Fow now, it is possible to define valid rectangle for which some
functions overflow (e.g. br(), ares() ...).
This patch fixes the intersection operator so that it works with
any rectangle.
---
 modules/core/include/opencv2/core/types.hpp | 34 ++++++++++++++++-----
 modules/core/test/test_misc.cpp             | 32 +++++++++++++++++++
 2 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp
index 0b1d948156..82dedcb850 100644
--- a/modules/core/include/opencv2/core/types.hpp
+++ b/modules/core/include/opencv2/core/types.hpp
@@ -1895,13 +1895,33 @@ Rect_<_Tp>& operator -= ( Rect_<_Tp>& a, const Size_<_Tp>& b )
 template<typename _Tp> static inline
 Rect_<_Tp>& operator &= ( Rect_<_Tp>& a, const Rect_<_Tp>& b )
 {
-    _Tp x1 = std::max(a.x, b.x);
-    _Tp y1 = std::max(a.y, b.y);
-    a.width = std::min(a.x + a.width, b.x + b.width) - x1;
-    a.height = std::min(a.y + a.height, b.y + b.height) - y1;
-    a.x = x1;
-    a.y = y1;
-    if( a.width <= 0 || a.height <= 0 )
+    if (a.empty() || b.empty()) {
+        a = Rect();
+        return a;
+    }
+    const Rect_<_Tp>& Rx_min = (a.x < b.x) ? a : b;
+    const Rect_<_Tp>& Rx_max = (a.x < b.x) ? b : a;
+    const Rect_<_Tp>& Ry_min = (a.y < b.y) ? a : b;
+    const Rect_<_Tp>& Ry_max = (a.y < b.y) ? b : a;
+    // Looking at the formula below, we will compute Rx_min.width - (Rx_max.x - Rx_min.x)
+    // but we want to avoid overflows. Rx_min.width >= 0 and (Rx_max.x - Rx_min.x) >= 0
+    // by definition so the difference does not overflow. The only thing that can overflow
+    // is (Rx_max.x - Rx_min.x). And it can only overflow if Rx_min.x < 0.
+    // Let us first deal with the following case.
+    if ((Rx_min.x < 0 && Rx_min.x + Rx_min.width < Rx_max.x) ||
+        (Ry_min.y < 0 && Ry_min.y + Ry_min.height < Ry_max.y)) {
+        a = Rect();
+        return a;
+    }
+    // We now know that either Rx_min.x >= 0, or
+    // Rx_min.x < 0 && Rx_min.x + Rx_min.width >= Rx_max.x and therefore
+    // Rx_min.width >= (Rx_max.x - Rx_min.x) which means (Rx_max.x - Rx_min.x)
+    // is inferior to a valid int and therefore does not overflow.
+    a.width = std::min(Rx_min.width - (Rx_max.x - Rx_min.x), Rx_max.width);
+    a.height = std::min(Ry_min.height - (Ry_max.y - Ry_min.y), Ry_max.height);
+    a.x = Rx_max.x;
+    a.y = Ry_max.y;
+    if (a.empty())
         a = Rect();
     return a;
 }
diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp
index 372aab7eb0..d9e9119230 100644
--- a/modules/core/test/test_misc.cpp
+++ b/modules/core/test/test_misc.cpp
@@ -784,4 +784,36 @@ TEST(Core_Check, testSize_1)
 }
 
 
+template <typename T> class Rect_Test : public testing::Test {};
+
+TYPED_TEST_CASE_P(Rect_Test);
+
+// Reimplement C++11 std::numeric_limits<>::lowest.
+template<typename T> T cv_numeric_limits_lowest();
+template<> int cv_numeric_limits_lowest<int>() { return INT_MIN; }
+template<> float cv_numeric_limits_lowest<float>() { return -FLT_MAX; }
+template<> double cv_numeric_limits_lowest<double>() { return -DBL_MAX; }
+
+TYPED_TEST_P(Rect_Test, Overflows) {
+  typedef Rect_<TypeParam> R;
+  TypeParam num_max = std::numeric_limits<TypeParam>::max();
+  TypeParam num_lowest = cv_numeric_limits_lowest<TypeParam>();
+  EXPECT_EQ(R(0, 0, 10, 10), R(0, 0, 10, 10) & R(0, 0, 10, 10));
+  EXPECT_EQ(R(5, 6, 4, 3), R(0, 0, 10, 10) & R(5, 6, 4, 3));
+  EXPECT_EQ(R(5, 6, 3, 2), R(0, 0, 8, 8) & R(5, 6, 4, 3));
+  // Test with overflowing dimenions.
+  EXPECT_EQ(R(5, 0, 5, 10), R(0, 0, 10, 10) & R(5, 0, num_max, num_max));
+  // Test with overflowing dimensions for floats/doubles.
+  EXPECT_EQ(R(num_max, 0, num_max / 4, 10), R(num_max, 0, num_max / 2, 10) & R(num_max, 0, num_max / 4, 10));
+  // Test with overflowing coordinates.
+  EXPECT_EQ(R(), R(20, 0, 10, 10) & R(num_lowest, 0, 10, 10));
+  EXPECT_EQ(R(), R(20, 0, 10, 10) & R(0, num_lowest, 10, 10));
+  EXPECT_EQ(R(), R(num_lowest, 0, 10, 10) & R(0, num_lowest, 10, 10));
+}
+REGISTER_TYPED_TEST_CASE_P(Rect_Test, Overflows);
+
+typedef ::testing::Types<int, float, double> RectTypes;
+INSTANTIATE_TYPED_TEST_CASE_P(Negative_Test, Rect_Test, RectTypes);
+
+
 }} // namespace