python: cv.Mat wrapper over numpy.ndarray

doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown

@@ -60,6 +60,14 @@ of C++.
 
 So this is the basic version of how OpenCV-Python bindings are generated.
 
+@note There is no 1:1 mapping of numpy.ndarray on cv::Mat. For example, cv::Mat has channels field,
+which is emulated as last dimension of numpy.ndarray and implicitly converted.
+However, such implicit conversion has problem with passing of 3D numpy arrays into C++ code
+(the last dimension is implicitly reinterpreted as number of channels).
+Refer to the [issue](https://github.com/opencv/opencv/issues/19091) for workarounds if you need to process 3D arrays or ND-arrays with channels.
+OpenCV 4.5.4+ has `cv.Mat` wrapper derived from `numpy.ndarray` to explicitly handle the channels behavior.
+
+
 How to extend new modules to Python?
 ------------------------------------
 

modules/core/misc/python/package/mat_wrapper/__init__.py

@@ -0,0 +1,33 @@
+__all__ = []
+
+import sys
+import numpy as np
+import cv2 as cv
+
+# NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html
+
+class Mat(np.ndarray):
+    '''
+    cv.Mat wrapper for numpy array.
+
+    Stores extra metadata information how to interpret and process of numpy array for underlying C++ code.
+    '''
+
+    def __new__(cls, arr, **kwargs):
+        obj = arr.view(Mat)
+        return obj
+
+    def __init__(self, arr, **kwargs):
+        self.wrap_channels = kwargs.pop('wrap_channels', getattr(arr, 'wrap_channels', False))
+        if len(kwargs) > 0:
+            raise TypeError('Unknown parameters: {}'.format(repr(kwargs)))
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self.wrap_channels = getattr(obj, 'wrap_channels', None)
+
+
+Mat.__module__ = cv.__name__
+cv.Mat = Mat
+cv._registerMatType(Mat)

modules/python/src2/cv2.cpp

@@ -49,6 +49,8 @@
 
 static PyObject* opencv_error = NULL;
 
+static PyTypeObject* pyopencv_Mat_TypePtr = nullptr;
+
 class ArgInfo
 {
 public:
@@ -638,10 +640,20 @@ static bool isBool(PyObject* obj) CV_NOEXCEPT
     return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj);
 }
 
+template <typename T>
+static std::string pycv_dumpArray(const T* arr, int n)
+{
+    std::ostringstream out;
+    out << "[";
+    for (int i = 0; i < n; ++i)
+        out << " " << arr[i];
+    out << " ]";
+    return out.str();
+}
+
 // special case, when the converter needs full ArgInfo structure
 static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info)
 {
-    bool allowND = true;
     if(!o || o == Py_None)
     {
         if( !m.data )
@@ -727,12 +739,29 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info)
         return false;
     }
 
-    int size[CV_MAX_DIM+1];
-    size_t step[CV_MAX_DIM+1];
     size_t elemsize = CV_ELEM_SIZE1(type);
     const npy_intp* _sizes = PyArray_DIMS(oarr);
     const npy_intp* _strides = PyArray_STRIDES(oarr);
+
+    CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << "  _sizes=" << pycv_dumpArray(_sizes, ndims) << "  _strides=" << pycv_dumpArray(_strides, ndims));
+
     bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX;
+    if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr))
+    {
+        bool wrapChannels = false;
+        PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels");
+        if (pyobj_wrap_channels)
+        {
+            if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0)))
+            {
+                // TODO extra message
+                Py_DECREF(pyobj_wrap_channels);
+                return false;
+            }
+            Py_DECREF(pyobj_wrap_channels);
+        }
+        ismultichannel = wrapChannels && ndims >= 1;
+    }
 
     for( int i = ndims-1; i >= 0 && !needcopy; i-- )
     {
@@ -746,14 +775,26 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info)
             needcopy = true;
     }
 
-    if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] )
+    if (ismultichannel)
-        needcopy = true;
+    {
+        int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1;
+        if (channels > CV_CN_MAX)
+        {
+            failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX);
+            return false;
+        }
+        ndims--;
+        type |= CV_MAKETYPE(0, channels);
+
+        if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims])
+            needcopy = true;
+    }
 
     if (needcopy)
     {
         if (info.outputarg)
         {
-            failmsg("Layout of the output array %s is incompatible with cv::Mat (step[ndims-1] != elemsize or step[1] != elemsize*nchannels)", info.name);
+            failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name);
             return false;
         }
 
@@ -769,6 +810,9 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info)
         _strides = PyArray_STRIDES(oarr);
     }
 
+    int size[CV_MAX_DIM+1] = {};
+    size_t step[CV_MAX_DIM+1] = {};
+
     // Normalize strides in case NPY_RELAXED_STRIDES is set
     size_t default_step = elemsize;
     for ( int i = ndims - 1; i >= 0; --i )
@@ -787,23 +831,16 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info)
     }
 
     // handle degenerate case
+    // FIXIT: Don't force 1D for Scalars
     if( ndims == 0) {
         size[ndims] = 1;
         step[ndims] = elemsize;
         ndims++;
     }
 
-    if( ismultichannel )
+#if 1
-    {
+    CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << "  step=" << pycv_dumpArray(step, ndims) << "  type=" << cv::typeToString(type));
-        ndims--;
+#endif
-        type |= CV_MAKETYPE(0, size[2]);
-    }
-
-    if( ndims > 2 && !allowND )
-    {
-        failmsg("%s has more than 2 dimensions", info.name);
-        return false;
-    }
 
     m = Mat(ndims, size, type, PyArray_DATA(oarr), step);
     m.u = g_numpyAllocator.allocate(o, ndims, size, type, step);
@@ -2183,7 +2220,24 @@ static int convert_to_char(PyObject *o, char *dst, const ArgInfo& info)
 #include "pyopencv_generated_types_content.h"
 #include "pyopencv_generated_funcs.h"
 
+static PyObject* pycvRegisterMatType(PyObject *self, PyObject *value)
+{
+    CV_LOG_DEBUG(NULL, cv::format("pycvRegisterMatType %p %p\n", self, value));
+
+    if (0 == PyType_Check(value))
+    {
+        PyErr_SetString(PyExc_TypeError, "Type argument is expected");
+        return NULL;
+    }
+
+    Py_INCREF(value);
+    pyopencv_Mat_TypePtr = (PyTypeObject*)value;
+
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef special_methods[] = {
+  {"_registerMatType", (PyCFunction)(pycvRegisterMatType), METH_O, "_registerMatType(cv.Mat) -> None (Internal)"},
   {"redirectError", CV_PY_FN_WITH_KW(pycvRedirectError), "redirectError(onError) -> None"},
 #ifdef HAVE_OPENCV_HIGHGUI
   {"createTrackbar", (PyCFunction)pycvCreateTrackbar, METH_VARARGS, "createTrackbar(trackbarName, windowName, value, count, onChange) -> None"},

modules/python/test/test_mat.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import numpy as np
+import cv2 as cv
+
+import os
+import sys
+import unittest
+
+from tests_common import NewOpenCVTests
+
+try:
+    if sys.version_info[:2] < (3, 0):
+        raise unittest.SkipTest('Python 2.x is not supported')
+
+
+    class MatTest(NewOpenCVTests):
+
+        def test_mat_construct(self):
+            data = np.random.random([10, 10, 3])
+
+            #print(np.ndarray.__dictoffset__)  # 0
+            #print(cv.Mat.__dictoffset__)  # 88 (> 0)
+            #print(cv.Mat)  # <class cv2.Mat>
+            #print(cv.Mat.__base__)  # <class 'numpy.ndarray'>
+
+            mat_data0 = cv.Mat(data)
+            assert isinstance(mat_data0, cv.Mat)
+            assert isinstance(mat_data0, np.ndarray)
+            self.assertEqual(mat_data0.wrap_channels, False)
+            res0 = cv.utils.dumpInputArray(mat_data0)
+            self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=300 dims(-1)=3 size(-1)=[10 10 3] type(-1)=CV_64FC1")
+
+            mat_data1 = cv.Mat(data, wrap_channels=True)
+            assert isinstance(mat_data1, cv.Mat)
+            assert isinstance(mat_data1, np.ndarray)
+            self.assertEqual(mat_data1.wrap_channels, True)
+            res1 = cv.utils.dumpInputArray(mat_data1)
+            self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3")
+
+            mat_data2 = cv.Mat(mat_data1)
+            assert isinstance(mat_data2, cv.Mat)
+            assert isinstance(mat_data2, np.ndarray)
+            self.assertEqual(mat_data2.wrap_channels, True)  # fail if __array_finalize__ doesn't work
+            res2 = cv.utils.dumpInputArray(mat_data2)
+            self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3")
+
+
+        def test_mat_construct_4d(self):
+            data = np.random.random([5, 10, 10, 3])
+
+            mat_data0 = cv.Mat(data)
+            assert isinstance(mat_data0, cv.Mat)
+            assert isinstance(mat_data0, np.ndarray)
+            self.assertEqual(mat_data0.wrap_channels, False)
+            res0 = cv.utils.dumpInputArray(mat_data0)
+            self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=1500 dims(-1)=4 size(-1)=[5 10 10 3] type(-1)=CV_64FC1")
+
+            mat_data1 = cv.Mat(data, wrap_channels=True)
+            assert isinstance(mat_data1, cv.Mat)
+            assert isinstance(mat_data1, np.ndarray)
+            self.assertEqual(mat_data1.wrap_channels, True)
+            res1 = cv.utils.dumpInputArray(mat_data1)
+            self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3")
+
+            mat_data2 = cv.Mat(mat_data1)
+            assert isinstance(mat_data2, cv.Mat)
+            assert isinstance(mat_data2, np.ndarray)
+            self.assertEqual(mat_data2.wrap_channels, True)  # __array_finalize__ doesn't work
+            res2 = cv.utils.dumpInputArray(mat_data2)
+            self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3")
+
+
+        def test_mat_wrap_channels_fail(self):
+            data = np.random.random([2, 3, 4, 520])
+
+            mat_data0 = cv.Mat(data)
+            assert isinstance(mat_data0, cv.Mat)
+            assert isinstance(mat_data0, np.ndarray)
+            self.assertEqual(mat_data0.wrap_channels, False)
+            res0 = cv.utils.dumpInputArray(mat_data0)
+            self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=12480 dims(-1)=4 size(-1)=[2 3 4 520] type(-1)=CV_64FC1")
+
+            with self.assertRaises(cv.error):
+                mat_data1 = cv.Mat(data, wrap_channels=True)  # argument unable to wrap channels, too high (520 > CV_CN_MAX=512)
+                res1 = cv.utils.dumpInputArray(mat_data1)
+                print(mat_data1.__dict__)
+                print(res1)
+
+
+        def test_ufuncs(self):
+            data = np.arange(10)
+            mat_data = cv.Mat(data)
+            mat_data2 = 2 * mat_data
+            self.assertEqual(type(mat_data2), cv.Mat)
+            np.testing.assert_equal(2 * data, 2 * mat_data)
+
+
+        def test_comparison(self):
+            # Undefined behavior, do NOT use that.
+            # Behavior may be changed in the future
+
+            data = np.ones((10, 10, 3))
+            mat_wrapped = cv.Mat(data, wrap_channels=True)
+            mat_simple = cv.Mat(data)
+            np.testing.assert_equal(mat_wrapped, mat_simple)  # ???: wrap_channels is not checked for now
+            np.testing.assert_equal(data, mat_simple)
+            np.testing.assert_equal(data, mat_wrapped)
+
+            #self.assertEqual(mat_wrapped, mat_simple)  # ???
+            #self.assertTrue(mat_wrapped == mat_simple)  # ???
+            #self.assertTrue((mat_wrapped == mat_simple).all())
+
+
+except unittest.SkipTest as e:
+
+    message = str(e)
+
+    class TestSkip(unittest.TestCase):
+        def setUp(self):
+            self.skipTest('Skip tests: ' + message)
+
+        def test_skip():
+            pass
+
+    pass
+
+
+if __name__ == '__main__':
+    NewOpenCVTests.bootstrap()

modules/python/test/tests_common.py

@@ -10,6 +10,7 @@ import random
 import argparse
 
 import numpy as np
+#sys.OpenCV_LOADER_DEBUG = True
 import cv2 as cv
 
 # Python 3 moved urlopen to urllib.requests