From 0c10ae18615f7c05f327ac9eebed24f2699b2394 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 15 Aug 2021 20:33:06 +0000 Subject: [PATCH] python: cv.Mat wrapper over numpy.ndarray --- .../py_bindings_basics.markdown | 8 ++ .../python/package/mat_wrapper/__init__.py | 33 +++++ modules/python/src2/cv2.cpp | 88 +++++++++--- modules/python/test/test_mat.py | 131 ++++++++++++++++++ modules/python/test/tests_common.py | 1 + 5 files changed, 244 insertions(+), 17 deletions(-) create mode 100644 modules/core/misc/python/package/mat_wrapper/__init__.py create mode 100644 modules/python/test/test_mat.py diff --git a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown index 2c5eccd0d6..001952deca 100644 --- a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown +++ b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown @@ -60,6 +60,14 @@ of C++. So this is the basic version of how OpenCV-Python bindings are generated. +@note There is no 1:1 mapping of numpy.ndarray on cv::Mat. For example, cv::Mat has channels field, +which is emulated as last dimension of numpy.ndarray and implicitly converted. +However, such implicit conversion has problem with passing of 3D numpy arrays into C++ code +(the last dimension is implicitly reinterpreted as number of channels). +Refer to the [issue](https://github.com/opencv/opencv/issues/19091) for workarounds if you need to process 3D arrays or ND-arrays with channels. +OpenCV 4.5.4+ has `cv.Mat` wrapper derived from `numpy.ndarray` to explicitly handle the channels behavior. + + How to extend new modules to Python? ------------------------------------ diff --git a/modules/core/misc/python/package/mat_wrapper/__init__.py b/modules/core/misc/python/package/mat_wrapper/__init__.py new file mode 100644 index 0000000000..7309c32b01 --- /dev/null +++ b/modules/core/misc/python/package/mat_wrapper/__init__.py @@ -0,0 +1,33 @@ +__all__ = [] + +import sys +import numpy as np +import cv2 as cv + +# NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html + +class Mat(np.ndarray): + ''' + cv.Mat wrapper for numpy array. + + Stores extra metadata information how to interpret and process of numpy array for underlying C++ code. + ''' + + def __new__(cls, arr, **kwargs): + obj = arr.view(Mat) + return obj + + def __init__(self, arr, **kwargs): + self.wrap_channels = kwargs.pop('wrap_channels', getattr(arr, 'wrap_channels', False)) + if len(kwargs) > 0: + raise TypeError('Unknown parameters: {}'.format(repr(kwargs))) + + def __array_finalize__(self, obj): + if obj is None: + return + self.wrap_channels = getattr(obj, 'wrap_channels', None) + + +Mat.__module__ = cv.__name__ +cv.Mat = Mat +cv._registerMatType(Mat) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index e97f17470c..6231fde67f 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -49,6 +49,8 @@ static PyObject* opencv_error = NULL; +static PyTypeObject* pyopencv_Mat_TypePtr = nullptr; + class ArgInfo { public: @@ -638,10 +640,20 @@ static bool isBool(PyObject* obj) CV_NOEXCEPT return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj); } +template +static std::string pycv_dumpArray(const T* arr, int n) +{ + std::ostringstream out; + out << "["; + for (int i = 0; i < n; ++i) + out << " " << arr[i]; + out << " ]"; + return out.str(); +} + // special case, when the converter needs full ArgInfo structure static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) { - bool allowND = true; if(!o || o == Py_None) { if( !m.data ) @@ -727,12 +739,29 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) return false; } - int size[CV_MAX_DIM+1]; - size_t step[CV_MAX_DIM+1]; size_t elemsize = CV_ELEM_SIZE1(type); const npy_intp* _sizes = PyArray_DIMS(oarr); const npy_intp* _strides = PyArray_STRIDES(oarr); + + CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << " _sizes=" << pycv_dumpArray(_sizes, ndims) << " _strides=" << pycv_dumpArray(_strides, ndims)); + bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX; + if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr)) + { + bool wrapChannels = false; + PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels"); + if (pyobj_wrap_channels) + { + if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0))) + { + // TODO extra message + Py_DECREF(pyobj_wrap_channels); + return false; + } + Py_DECREF(pyobj_wrap_channels); + } + ismultichannel = wrapChannels && ndims >= 1; + } for( int i = ndims-1; i >= 0 && !needcopy; i-- ) { @@ -746,14 +775,26 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) needcopy = true; } - if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] ) - needcopy = true; + if (ismultichannel) + { + int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1; + if (channels > CV_CN_MAX) + { + failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX); + return false; + } + ndims--; + type |= CV_MAKETYPE(0, channels); + + if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) + needcopy = true; + } if (needcopy) { if (info.outputarg) { - failmsg("Layout of the output array %s is incompatible with cv::Mat (step[ndims-1] != elemsize or step[1] != elemsize*nchannels)", info.name); + failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name); return false; } @@ -769,6 +810,9 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) _strides = PyArray_STRIDES(oarr); } + int size[CV_MAX_DIM+1] = {}; + size_t step[CV_MAX_DIM+1] = {}; + // Normalize strides in case NPY_RELAXED_STRIDES is set size_t default_step = elemsize; for ( int i = ndims - 1; i >= 0; --i ) @@ -787,23 +831,16 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } // handle degenerate case + // FIXIT: Don't force 1D for Scalars if( ndims == 0) { size[ndims] = 1; step[ndims] = elemsize; ndims++; } - if( ismultichannel ) - { - ndims--; - type |= CV_MAKETYPE(0, size[2]); - } - - if( ndims > 2 && !allowND ) - { - failmsg("%s has more than 2 dimensions", info.name); - return false; - } +#if 1 + CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << " step=" << pycv_dumpArray(step, ndims) << " type=" << cv::typeToString(type)); +#endif m = Mat(ndims, size, type, PyArray_DATA(oarr), step); m.u = g_numpyAllocator.allocate(o, ndims, size, type, step); @@ -2183,7 +2220,24 @@ static int convert_to_char(PyObject *o, char *dst, const ArgInfo& info) #include "pyopencv_generated_types_content.h" #include "pyopencv_generated_funcs.h" +static PyObject* pycvRegisterMatType(PyObject *self, PyObject *value) +{ + CV_LOG_DEBUG(NULL, cv::format("pycvRegisterMatType %p %p\n", self, value)); + + if (0 == PyType_Check(value)) + { + PyErr_SetString(PyExc_TypeError, "Type argument is expected"); + return NULL; + } + + Py_INCREF(value); + pyopencv_Mat_TypePtr = (PyTypeObject*)value; + + Py_RETURN_NONE; +} + static PyMethodDef special_methods[] = { + {"_registerMatType", (PyCFunction)(pycvRegisterMatType), METH_O, "_registerMatType(cv.Mat) -> None (Internal)"}, {"redirectError", CV_PY_FN_WITH_KW(pycvRedirectError), "redirectError(onError) -> None"}, #ifdef HAVE_OPENCV_HIGHGUI {"createTrackbar", (PyCFunction)pycvCreateTrackbar, METH_VARARGS, "createTrackbar(trackbarName, windowName, value, count, onChange) -> None"}, diff --git a/modules/python/test/test_mat.py b/modules/python/test/test_mat.py new file mode 100644 index 0000000000..72614fda36 --- /dev/null +++ b/modules/python/test/test_mat.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +from __future__ import print_function + +import numpy as np +import cv2 as cv + +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + + class MatTest(NewOpenCVTests): + + def test_mat_construct(self): + data = np.random.random([10, 10, 3]) + + #print(np.ndarray.__dictoffset__) # 0 + #print(cv.Mat.__dictoffset__) # 88 (> 0) + #print(cv.Mat) # + #print(cv.Mat.__base__) # + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=300 dims(-1)=3 size(-1)=[10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # fail if __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + + def test_mat_construct_4d(self): + data = np.random.random([5, 10, 10, 3]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=1500 dims(-1)=4 size(-1)=[5 10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + + def test_mat_wrap_channels_fail(self): + data = np.random.random([2, 3, 4, 520]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=12480 dims(-1)=4 size(-1)=[2 3 4 520] type(-1)=CV_64FC1") + + with self.assertRaises(cv.error): + mat_data1 = cv.Mat(data, wrap_channels=True) # argument unable to wrap channels, too high (520 > CV_CN_MAX=512) + res1 = cv.utils.dumpInputArray(mat_data1) + print(mat_data1.__dict__) + print(res1) + + + def test_ufuncs(self): + data = np.arange(10) + mat_data = cv.Mat(data) + mat_data2 = 2 * mat_data + self.assertEqual(type(mat_data2), cv.Mat) + np.testing.assert_equal(2 * data, 2 * mat_data) + + + def test_comparison(self): + # Undefined behavior, do NOT use that. + # Behavior may be changed in the future + + data = np.ones((10, 10, 3)) + mat_wrapped = cv.Mat(data, wrap_channels=True) + mat_simple = cv.Mat(data) + np.testing.assert_equal(mat_wrapped, mat_simple) # ???: wrap_channels is not checked for now + np.testing.assert_equal(data, mat_simple) + np.testing.assert_equal(data, mat_wrapped) + + #self.assertEqual(mat_wrapped, mat_simple) # ??? + #self.assertTrue(mat_wrapped == mat_simple) # ??? + #self.assertTrue((mat_wrapped == mat_simple).all()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index a938a8e2cb..2245722939 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -10,6 +10,7 @@ import random import argparse import numpy as np +#sys.OpenCV_LOADER_DEBUG = True import cv2 as cv # Python 3 moved urlopen to urllib.requests