Merge pull request #6078 from PolarNick239:master

modules/java/generator/gen_java.py

@@ -1041,7 +1041,8 @@ class JavaWrapperGenerator(object):
         self.clear()
         self.module = module
         self.Module = module.capitalize()
-        parser = hdr_parser.CppHeaderParser()
+        # TODO: support UMat versions of declarations (implement UMat-wrapper for Java)
+        parser = hdr_parser.CppHeaderParser(generate_umat_decls=False)
 
         self.add_class( ['class ' + self.Module, '', [], []] ) # [ 'class/struct cname', ':bases', [modlist] [props] ]
 

modules/python/src2/cv2.cpp

@@ -101,6 +101,7 @@ typedef std::vector<Rect> vector_Rect;
 typedef std::vector<Rect2d> vector_Rect2d;
 typedef std::vector<KeyPoint> vector_KeyPoint;
 typedef std::vector<Mat> vector_Mat;
+typedef std::vector<UMat> vector_UMat;
 typedef std::vector<DMatch> vector_DMatch;
 typedef std::vector<String> vector_String;
 typedef std::vector<Scalar> vector_Scalar;
@@ -409,6 +410,152 @@ PyObject* pyopencv_from(const Mat& m)
     return o;
 }
 
+
+typedef struct {
+    PyObject_HEAD
+    UMat* um;
+} cv2_UMatWrapperObject;
+
+// UMatWrapper init - takes one optional argument, that converts to Mat, that converts to UMat and stored inside.
+// If no argument given - empty UMat created.
+static int UMatWrapper_init(cv2_UMatWrapperObject *self, PyObject *args, PyObject *kwds)
+{
+    self->um = new UMat();
+
+    PyObject *np_mat = NULL;
+
+    static char *kwlist[] = {new char[3], NULL};
+    strcpy(kwlist[0], "mat");
+
+    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &np_mat))
+        return -1;
+
+    if (np_mat) {
+        Mat m;
+        if (!pyopencv_to(np_mat, m, ArgInfo("UMatWrapper.np_mat", 0)))
+            return -1;
+
+        m.copyTo(*self->um);
+    }
+
+    return 0;
+}
+
+static void UMatWrapper_dealloc(cv2_UMatWrapperObject* self)
+{
+    delete self->um;
+#if PY_MAJOR_VERSION >= 3
+    Py_TYPE(self)->tp_free((PyObject*)self);
+#else
+    self->ob_type->tp_free((PyObject*)self);
+#endif
+}
+
+// UMatWrapper.get() - returns numpy array by transferring UMat data to Mat and than wrapping it to numpy array
+// (using numpy allocator - and so without unnecessary copy)
+static PyObject * UMatWrapper_get(cv2_UMatWrapperObject* self)
+{
+    Mat m;
+    m.allocator = &g_numpyAllocator;
+    self->um->copyTo(m);
+
+    return pyopencv_from(m);
+}
+
+static PyMethodDef UMatWrapper_methods[] = {
+        {"get", (PyCFunction)UMatWrapper_get, METH_NOARGS,
+                "Returns numpy array"
+        },
+        {NULL, NULL, 0, NULL}  /* Sentinel */
+};
+
+
+static PyTypeObject cv2_UMatWrapperType = {
+#if PY_MAJOR_VERSION >= 3
+        PyVarObject_HEAD_INIT(NULL, 0)
+#else
+        PyObject_HEAD_INIT(NULL)
+        0,                             /*ob_size*/
+#endif
+        "cv2.UMat",                    /* tp_name */
+        sizeof(cv2_UMatWrapperObject), /* tp_basicsize */
+        0,                             /* tp_itemsize */
+      (destructor)UMatWrapper_dealloc, /* tp_dealloc */
+        0,                             /* tp_print */
+        0,                             /* tp_getattr */
+        0,                             /* tp_setattr */
+        0,                             /* tp_reserved */
+        0,                             /* tp_repr */
+        0,                             /* tp_as_number */
+        0,                             /* tp_as_sequence */
+        0,                             /* tp_as_mapping */
+        0,                             /* tp_hash  */
+        0,                             /* tp_call */
+        0,                             /* tp_str */
+        0,                             /* tp_getattro */
+        0,                             /* tp_setattro */
+        0,                             /* tp_as_buffer */
+        Py_TPFLAGS_DEFAULT,            /* tp_flags */
+        "OpenCV 3 UMat wrapper. Used for T-API support.", /* tp_doc */
+        0,                             /* tp_traverse */
+        0,                             /* tp_clear */
+        0,                             /* tp_richcompare */
+        0,                             /* tp_weaklistoffset */
+        0,                             /* tp_iter */
+        0,                             /* tp_iternext */
+        UMatWrapper_methods,           /* tp_methods */
+        0,                             /* tp_members */
+        0,                             /* tp_getset */
+        0,                             /* tp_base */
+        0,                             /* tp_dict */
+        0,                             /* tp_descr_get */
+        0,                             /* tp_descr_set */
+        0,                             /* tp_dictoffset */
+        (initproc)UMatWrapper_init,    /* tp_init */
+        0,                             /* tp_alloc */
+        PyType_GenericNew,             /* tp_new */
+        0,                             /* tp_free */
+        0,                             /* tp_is_gc */
+        0,                             /* tp_bases */
+        0,                             /* tp_mro */
+        0,                             /* tp_cache */
+        0,                             /* tp_subclasses */
+        0,                             /* tp_weaklist */
+        0,                             /* tp_del */
+        0,                             /* tp_version_tag */
+#if PY_MAJOR_VERSION >= 3
+        0,                             /* tp_finalize */
+#endif
+};
+
+static bool pyopencv_to(PyObject* o, UMat& um, const ArgInfo info) {
+    if (o != NULL && PyObject_TypeCheck(o, &cv2_UMatWrapperType) ) {
+        um = *((cv2_UMatWrapperObject *) o)->um;
+        return true;
+    }
+
+    Mat m;
+    if (!pyopencv_to(o, m, info)) {
+        return false;
+    }
+
+    m.copyTo(um);
+    return true;
+}
+
+template<>
+bool pyopencv_to(PyObject* o, UMat& um, const char* name)
+{
+    return pyopencv_to(o, um, ArgInfo(name, 0));
+}
+
+template<>
+PyObject* pyopencv_from(const UMat& m) {
+    PyObject *o = PyObject_CallObject((PyObject *) &cv2_UMatWrapperType, NULL);
+    *((cv2_UMatWrapperObject *) o)->um = m;
+    return o;
+}
+
 template<>
 bool pyopencv_to(PyObject *o, Scalar& s, const char *name)
 {
@@ -1321,6 +1468,23 @@ void initcv2()
   opencv_error = PyErr_NewException((char*)MODULESTR".error", NULL, NULL);
   PyDict_SetItemString(d, "error", opencv_error);
 
+//Registering UMatWrapper python class in cv2 module:
+  if (PyType_Ready(&cv2_UMatWrapperType) < 0)
+#if PY_MAJOR_VERSION >= 3
+    return NULL;
+#else
+    return;
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+  Py_INCREF(&cv2_UMatWrapperType);
+#else
+  // Unrolled Py_INCREF(&cv2_UMatWrapperType) without (PyObject*) cast
+  // due to "warning: dereferencing type-punned pointer will break strict-aliasing rules"
+  _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA (&cv2_UMatWrapperType)->ob_refcnt++;
+#endif
+  PyModule_AddObject(m, "UMat", (PyObject *)&cv2_UMatWrapperType);
+
 #define PUBLISH(I) PyDict_SetItemString(d, #I, PyInt_FromLong(I))
 //#define PUBLISHU(I) PyDict_SetItemString(d, #I, PyLong_FromUnsignedLong(I))
 #define PUBLISH2(I, value) PyDict_SetItemString(d, #I, PyLong_FromLong(value))

modules/python/src2/gen2.py

@@ -392,7 +392,8 @@ class ArgInfo(object):
         self.py_outputarg = False
 
     def isbig(self):
-        return self.tp == "Mat" or self.tp == "vector_Mat"# or self.tp.startswith("vector")
+        return self.tp == "Mat" or self.tp == "vector_Mat"\
+               or self.tp == "UMat" or self.tp == "vector_UMat" # or self.tp.startswith("vector")
 
     def crepr(self):
         return "ArgInfo(\"%s\", %d)" % (self.name, self.outputarg)
@@ -634,6 +635,10 @@ class FuncInfo(object):
                 defval = a.defval
                 if not defval:
                     defval = amapping[2]
+                else:
+                    if "UMat" in tp:
+                        if "Mat" in defval and "UMat" not in defval:
+                            defval = defval.replace("Mat", "UMat")
                 # "tp arg = tp();" is equivalent to "tp arg;" in the case of complex types
                 if defval == tp + "()" and amapping[1] == "O":
                     defval = ""
@@ -856,7 +861,7 @@ class PythonWrapperGenerator(object):
 
     def gen(self, srcfiles, output_path):
         self.clear()
-        self.parser = hdr_parser.CppHeaderParser()
+        self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True)
 
         # step 1: scan the headers and build more descriptive maps of classes, consts, functions
         for hdr in srcfiles:

modules/python/src2/hdr_parser.py

@@ -17,7 +17,7 @@ opencv_hdr_list = [
 "../../objdetect/include/opencv2/objdetect.hpp",
 "../../imgcodecs/include/opencv2/imgcodecs.hpp",
 "../../videoio/include/opencv2/videoio.hpp",
-"../../highgui/include/opencv2/highgui.hpp"
+"../../highgui/include/opencv2/highgui.hpp",
 ]
 
 """
@@ -31,7 +31,9 @@ where the list of modifiers is yet another nested list of strings
 
 class CppHeaderParser(object):
 
-    def __init__(self):
+    def __init__(self, generate_umat_decls=False):
+        self._generate_umat_decls = generate_umat_decls
+
         self.BLOCK_TYPE = 0
         self.BLOCK_NAME = 1
         self.PROCESS_FLAG = 2
@@ -368,7 +370,7 @@ class CppHeaderParser(object):
             print(decl_str)
         return decl
 
-    def parse_func_decl(self, decl_str):
+    def parse_func_decl(self, decl_str, use_umat=False):
         """
         Parses the function or method declaration in the form:
         [([CV_EXPORTS] <rettype>) | CVAPI(rettype)]
@@ -537,28 +539,34 @@ class CppHeaderParser(object):
                         a = a[:eqpos].strip()
                     arg_type, arg_name, modlist, argno = self.parse_arg(a, argno)
                     if self.wrap_mode:
+                        mat = "UMat" if use_umat else "Mat"
+
+                        # TODO: Vectors should contain UMat, but this is not very easy to support and not very needed
+                        vector_mat = "vector_{}".format("Mat")
+                        vector_mat_template = "vector<{}>".format("Mat")
+
                         if arg_type == "InputArray":
-                            arg_type = "Mat"
+                            arg_type = mat
                         elif arg_type == "InputOutputArray":
-                            arg_type = "Mat"
+                            arg_type = mat
                             modlist.append("/IO")
                         elif arg_type == "OutputArray":
-                            arg_type = "Mat"
+                            arg_type = mat
                             modlist.append("/O")
                         elif arg_type == "InputArrayOfArrays":
-                            arg_type = "vector_Mat"
+                            arg_type = vector_mat
                         elif arg_type == "InputOutputArrayOfArrays":
-                            arg_type = "vector_Mat"
+                            arg_type = vector_mat
                             modlist.append("/IO")
                         elif arg_type == "OutputArrayOfArrays":
-                            arg_type = "vector_Mat"
+                            arg_type = vector_mat
                             modlist.append("/O")
-                        defval = self.batch_replace(defval, [("InputArrayOfArrays", "vector<Mat>"),
-                                                             ("InputOutputArrayOfArrays", "vector<Mat>"),
-                                                             ("OutputArrayOfArrays", "vector<Mat>"),
-                                                             ("InputArray", "Mat"),
-                                                             ("InputOutputArray", "Mat"),
-                                                             ("OutputArray", "Mat"),
+                        defval = self.batch_replace(defval, [("InputArrayOfArrays", vector_mat_template),
+                                                             ("InputOutputArrayOfArrays", vector_mat_template),
+                                                             ("OutputArrayOfArrays", vector_mat_template),
+                                                             ("InputArray", mat),
+                                                             ("InputOutputArray", mat),
+                                                             ("OutputArray", mat),
                                                              ("noArray", arg_type)]).strip()
                     args.append([arg_type, arg_name, defval, modlist])
                 npos = arg_start-1
@@ -604,7 +612,7 @@ class CppHeaderParser(object):
             n = "cv.Algorithm"
         return n
 
-    def parse_stmt(self, stmt, end_token):
+    def parse_stmt(self, stmt, end_token, use_umat=False):
         """
         parses the statement (ending with ';' or '}') or a block head (ending with '{')
 
@@ -696,7 +704,7 @@ class CppHeaderParser(object):
             # since we filtered off the other places where '(' can normally occur:
             #   - code blocks
             #   - function pointer typedef's
-            decl = self.parse_func_decl(stmt)
+            decl = self.parse_func_decl(stmt, use_umat=use_umat)
             # we return parse_flag == False to prevent the parser to look inside function/method bodies
             # (except for tracking the nested blocks)
             return stmt_type, "", False, decl
@@ -839,6 +847,15 @@ class CppHeaderParser(object):
                                 decls.append(d)
                         else:
                             decls.append(decl)
+
+                            if self._generate_umat_decls:
+                                # If function takes as one of arguments Mat or vector<Mat> - we want to create the
+                                # same declaration working with UMat (this is important for T-Api access)
+                                args = decl[3]
+                                has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0
+                                if has_mat:
+                                    _, _, _, umat_decl = self.parse_stmt(stmt, token, use_umat=True)
+                                    decls.append(umat_decl)
                     if stmt_type == "namespace":
                         chunks = [block[1] for block in self.block_stack if block[0] == 'namespace'] + [name]
                         self.namespaces.add('.'.join(chunks))
@@ -878,7 +895,7 @@ class CppHeaderParser(object):
                     print()
 
 if __name__ == '__main__':
-    parser = CppHeaderParser()
+    parser = CppHeaderParser(generate_umat_decls=True)
     decls = []
     for hname in opencv_hdr_list:
         decls += parser.parse(hname)

modules/python/test/test.py

@@ -123,6 +123,60 @@ class Hackathon244Tests(NewOpenCVTests):
         boost.getMaxDepth() # from ml::DTrees
         boost.isClassifier() # from ml::StatModel
 
+    def test_umat_matching(self):
+        img1 = self.get_sample("samples/data/right01.jpg")
+        img2 = self.get_sample("samples/data/right02.jpg")
+
+        orb = cv2.ORB_create()
+
+        img1, img2 = cv2.UMat(img1), cv2.UMat(img2)
+        ps1, descs_umat1 = orb.detectAndCompute(img1, None)
+        ps2, descs_umat2 = orb.detectAndCompute(img2, None)
+
+        self.assertIsInstance(descs_umat1, cv2.UMat)
+        self.assertIsInstance(descs_umat2, cv2.UMat)
+        self.assertGreater(len(ps1), 0)
+        self.assertGreater(len(ps2), 0)
+
+        bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
+
+        res_umats = bf.match(descs_umat1, descs_umat2)
+        res = bf.match(descs_umat1.get(), descs_umat2.get())
+
+        self.assertGreater(len(res), 0)
+        self.assertEqual(len(res_umats), len(res))
+
+    def test_umat_optical_flow(self):
+        img1 = self.get_sample("samples/data/right01.jpg", cv2.IMREAD_GRAYSCALE)
+        img2 = self.get_sample("samples/data/right02.jpg", cv2.IMREAD_GRAYSCALE)
+        # Note, that if you want to see performance boost by OCL implementation - you need enough data
+        # For example you can increase maxCorners param to 10000 and increase img1 and img2 in such way:
+        # img = np.hstack([np.vstack([img] * 6)] * 6)
+
+        feature_params = dict(maxCorners=239,
+                              qualityLevel=0.3,
+                              minDistance=7,
+                              blockSize=7)
+
+        p0 = cv2.goodFeaturesToTrack(img1, mask=None, **feature_params)
+        p0_umat = cv2.goodFeaturesToTrack(cv2.UMat(img1), mask=None, **feature_params)
+        self.assertEqual(p0_umat.get().shape, p0.shape)
+
+        p0 = np.array(sorted(p0, key=lambda p: tuple(p[0])))
+        p0_umat = cv2.UMat(np.array(sorted(p0_umat.get(), key=lambda p: tuple(p[0]))))
+        self.assertTrue(np.allclose(p0_umat.get(), p0))
+
+        p1_mask_err = cv2.calcOpticalFlowPyrLK(img1, img2, p0, None)
+
+        p1_mask_err_umat0 = map(cv2.UMat.get, cv2.calcOpticalFlowPyrLK(img1, img2, p0_umat, None))
+        p1_mask_err_umat1 = map(cv2.UMat.get, cv2.calcOpticalFlowPyrLK(cv2.UMat(img1), img2, p0_umat, None))
+        p1_mask_err_umat2 = map(cv2.UMat.get, cv2.calcOpticalFlowPyrLK(img1, cv2.UMat(img2), p0_umat, None))
+
+        # # results of OCL optical flow differs from CPU implementation, so result can not be easily compared
+        # for p1_mask_err_umat in [p1_mask_err_umat0, p1_mask_err_umat1, p1_mask_err_umat2]:
+        #     for data, data_umat in zip(p1_mask_err, p1_mask_err_umat):
+        #         self.assertTrue(np.allclose(data, data_umat))
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='run OpenCV python tests')
     parser.add_argument('--repo', help='use sample image files from local git repository (path to folder), '
@@ -139,4 +193,4 @@ if __name__ == '__main__':
         print('Missing opencv extra repository. Some of tests may fail.')
     random.seed(0)
     unit_argv = [sys.argv[0]] + other;
-    unittest.main(argv=unit_argv)
+    unittest.main(argv=unit_argv)