Merge pull request #20611 from VadimLevin:dev/vlevin/pure-python-modules

* feat: OpenCV extension with pure Python modules

* feat: cv2 is now a Python package instead of extension module

Python package cv2 now can handle both Python and C extension modules
properly without additional "subfolders" like "_extra_py_code".

* feat: can call native function from its reimplementation in Python

modules/core/include/opencv2/core/bindings_utils.hpp

@@ -116,6 +116,12 @@ String dumpRange(const Range& argument)
     }
 }
 
+CV_WRAP static inline
+int testOverwriteNativeMethod(int argument)
+{
+    return argument;
+}
+
 CV_WRAP static inline
 String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3)
 {

modules/core/misc/python/package/utils/__init__.py

@@ -0,0 +1,14 @@
+from collections import namedtuple
+
+import cv2
+
+
+NativeMethodPatchedResult = namedtuple("NativeMethodPatchedResult",
+                                       ("py", "native"))
+
+
+def testOverwriteNativeMethod(arg):
+    return NativeMethodPatchedResult(
+        arg + 1,
+        cv2.utils._native.testOverwriteNativeMethod(arg)
+    )

modules/python/common.cmake

@@ -1,6 +1,31 @@
 # This file is included from a subdirectory
 set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}")
 
+function(ocv_add_python_files_from_path search_path)
+  file(GLOB_RECURSE extra_py_files
+       RELATIVE "${search_path}"
+       # Plain Python code
+       "${search_path}/*.py"
+       # Type annotations
+       "${search_path}/*.pyi"
+  )
+  message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}")
+  if(extra_py_files)
+    list(SORT extra_py_files)
+    foreach(filename ${extra_py_files})
+      get_filename_component(module "${filename}" DIRECTORY)
+      if(NOT ${module} IN_LIST extra_modules)
+        list(APPEND extra_modules ${module})
+      endif()
+      configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY)
+      install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python)
+    endforeach()
+    message(STATUS "Found ${extra_modules} Python modules from ${search_path}")
+  else()
+    message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files")
+  endif()
+endfunction()
+
 ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator)
 
 include_directories(SYSTEM
@@ -224,23 +249,15 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER)
     if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m}
         AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package"
     )
-      set(__base "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package")
-      file(GLOB_RECURSE extra_py_files
-          RELATIVE "${__base}"
-          "${__base}/**/*.py"
-      )
-      if(extra_py_files)
-        list(SORT extra_py_files)
-        foreach(f ${extra_py_files})
-          get_filename_component(__dir "${f}" DIRECTORY)
-          configure_file("${__base}/${f}" "${__loader_path}/cv2/_extra_py_code/${f}" COPYONLY)
-          install(FILES "${__base}/${f}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/_extra_py_code/${__dir}/" COMPONENT python)
-        endforeach()
-      else()
-        message(WARNING "Module ${m} has no .py files in misc/python/package")
-      endif()
+      ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package")
     endif()
   endforeach(m)
+
+  if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "")
+    foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH})
+      ocv_add_python_files_from_path(${extra_ocv_py_modules_path})
+    endforeach()
+  endif()
 endif()  # NOT OPENCV_SKIP_PYTHON_LOADER
 
 unset(PYTHON_SRC_DIR)

modules/python/package/cv2/__init__.py

@@ -2,6 +2,7 @@
 OpenCV Python binary extension loader
 '''
 import os
+import importlib
 import sys
 
 __all__ = []
@@ -15,16 +16,53 @@ except ImportError:
     print('    pip install numpy')
     raise
 
+# TODO
+# is_x64 = sys.maxsize > 2**32
+
 
-py_code_loader = None
-if sys.version_info[:2] >= (3, 0):
+def __load_extra_py_code_for_module(base, name, enable_debug_print=False):
+    module_name = "{}.{}".format(__name__, name)
+    export_module_name = "{}.{}".format(base, name)
+    native_module = sys.modules.pop(module_name, None)
     try:
-        from . import _extra_py_code as py_code_loader
-    except:
-        pass
+        py_module = importlib.import_module(module_name)
+    except ImportError as err:
+        if enable_debug_print:
+            print("Can't load Python code for module:", module_name,
+                  ". Reason:", err)
+        # Extension doesn't contain extra py code
+        return False
+
+    if not hasattr(base, name):
+        setattr(sys.modules[base], name, py_module)
+    sys.modules[export_module_name] = py_module
+    # If it is C extension module it is already loaded by cv2 package
+    if native_module:
+        setattr(py_module, "_native", native_module)
+        for k, v in filter(lambda kv: not hasattr(py_module, kv[0]),
+                           native_module.__dict__.items()):
+            if enable_debug_print: print('    symbol: {} = {}'.format(k, v))
+            setattr(py_module, k, v)
+    return True
+
+
+def __collect_extra_submodules(enable_debug_print=False):
+    def modules_filter(module):
+        return all((
+             # module is not internal
+             not module.startswith("_"),
+             # it is not a file
+             os.path.isdir(os.path.join(_extra_submodules_init_path, module))
+        ))
+    if sys.version_info[0] < 3:
+        if enable_debug_print:
+            print("Extra submodules is loaded only for Python 3")
+        return []
+
+    __INIT_FILE_PATH = os.path.abspath(__file__)
+    _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH)
+    return filter(modules_filter, os.listdir(_extra_submodules_init_path))
 
-# TODO
-# is_x64 = sys.maxsize > 2**32
 
 def bootstrap():
     import sys
@@ -107,23 +145,36 @@ def bootstrap():
         # amending of LD_LIBRARY_PATH works for sub-processes only
         os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '')
 
-    if DEBUG: print('OpenCV loader: replacing cv2 module')
-    del sys.modules['cv2']
-    import cv2
+    if DEBUG: print("Relink everything from native cv2 module to cv2 package")
+
+    py_module = sys.modules.pop("cv2")
+
+    native_module = importlib.import_module("cv2")
+
+    sys.modules["cv2"] = py_module
+    setattr(py_module, "_native", native_module)
+
+    for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__",
+                                                           "__name__", "__package__"),
+                                  native_module.__dict__.items()):
+        if item_name not in g_vars:
+            g_vars[item_name] = item
 
     sys.path = save_sys_path  # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502)
 
     try:
-        import sys
         del sys.OpenCV_LOADER
-    except:
-        pass
+    except Exception as e:
+        if DEBUG:
+            print("Exception during delete OpenCV_LOADER:", e)
 
     if DEBUG: print('OpenCV loader: binary extension... OK')
 
-    if py_code_loader:
-        py_code_loader.init('cv2')
+    for submodule in __collect_extra_submodules(DEBUG):
+        if __load_extra_py_code_for_module("cv2", submodule, DEBUG):
+            if DEBUG: print("Extra Python code for", submodule, "is loaded")
 
     if DEBUG: print('OpenCV loader: DONE')
 
+
 bootstrap()

modules/python/package/cv2/_extra_py_code/__init__.py

@@ -1,53 +0,0 @@
-import sys
-import importlib
-
-__all__ = ['init']
-
-
-DEBUG = False
-if hasattr(sys, 'OpenCV_LOADER_DEBUG'):
-    DEBUG = True
-
-
-def _load_py_code(base, name):
-    try:
-        m = importlib.import_module(__name__ + name)
-    except ImportError:
-        return  # extension doesn't exist?
-
-    if DEBUG: print('OpenCV loader: added python code extension for: ' + name)
-
-    if hasattr(m, '__all__'):
-        export_members = { k : getattr(m, k) for k in m.__all__ }
-    else:
-        export_members = m.__dict__
-
-    for k, v in export_members.items():
-        if k.startswith('_'):  # skip internals
-            continue
-        if isinstance(v, type(sys)):  # don't bring modules
-            continue
-        if DEBUG: print('    symbol: {} = {}'.format(k, v))
-        setattr(sys.modules[base + name ], k, v)
-
-    del sys.modules[__name__ + name]
-
-
-# TODO: listdir
-def init(base):
-    _load_py_code(base, '.cv2')  # special case
-    prefix = base
-    prefix_len = len(prefix)
-
-    modules = [ m for m in sys.modules.keys() if m.startswith(prefix) ]
-    for m in modules:
-        m2 = m[prefix_len:]  # strip prefix
-        if len(m2) == 0:
-            continue
-        if m2.startswith('._'):  # skip internals
-            continue
-        if m2.startswith('.load_config_'):  # skip helper files
-            continue
-        _load_py_code(base, m2)
-
-    del sys.modules[__name__]

modules/python/package/extra_modules/misc/__init__.py

@@ -0,0 +1 @@
+from .version import get_ocv_version

modules/python/package/extra_modules/misc/version.py

@@ -0,0 +1,5 @@
+import cv2
+
+
+def get_ocv_version():
+    return getattr(cv2, "__version__", "unavailable")

modules/python/python3/CMakeLists.txt

@@ -15,9 +15,23 @@ set(the_description "The python3 bindings")
 set(MODULE_NAME python3)
 set(MODULE_INSTALL_SUBDIR python3)
 
+set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules)
+set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH})
+
+if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "")
+  set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path})
+else()
+  list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path})
+endif()
+
+unset(_ocv_extra_modules_path)
+
 set(PYTHON PYTHON3)
 
 include(../common.cmake)
 
+set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path})
+
+unset(_old_ocv_python_extra_modules_path)
 unset(MODULE_NAME)
 unset(MODULE_INSTALL_SUBDIR)

modules/python/python_loader.cmake

@@ -25,7 +25,6 @@ endif()
 set(PYTHON_LOADER_FILES
     "setup.py" "cv2/__init__.py"
     "cv2/load_config_py2.py" "cv2/load_config_py3.py"
-    "cv2/_extra_py_code/__init__.py"
 )
 foreach(fname ${PYTHON_LOADER_FILES})
   get_filename_component(__dir "${fname}" DIRECTORY)

modules/python/test/test_misc.py

@@ -4,6 +4,12 @@ from __future__ import print_function
 import ctypes
 from functools import partial
 from collections import namedtuple
+import sys
+
+if sys.version_info[0] < 3:
+    from collections import Sequence
+else:
+    from collections.abc import Sequence
 
 import numpy as np
 import cv2 as cv
@@ -585,6 +591,31 @@ class Arguments(NewOpenCVTests):
         self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.")
 
 
+class CanUsePurePythonModuleFunction(NewOpenCVTests):
+    def test_can_get_ocv_version(self):
+        import sys
+        if sys.version_info[0] < 3:
+            raise unittest.SkipTest('Python 2.x is not supported')
+
+        self.assertEqual(cv.misc.get_ocv_version(), cv.__version__,
+                         "Can't get package version using Python misc module")
+
+    def test_native_method_can_be_patched(self):
+        import sys
+
+        if sys.version_info[0] < 3:
+            raise unittest.SkipTest('Python 2.x is not supported')
+
+        res = cv.utils.testOverwriteNativeMethod(10)
+        self.assertTrue(isinstance(res, Sequence),
+                        msg="Overwritten method should return sequence. "
+                            "Got: {} of type {}".format(res, type(res)))
+        self.assertSequenceEqual(res, (11, 10),
+                                 msg="Failed to overwrite native method")
+        res = cv.utils._native.testOverwriteNativeMethod(123)
+        self.assertEqual(res, 123, msg="Failed to call native method implementation")
+
+
 class SamplesFindFile(NewOpenCVTests):
 
     def test_ExistedFile(self):