redesigned context sharing through texture sharing

modules/v4d/include/opencv2/v4d/v4d.hpp

@@ -53,6 +53,7 @@ namespace viz {
 namespace detail {
 class FrameBufferContext;
 class CLVAContext;
+class GLContext;
 class NanoVGContext;
 
 /*!
@@ -125,6 +126,7 @@ class CV_EXPORTS V4D {
     bool offscreen_;
     FrameBufferContext* mainFramebufferContext_ = nullptr;
     CLVAContext* clvaContext_ = nullptr;
+    GLContext* glContext_ = nullptr;
     NanoVGContext* nvgContext_ = nullptr;
     cv::VideoCapture* capture_ = nullptr;
     cv::VideoWriter* writer_ = nullptr;
@@ -426,11 +428,11 @@ private:
     void setMousePosition(int x, int y);
     nanogui::Screen& screen();
     FormHelper& form();
-    FrameBufferContext& fb();
-    CLVAContext& clva();
-    NanoVGContext& nvg();
+    FrameBufferContext& fbCtx();
+    CLVAContext& clvaCtx();
+    NanoVGContext& nvgCtx();
+    GLContext& glCtx();
     GLFWwindow* getGLFWWindow();
-    NVGcontext* getNVGcontext();
 };
 }
 } /* namespace kb */

modules/v4d/samples/cube-demo.cpp

@@ -72,15 +72,15 @@ static GLuint load_shader() {
     return cv::viz::init_shader(vert.c_str(), frag.c_str(), "fragColor");
 }
 
-static void init_scene() {
+static void init_scene(const cv::Size& sz) {
     glEnable (GL_DEPTH_TEST);
 
     float vertices[] = {
     // Front face
-            0.5, 0.5, 0.5, -0.5, 0.5, 0.5, -0.5, -0.5, 0.5, 0.5, -0.5, 0.5,
+            1.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0,
 
             // Back face
-            0.5, 0.5, -0.5, -0.5, 0.5, -0.5, -0.5, -0.5, -0.5, 0.5, -0.5, -0.5, };
+            1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, };
 
     float vertex_colors[] = { 1.0, 0.4, 0.6, 1.0, 0.9, 0.2, 0.7, 0.3, 0.8, 0.5, 0.3, 1.0,
 
@@ -137,6 +137,7 @@ static void init_scene() {
 
     shader_program = load_shader();
     uniform_transform = glGetUniformLocation(shader_program, "transform");
+    glViewport(0,0, sz.width, sz.height);
 }
 
 static void render_scene() {
@@ -148,19 +149,29 @@ static void render_scene() {
     float angle = fmod(double(cv::getTickCount()) / double(cv::getTickFrequency()), 2 * M_PI);
     float scale = 0.25;
 
-    cv::Matx44f scaleMat(scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0,
-            0.0, 1.0);
+    cv::Matx44f scaleMat(
+            scale, 0.0, 0.0, 0.0,
+            0.0, scale, 0.0, 0.0,
+            0.0, 0.0, scale, 0.0,
+            0.0, 0.0, 0.0, 1.0);
 
-    cv::Matx44f rotXMat(1.0, 0.0, 0.0, 0.0, 0.0, cos(angle), -sin(angle), 0.0, 0.0, sin(angle),
-            cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0);
+    cv::Matx44f rotXMat(
+            1.0, 0.0, 0.0, 0.0,
+            0.0, cos(angle), -sin(angle), 0.0,
+            0.0, sin(angle), cos(angle), 0.0,
+            0.0, 0.0, 0.0, 1.0);
 
-    cv::Matx44f rotYMat(cos(angle), 0.0, sin(angle), 0.0, 0.0, 1.0, 0.0, 0.0, -sin(angle), 0.0,
-            cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0);
+    cv::Matx44f rotYMat(
+            cos(angle), 0.0, sin(angle), 0.0,
+            0.0, 1.0, 0.0, 0.0,
+            -sin(angle), 0.0,cos(angle), 0.0,
+            0.0, 0.0, 0.0, 1.0);
 
     cv::Matx44f rotZMat(cos(angle), -sin(angle), 0.0, 0.0, sin(angle), cos(angle), 0.0, 0.0, 0.0,
             0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0);
 
     cv::Matx44f transform = scaleMat * rotXMat * rotYMat * rotZMat;
+
     glUniformMatrix4fv(uniform_transform, 1, GL_FALSE, transform.val);
     glBindVertexArray(vao);
     glDrawElements(GL_TRIANGLES, triangles * 3, GL_UNSIGNED_SHORT, NULL);

modules/v4d/samples/optflow-demo.cpp

@@ -423,8 +423,8 @@ static bool iteration() {
     //Detect trackable points in the motion mask
     detect_points(downMotionMaskGrey, detectedPoints);
 
+    v4d->clear();
     v4d->nvg([=]() {
-        v4d->clear();
         if (!downPrevGrey.empty()) {
             //We don't want the algorithm to get out of hand when there is a scene change, so we suppress it when we detect one.
             if (!detect_scene_change(downMotionMaskGrey, scene_change_thresh, scene_change_thresh_diff)) {

modules/v4d/samples/pedestrian-demo.cpp

@@ -3,14 +3,12 @@
 // of this distribution and at http://opencv.org/license.html.
 // Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
 
-#include "opencv2/v4d/v4d.hpp"
-#include "opencv2/v4d/util.hpp"
-
-#include <string>
-
+#include <opencv2/v4d/v4d.hpp>
 #include <opencv2/tracking.hpp>
 #include <opencv2/objdetect.hpp>
 
+#include <string>
+
 constexpr unsigned int WIDTH = 1920;
 constexpr unsigned int HEIGHT = 1080;
 const unsigned long DIAG = hypot(double(WIDTH), double(HEIGHT));
@@ -31,7 +29,7 @@ using std::endl;
 using std::vector;
 using std::string;
 
-static cv::Ptr<cv::viz::V4D> v4d = cv::viz::V4D::make(cv::Size(WIDTH, HEIGHT), cv::Size(WIDTH, HEIGHT), OFFSCREEN, "Beauty Demo");
+static cv::Ptr<cv::viz::V4D> v4d = cv::viz::V4D::make(cv::Size(WIDTH, HEIGHT), cv::Size(WIDTH, HEIGHT), OFFSCREEN, "Pedestrian Demo");
 static cv::HOGDescriptor hog;
 
 //adapted from cv::dnn_objdetect::InferBbox
@@ -178,10 +176,11 @@ static bool iteration() {
         }
     }
 
+    v4d->clear();
+
     v4d->nvg([&](const cv::Size& sz) {
         using namespace cv::viz::nvg;
 
-        v4d->clear();
         beginPath();
         strokeWidth(std::fmax(2.0, sz.width / 960.0));
         strokeColor(cv::viz::colorConvert(cv::Scalar(0, 127, 255, 200), cv::COLOR_HLS2BGR));

modules/v4d/samples/shader-demo.cpp

@@ -3,7 +3,8 @@
 // of this distribution and at http://opencv.org/license.html.
 // Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
 
-#include "opencv2/v4d/v4d.hpp"
+#include <opencv2/v4d/v4d.hpp>
+#include <opencv2/highgui.hpp>
 
 using std::cerr;
 using std::endl;
@@ -49,10 +50,8 @@ GLint zoom_hdl;
 /** shader and program handle **/
 GLuint shader_program_hdl;
 
-#ifndef OPENCV_V4D_USE_ES3
 //vertex array
 GLuint VAO;
-#endif
 GLuint VBO, EBO;
 
 // vertex position, color
@@ -68,10 +67,8 @@ unsigned int indices[] = {
 
 static void load_buffer_data() {
 
-#ifndef OPENCV_V4D_USE_ES3
     glGenVertexArrays(1, &VAO);
     glBindVertexArray(VAO);
-#endif
 
     glGenBuffers(1, &VBO);
     glGenBuffers(1, &EBO);
@@ -86,12 +83,10 @@ static void load_buffer_data() {
     glEnableVertexAttribArray(0);
 
     glBindBuffer(GL_ARRAY_BUFFER, 0);
-#ifndef OPENCV_V4D_USE_ES3
     glBindVertexArray(0);
-#endif
 }
 
-//workaround: required with emscripten + nanogui on every iteration before rendering
+//workaround: required with emscripten on every iteration before rendering
 static void rebind_buffers() {
     glBindBuffer(GL_ARRAY_BUFFER, VBO);
     glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
@@ -103,6 +98,7 @@ static void rebind_buffers() {
     glEnableVertexAttribArray(0);
 
     glBindBuffer(GL_ARRAY_BUFFER, 0);
+    glBindVertexArray(0);
 }
 
 //mandelbrot shader code adapted from my own project: https://github.com/kallaballa/FractalDive#after
@@ -163,23 +159,21 @@ static void load_shader() {
         return iterations;
     }
      
-    vec4 return_color()
+    void determine_color()
     {
         int iter = get_iterations();
-        if (iter == max_iterations) {   
-            return vec4(0.0f, 0.0f, 0.0f, 0.0f);
+        if (iter != max_iterations) {   
+            float iterations = float(iter) / float(max_iterations);
+            //convert to float
+            float cb = float(contrast_boost);
+    
+            outColor = vec4(base_color[0] * iterations * cb, base_color[1] * iterations * cb, base_color[2] * iterations * cb, base_color[3]);
         }
-
-        float iterations = float(iter) / float(max_iterations);
-        //convert to float
-        float cb = float(contrast_boost);
-
-        return vec4(base_color[0] * iterations * cb, base_color[1] * iterations * cb, base_color[2] * iterations * cb, base_color[3]);
     }
 
     void main()
     {
-        outColor = return_color();
+        determine_color();
     })";
 
     cerr << "##### Vertex Shader #####" << endl;
@@ -234,9 +228,7 @@ static void render_scene() {
         glUniform1f(zoom_hdl, zoom);
     }
 
-#ifndef OPENCV_V4D_USE_ES3
     glBindVertexArray(VAO);
-#endif
     glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
 }
 
@@ -335,9 +327,12 @@ static void setup_gui(cv::Ptr<cv::viz::V4D> v4dMain) {
 static bool iteration() {
     //ignore failed capture attempts
     v4d->capture();
-
+    v4d->fb([](cv::UMat& frameBuffer) {
+        imshow("fb1", frameBuffer);
+        cv::waitKey(1);
+    });
 #ifdef __EMSCRIPTEN__
-    //required in conjunction with emscripten + nanovg
+    //required in conjunction with emscripten
     rebind_buffers();
 #endif
     //Render using OpenGL

modules/v4d/samples/video-demo.cpp

@@ -140,8 +140,8 @@ static void init_scene() {
 }
 
 static void render_scene() {
-    glClearColor(0.1, 0.12, 0.2, 1);
-    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+//    glClearColor(0.1, 0.12, 0.2, 1);
+    glClear(GL_DEPTH_BUFFER_BIT);
 
     glUseProgram(shader_program);
 
@@ -194,6 +194,7 @@ static void glow_effect(const cv::UMat& src, cv::UMat& dst, const int ksize) {
 
 static bool iteration() {
     using namespace cv::viz;
+    v4d->capture();
 
     //Render using OpenGL
     v4d->gl(render_scene);

modules/v4d/src/detail/framebuffercontext.cpp

@@ -12,13 +12,13 @@ namespace cv {
 namespace viz {
 namespace detail {
 
-FrameBufferContext::FrameBufferContext(const FrameBufferContext& other) : FrameBufferContext(other.frameBufferSize_, true, other.title_, other.major_,  other.minor_, other.compat_, other.samples_, other.debug_) {
+FrameBufferContext::FrameBufferContext(const FrameBufferContext& other) : FrameBufferContext(other.frameBufferSize_, true, other.title_, other.major_,  other.minor_, other.compat_, other.samples_, other.debug_, other.glfwWindow_, other.textureID_) {
 }
 
 FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen,
-        const string& title, int major, int minor, bool compat, int samples, bool debug) :
+        const string& title, int major, int minor, bool compat, int samples, bool debug, GLFWwindow* sharedWindow, GLuint sharedTexture) :
         offscreen_(offscreen), title_(title), major_(major), minor_(
-                minor), compat_(compat), samples_(samples), debug_(debug), frameBufferSize_(frameBufferSize) {
+                minor), compat_(compat), samples_(samples), debug_(debug), frameBufferSize_(frameBufferSize), textureID_(sharedTexture) {
     if (glfwInit() != GLFW_TRUE)
         assert(false);
 
@@ -62,14 +62,10 @@ FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool off
 #endif
     glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
 
-    /* I figure we don't need double buffering because the FBO (and the bound texture) is our backbuffer that
-     * we blit to the front on every iteration.
-     * On X11, wayland and in WASM it works and boosts performance a bit.
-     */
-    glfwWindowHint(GLFW_DOUBLEBUFFER, GL_FALSE);
+//    glfwWindowHint(GLFW_DOUBLEBUFFER, GL_FALSE);
 
     glfwWindow_ = glfwCreateWindow(frameBufferSize.width, frameBufferSize.height, title_.c_str(), nullptr,
-            nullptr);
+            sharedWindow);
     if (glfwWindow_ == NULL) {
         assert(false);
     }
@@ -93,27 +89,48 @@ FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool off
 #else
     clglSharing_ = false;
 #endif
-    frameBufferID_ = 0;
-    GL_CHECK(glGenFramebuffers(1, &frameBufferID_));
-    GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_));
-    GL_CHECK(glGenRenderbuffers(1, &renderBufferID_));
-    textureID_ = 0;
-    GL_CHECK(glGenTextures(1, &textureID_));
-    GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_));
-    texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_);
-    GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1));
-    GL_CHECK(
-            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0));
-
-    GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_));
-    GL_CHECK(
-            glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height));
-    GL_CHECK(
-            glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_));
-
-    GL_CHECK(
-            glFramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, textureID_, 0));
-    assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE);
+    if(sharedWindow == nullptr) {
+        GL_CHECK(glGenFramebuffers(1, &frameBufferID_));
+        GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_));
+        GL_CHECK(glGenRenderbuffers(1, &renderBufferID_));
+
+        GL_CHECK(glGenTextures(1, &textureID_));
+        GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_));
+        texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_);
+        GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1));
+        GL_CHECK(
+                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0));
+
+        GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_));
+        GL_CHECK(
+                glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height));
+        GL_CHECK(
+                glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_));
+
+        GL_CHECK(
+                glNamedFramebufferTexture(frameBufferID_, GL_COLOR_ATTACHMENT0, textureID_, 0));
+        assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE);
+    } else {
+        GL_CHECK(glGenFramebuffers(1, &frameBufferID_));
+        GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_));
+
+        GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_));
+        texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_);
+        GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1));
+        GL_CHECK(
+                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0));
+
+        GL_CHECK(glGenRenderbuffers(1, &renderBufferID_));
+        GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_));
+        GL_CHECK(
+                glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height));
+        GL_CHECK(
+                glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_));
+
+        GL_CHECK(
+                glNamedFramebufferTexture(frameBufferID_, GL_COLOR_ATTACHMENT1, textureID_, 0));
+        assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE);
+    }
 #ifndef __EMSCRIPTEN__
     context_ = CLExecContext_t::getCurrent();
 #endif

modules/v4d/src/detail/framebuffercontext.hpp

@@ -44,6 +44,7 @@ typedef cv::ocl::OpenCLExecutionContextScope CLExecScope_t;
  */
 class FrameBufferContext {
     friend class CLVAContext;
+    friend class GLContext;
     friend class NanoVGContext;
     friend class cv::viz::V4D;
     bool offscreen_;
@@ -141,7 +142,7 @@ public:
      * @param frameBufferSize The frame buffer size.
      */
     FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen,
-            const string& title, int major, int minor, bool compat, int samples, bool debug);
+            const string& title, int major, int minor, bool compat, int samples, bool debug, GLFWwindow* sharedWindow, GLuint sharedTexture);
 
     FrameBufferContext(const FrameBufferContext& other);
 

modules/v4d/src/detail/glcontext.cpp

@@ -0,0 +1,29 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
+
+#include "glcontext.hpp"
+
+namespace cv {
+namespace viz {
+namespace detail {
+GLContext::GLContext(FrameBufferContext& fbContext) :
+        mainFbContext_(fbContext), glFbContext_(fbContext) {
+}
+
+void GLContext::render(std::function<void(const cv::Size&)> fn) {
+#ifndef __EMSCRIPTEN__
+    CLExecScope_t scope(glFbContext_.getCLExecContext());
+#endif
+    FrameBufferContext::GLScope glScope(glFbContext_);
+    fn(glFbContext_.getSize());
+}
+
+FrameBufferContext& GLContext::fbCtx() {
+    return glFbContext_;
+}
+
+}
+}
+}

modules/v4d/src/detail/glcontext.hpp

@@ -0,0 +1,59 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
+
+#ifndef SRC_OPENCV_GLCONTEXT_HPP_
+#define SRC_OPENCV_GLCONTEXT_HPP_
+
+#include "framebuffercontext.hpp"
+#include <nanogui/nanogui.h>
+
+#ifdef __EMSCRIPTEN__
+#include <emscripten.h>
+#endif
+
+#ifndef OPENCV_V4D_USE_ES3
+#define NANOGUI_USE_OPENGL
+#else
+#define NANOGUI_USE_GLES
+#define NANOGUI_GLES_VERSION 3
+#endif
+
+#include <nanogui/opengl.h>
+#include "opencv2/v4d/util.hpp"
+#include "opencv2/v4d/nvg.hpp"
+
+namespace cv {
+namespace viz {
+namespace detail {
+/*!
+ * Used to setup a nanovg context
+ */
+class GLContext {
+    FrameBufferContext& mainFbContext_;
+    FrameBufferContext glFbContext_;
+    cv::UMat preFB_;
+    cv::UMat fb_;
+    cv::UMat postFB_;
+public:
+     /*!
+     * Creates a OpenGL Context
+     * @param fbContext The framebuffer context
+     */
+    GLContext(FrameBufferContext& fbContext);
+    /*!
+     * Execute function object fn inside a gl context.
+     * The context takes care of setting up opengl states.
+     * @param fn A function that is passed the size of the framebuffer
+     * and performs drawing using opengl
+     */
+    void render(std::function<void(const cv::Size&)> fn);
+
+    FrameBufferContext& fbCtx();
+};
+}
+}
+}
+
+#endif /* SRC_OPENCV_GLCONTEXT_HPP_ */

modules/v4d/src/detail/nanovgcontext.cpp

@@ -8,61 +8,32 @@
 namespace cv {
 namespace viz {
 namespace detail {
-NanoVGContext::NanoVGContext(NVGcontext* context, FrameBufferContext& fbContext) :
-        context_(context), mainFbContext_(fbContext), nvgFbContext_(fbContext) {
-    //FIXME workaround for first frame color glitch
-    FrameBufferContext::GLScope glScope(mainFbContext_);
-    FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_);
+NanoVGContext::NanoVGContext(FrameBufferContext& fbContext) :
+        mainFbContext_(fbContext), nvgFbContext_(fbContext) {
+#ifndef __EMSCRIPTEN__
+    CLExecScope_t scope(nvgFbContext_.getCLExecContext());
+#endif
+    FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
+    FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_);
+    screen_ = new nanogui::Screen();
+    screen_->initialize(nvgFbContext_.getGLFWWindow(), false);
+    context_ = screen_->nvg_context();
 }
 
 void NanoVGContext::render(std::function<void(const cv::Size&)> fn) {
-    {
-#ifndef __EMSCRIPTEN__
-        CLExecScope_t scope(mainFbContext_.getCLExecContext());
-#endif
-        FrameBufferContext::GLScope mainGlScope(mainFbContext_);
-        FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_);
-        fb_.copyTo(preFB_);
-    }
-    {
-#ifndef __EMSCRIPTEN__
-        CLExecScope_t scope(nvgFbContext_.getCLExecContext());
-#endif
-        FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
-        FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_);
-        preFB_.copyTo(fb_);
-    }
-    {
 #ifndef __EMSCRIPTEN__
-        CLExecScope_t scope(nvgFbContext_.getCLExecContext());
+    CLExecScope_t scope(nvgFbContext_.getCLExecContext());
 #endif
-        FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
-        NanoVGContext::Scope nvgScope(*this);
-        cv::viz::nvg::detail::NVG::initializeContext(context_);
-        fn(nvgFbContext_.getSize());
-    }
-    {
-#ifndef __EMSCRIPTEN__
-        CLExecScope_t scope(nvgFbContext_.getCLExecContext());
-#endif
-        FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
-        FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_);
-        fb_.copyTo(postFB_);
-    }
-    {
-#ifndef __EMSCRIPTEN__
-        CLExecScope_t scope(mainFbContext_.getCLExecContext());
-#endif
-        FrameBufferContext::GLScope mainGlScope(mainFbContext_);
-        FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_);
-        postFB_.copyTo(fb_);
-    }
+    FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
+    NanoVGContext::Scope nvgScope(*this);
+    cv::viz::nvg::detail::NVG::initializeContext(context_);
+    fn(nvgFbContext_.getSize());
 }
 
 void NanoVGContext::begin() {
-    float w = mainFbContext_.getSize().width;
-    float h = mainFbContext_.getSize().height;
-    float r = mainFbContext_.getXPixelRatio();
+    float w = nvgFbContext_.getSize().width;
+    float h = nvgFbContext_.getSize().height;
+    float r = nvgFbContext_.getXPixelRatio();
 
     nvgSave(context_);
     nvgBeginFrame(context_, w, h, r);
@@ -77,6 +48,10 @@ void NanoVGContext::end() {
     nvgEndFrame(context_);
     nvgRestore(context_);
 }
+
+FrameBufferContext& NanoVGContext::fbCtx() {
+    return nvgFbContext_;
+}
 }
 }
 }

modules/v4d/src/detail/nanovgcontext.hpp

@@ -7,7 +7,6 @@
 #define SRC_OPENCV_NANOVGCONTEXT_HPP_
 
 #include "framebuffercontext.hpp"
-#include <nanogui/nanogui.h>
 
 #ifdef __EMSCRIPTEN__
 #include <emscripten.h>
@@ -31,9 +30,10 @@ namespace detail {
  * Used to setup a nanovg context
  */
 class NanoVGContext {
+    nanogui::Screen* screen_;
     NVGcontext* context_;
     FrameBufferContext& mainFbContext_;
-    FrameBufferContext& nvgFbContext_;
+    FrameBufferContext nvgFbContext_;
     cv::UMat preFB_;
     cv::UMat fb_;
     cv::UMat postFB_;
@@ -65,7 +65,7 @@ public:
      * @param context The native NVGContext
      * @param fbContext The framebuffer context
      */
-    NanoVGContext(NVGcontext* context, FrameBufferContext& fbContext);
+    NanoVGContext(FrameBufferContext& fbContext);
     /*!
      * Execute function object fn inside a nanovg context.
      * The context takes care of setting up opengl and nanovg states.
@@ -74,6 +74,8 @@ public:
      * and performs drawing using cv::viz::nvg
      */
     void render(std::function<void(const cv::Size&)> fn);
+
+    FrameBufferContext& fbCtx();
 private:
     /*!
      * Setup NanoVG context
@@ -83,6 +85,7 @@ private:
      * Tear down NanoVG context
      */
     void end();
+
 };
 }
 }

modules/v4d/src/v4d.cpp

@@ -6,6 +6,7 @@
 #include "opencv2/v4d/v4d.hpp"
 #include "detail/clvacontext.hpp"
 #include "detail/framebuffercontext.hpp"
+#include "detail/glcontext.hpp"
 #include "detail/nanovgcontext.hpp"
 
 namespace cv {
@@ -80,7 +81,7 @@ V4D::V4D(const cv::Size& size, const cv::Size& frameBufferSize, bool offscreen,
     assert(
             frameBufferSize_.width >= initialSize_.width
                     && frameBufferSize_.height >= initialSize_.height);
-    mainFramebufferContext_ = new detail::FrameBufferContext(this->getFrameBufferSize(), offscreen, title, major, minor, compat, samples, debug);
+    mainFramebufferContext_ = new detail::FrameBufferContext(this->getFrameBufferSize(), offscreen, title, major, minor, compat, samples, debug, nullptr, 0);
 
     if(!initializeGUI())
         assert(false);
@@ -94,6 +95,8 @@ V4D::~V4D() {
         delete writer_;
     if (capture_)
         delete capture_;
+    if (glContext_)
+        delete glContext_;
     if (nvgContext_)
         delete nvgContext_;
     if (clvaContext_)
@@ -156,7 +159,7 @@ bool V4D::initializeGUI() {
             std::vector<nanogui::Widget*> widgets;
             find_widgets(&v4d->screen(), widgets);
             for (auto* w : widgets) {
-                auto mousePos = nanogui::Vector2i(v4d->getMousePosition()[0] / v4d->fb().getXPixelRatio(), v4d->getMousePosition()[1] / v4d->fb().getYPixelRatio());
+                auto mousePos = nanogui::Vector2i(v4d->getMousePosition()[0] / v4d->fbCtx().getXPixelRatio(), v4d->getMousePosition()[1] / v4d->fbCtx().getYPixelRatio());
         if(contains_absolute(w, mousePos)) {
             v4d->screen().scroll_callback_event(x, y);
             return;
@@ -179,7 +182,8 @@ bool V4D::initializeGUI() {
         });
 
         clvaContext_ = new detail::CLVAContext(*mainFramebufferContext_);
-        nvgContext_ = new detail::NanoVGContext(getNVGcontext(), *mainFramebufferContext_);
+        glContext_ = new detail::GLContext(*mainFramebufferContext_);
+        nvgContext_ = new detail::NanoVGContext(*mainFramebufferContext_);
     } catch(std::exception& ex) {
         cerr << "V4D initialization failed: " << ex.what() << endl;
         return false;
@@ -207,63 +211,63 @@ bool V4D::keyboard_event(int key, int scancode, int action, int modifiers) {
     return screen().keyboard_event(key, scancode, action, modifiers);
 }
 
-FrameBufferContext& V4D::fb() {
+FrameBufferContext& V4D::fbCtx() {
     assert(mainFramebufferContext_ != nullptr);
     mainFramebufferContext_->makeCurrent();
     return *mainFramebufferContext_;
 }
 
-CLVAContext& V4D::clva() {
+CLVAContext& V4D::clvaCtx() {
     assert(clvaContext_ != nullptr);
     return *clvaContext_;
 }
 
-NanoVGContext& V4D::nvg() {
+NanoVGContext& V4D::nvgCtx() {
     assert(nvgContext_ != nullptr);
-    fb().makeCurrent();
+    nvgContext_->fbCtx().makeCurrent();
     return *nvgContext_;
 }
 
+GLContext& V4D::glCtx() {
+    assert(glContext_ != nullptr);
+    glContext_->fbCtx().makeCurrent();
+    return *glContext_;
+}
+
 nanogui::Screen& V4D::screen() {
     assert(screen_ != nullptr);
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     return *screen_;
 }
 
 cv::Size V4D::getVideoFrameSize() {
-    return clva().getVideoFrameSize();
+    return clvaCtx().getVideoFrameSize();
 }
 
 void V4D::gl(std::function<void()> fn) {
-#ifndef __EMSCRIPTEN__
-    detail::CLExecScope_t scope(fb().getCLExecContext());
-#endif
-    detail::FrameBufferContext::GLScope glScope(fb());
-    fn();
+    glCtx().render([=](const cv::Size& sz){
+        CV_UNUSED(sz);
+        fn();
+    });
 }
 
 void V4D::gl(std::function<void(const cv::Size&)> fn) {
-    auto fbSize = getFrameBufferSize();
-#ifndef __EMSCRIPTEN__
-    detail::CLExecScope_t scope(fb().getCLExecContext());
-#endif
-    detail::FrameBufferContext::GLScope glScope(fb());
-    fn(fbSize);
+    glCtx().render(fn);
 }
 
 void V4D::fb(std::function<void(cv::UMat&)> fn) {
-    fb().execute(fn);
+    fbCtx().execute(fn);
 }
 
 void V4D::nvg(std::function<void()> fn) {
-    nvg().render([=](const cv::Size& sz){
+    nvgCtx().render([=](const cv::Size& sz){
         CV_UNUSED(sz);
         fn();
     });
 }
 
 void V4D::nvg(std::function<void(const cv::Size&)> fn) {
-    nvg().render(fn);
+    nvgCtx().render(fn);
 }
 
 void V4D::nanogui(std::function<void(FormHelper& form)> fn) {
@@ -292,8 +296,8 @@ void V4D::run(std::function<bool()> fn) {
 }
 
 void V4D::setSource(const Source& src) {
-    if (!clva().hasContext())
-        clva().copyContext();
+    if (!clvaCtx().hasContext())
+        clvaCtx().copyContext();
     source_ = src;
 }
 
@@ -316,20 +320,17 @@ bool V4D::capture(std::function<void(cv::UMat&)> fn) {
             return false;
 
     if(nextReaderFrame_.empty()) {
-        if(!clva().capture(fn, nextReaderFrame_))
+        if(!clvaCtx().capture(fn, nextReaderFrame_))
             return false;
     }
 
     currentReaderFrame_ = nextReaderFrame_;
-    {
-        FrameBufferContext::GLScope glScope(*mainFramebufferContext_);
-        FrameBufferContext::FrameBufferScope fbScope(*mainFramebufferContext_, readerFrameBuffer_);
-
-        currentReaderFrame_.copyTo(readerFrameBuffer_);
-    }
+    fb([=,this](cv::UMat frameBuffer) {
+        currentReaderFrame_.copyTo(frameBuffer);
+    });
 
     futureReader_ = pool.push([=,this](){
-        return clva().capture(fn, nextReaderFrame_);
+        return clvaCtx().capture(fn, nextReaderFrame_);
     });
     return captureSuccessful_;
 }
@@ -339,8 +340,8 @@ bool V4D::isSourceReady() {
 }
 
 void V4D::setSink(const Sink& sink) {
-    if (!clva().hasContext())
-        clva().copyContext();
+    if (!clvaCtx().hasContext())
+        clvaCtx().copyContext();
     sink_ = sink;
 }
 
@@ -355,14 +356,11 @@ void V4D::write(std::function<void(const cv::UMat&)> fn) {
     if(futureWriter_.valid())
         futureWriter_.get();
 
-    {
-        FrameBufferContext::GLScope glScope(*mainFramebufferContext_);
-        FrameBufferContext::FrameBufferScope fbScope(*mainFramebufferContext_, writerFrameBuffer_);
-
-        writerFrameBuffer_.copyTo(currentWriterFrame_);
-    }
+    fb([=, this](cv::UMat frameBuffer) {
+        frameBuffer.copyTo(currentWriterFrame_);
+    });
     futureWriter_ = pool.push([=,this](){
-        clva().write(fn, currentWriterFrame_);
+        clvaCtx().write(fn, currentWriterFrame_);
     });
 }
 
@@ -371,12 +369,14 @@ bool V4D::isSinkReady() {
 }
 
 void V4D::clear(const cv::Scalar& bgra) {
-    const float& b = bgra[0] / 255.0f;
-    const float& g = bgra[1] / 255.0f;
-    const float& r = bgra[2] / 255.0f;
-    const float& a = bgra[3] / 255.0f;
-    GL_CHECK(glClearColor(r, g, b, a));
-    GL_CHECK(glClear(GL_COLOR_BUFFER_BIT));
+    this->gl([&](){
+        const float& b = bgra[0] / 255.0f;
+        const float& g = bgra[1] / 255.0f;
+        const float& r = bgra[2] / 255.0f;
+        const float& a = bgra[3] / 255.0f;
+        GL_CHECK(glClearColor(r, g, b, a));
+        GL_CHECK(glClear(GL_COLOR_BUFFER_BIT));
+    });
 }
 
 void V4D::showGui(bool s) {
@@ -465,7 +465,7 @@ void V4D::zoom(float factor) {
 }
 
 cv::Vec2f V4D::getPosition() {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     int x, y;
     glfwGetWindowPos(getGLFWWindow(), &x, &y);
     return {float(x), float(y)};
@@ -488,7 +488,7 @@ cv::Rect V4D::getViewport() {
 }
 
 cv::Size V4D::getNativeFrameBufferSize() {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     int w, h;
     glfwGetFramebufferSize(getGLFWWindow(), &w, &h);
     return {w, h};
@@ -499,7 +499,7 @@ cv::Size V4D::getFrameBufferSize() {
 }
 
 cv::Size V4D::getWindowSize() {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     int w, h;
     glfwGetWindowSize(getGLFWWindow(), &w, &h);
     return {w, h};
@@ -510,17 +510,17 @@ cv::Size V4D::getInitialSize() {
 }
 
 void V4D::setWindowSize(const cv::Size& sz) {
-    fb().makeCurrent();
-    screen().set_size(nanogui::Vector2i(sz.width / fb().getXPixelRatio(), sz.height / fb().getYPixelRatio()));
+    fbCtx().makeCurrent();
+    screen().set_size(nanogui::Vector2i(sz.width / fbCtx().getXPixelRatio(), sz.height / fbCtx().getYPixelRatio()));
 }
 
 bool V4D::isFullscreen() {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     return glfwGetWindowMonitor(getGLFWWindow()) != nullptr;
 }
 
 void V4D::setFullscreen(bool f) {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     auto monitor = glfwGetPrimaryMonitor();
     const GLFWvidmode* mode = glfwGetVideoMode(monitor);
     if (f) {
@@ -535,22 +535,22 @@ void V4D::setFullscreen(bool f) {
 }
 
 bool V4D::isResizable() {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     return glfwGetWindowAttrib(getGLFWWindow(), GLFW_RESIZABLE) == GLFW_TRUE;
 }
 
 void V4D::setResizable(bool r) {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     glfwWindowHint(GLFW_RESIZABLE, r ? GLFW_TRUE : GLFW_FALSE);
 }
 
 bool V4D::isVisible() {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     return glfwGetWindowAttrib(getGLFWWindow(), GLFW_VISIBLE) == GLFW_TRUE;
 }
 
 void V4D::setVisible(bool v) {
-    fb().makeCurrent();
+    fbCtx().makeCurrent();
     glfwWindowHint(GLFW_VISIBLE, v ? GLFW_TRUE : GLFW_FALSE);
     screen().set_visible(v);
     screen().perform_layout();
@@ -595,7 +595,7 @@ void V4D::setDefaultKeyboardEventCallback() {
 bool V4D::display() {
     bool result = true;
     if (!offscreen_) {
-        fb().makeCurrent();
+        fbCtx().makeCurrent();
         screen().draw_contents();
 #ifndef __EMSCRIPTEN__
         mainFramebufferContext_->blitFrameBufferToScreen(getViewport(), getWindowSize(), isStretching());
@@ -605,6 +605,7 @@ bool V4D::display() {
         screen().draw_widgets();
         glfwSwapBuffers(getGLFWWindow());
         glfwPollEvents();
+
         result = !glfwWindowShouldClose(getGLFWWindow());
     }
 
@@ -619,13 +620,8 @@ void V4D::close() {
     setVisible(false);
     closed_ = true;
 }
-
 GLFWwindow* V4D::getGLFWWindow() {
-    return mainFramebufferContext_->getGLFWWindow();
-}
-
-NVGcontext* V4D::getNVGcontext() {
-    return screen().nvg_context();
+    return fbCtx().getGLFWWindow();
 }
 }
 }