diff --git a/modules/v4d/include/opencv2/v4d/v4d.hpp b/modules/v4d/include/opencv2/v4d/v4d.hpp index 527369590..4dc5b1c0b 100644 --- a/modules/v4d/include/opencv2/v4d/v4d.hpp +++ b/modules/v4d/include/opencv2/v4d/v4d.hpp @@ -53,6 +53,7 @@ namespace viz { namespace detail { class FrameBufferContext; class CLVAContext; +class GLContext; class NanoVGContext; /*! @@ -125,6 +126,7 @@ class CV_EXPORTS V4D { bool offscreen_; FrameBufferContext* mainFramebufferContext_ = nullptr; CLVAContext* clvaContext_ = nullptr; + GLContext* glContext_ = nullptr; NanoVGContext* nvgContext_ = nullptr; cv::VideoCapture* capture_ = nullptr; cv::VideoWriter* writer_ = nullptr; @@ -426,11 +428,11 @@ private: void setMousePosition(int x, int y); nanogui::Screen& screen(); FormHelper& form(); - FrameBufferContext& fb(); - CLVAContext& clva(); - NanoVGContext& nvg(); + FrameBufferContext& fbCtx(); + CLVAContext& clvaCtx(); + NanoVGContext& nvgCtx(); + GLContext& glCtx(); GLFWwindow* getGLFWWindow(); - NVGcontext* getNVGcontext(); }; } } /* namespace kb */ diff --git a/modules/v4d/samples/cube-demo.cpp b/modules/v4d/samples/cube-demo.cpp index afc7a75d2..0c1a49f1d 100644 --- a/modules/v4d/samples/cube-demo.cpp +++ b/modules/v4d/samples/cube-demo.cpp @@ -72,15 +72,15 @@ static GLuint load_shader() { return cv::viz::init_shader(vert.c_str(), frag.c_str(), "fragColor"); } -static void init_scene() { +static void init_scene(const cv::Size& sz) { glEnable (GL_DEPTH_TEST); float vertices[] = { // Front face - 0.5, 0.5, 0.5, -0.5, 0.5, 0.5, -0.5, -0.5, 0.5, 0.5, -0.5, 0.5, + 1.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, // Back face - 0.5, 0.5, -0.5, -0.5, 0.5, -0.5, -0.5, -0.5, -0.5, 0.5, -0.5, -0.5, }; + 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, }; float vertex_colors[] = { 1.0, 0.4, 0.6, 1.0, 0.9, 0.2, 0.7, 0.3, 0.8, 0.5, 0.3, 1.0, @@ -137,6 +137,7 @@ static void init_scene() { shader_program = load_shader(); uniform_transform = glGetUniformLocation(shader_program, "transform"); + glViewport(0,0, sz.width, sz.height); } static void render_scene() { @@ -148,19 +149,29 @@ static void render_scene() { float angle = fmod(double(cv::getTickCount()) / double(cv::getTickFrequency()), 2 * M_PI); float scale = 0.25; - cv::Matx44f scaleMat(scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, - 0.0, 1.0); + cv::Matx44f scaleMat( + scale, 0.0, 0.0, 0.0, + 0.0, scale, 0.0, 0.0, + 0.0, 0.0, scale, 0.0, + 0.0, 0.0, 0.0, 1.0); - cv::Matx44f rotXMat(1.0, 0.0, 0.0, 0.0, 0.0, cos(angle), -sin(angle), 0.0, 0.0, sin(angle), - cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0); + cv::Matx44f rotXMat( + 1.0, 0.0, 0.0, 0.0, + 0.0, cos(angle), -sin(angle), 0.0, + 0.0, sin(angle), cos(angle), 0.0, + 0.0, 0.0, 0.0, 1.0); - cv::Matx44f rotYMat(cos(angle), 0.0, sin(angle), 0.0, 0.0, 1.0, 0.0, 0.0, -sin(angle), 0.0, - cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0); + cv::Matx44f rotYMat( + cos(angle), 0.0, sin(angle), 0.0, + 0.0, 1.0, 0.0, 0.0, + -sin(angle), 0.0,cos(angle), 0.0, + 0.0, 0.0, 0.0, 1.0); cv::Matx44f rotZMat(cos(angle), -sin(angle), 0.0, 0.0, sin(angle), cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0); cv::Matx44f transform = scaleMat * rotXMat * rotYMat * rotZMat; + glUniformMatrix4fv(uniform_transform, 1, GL_FALSE, transform.val); glBindVertexArray(vao); glDrawElements(GL_TRIANGLES, triangles * 3, GL_UNSIGNED_SHORT, NULL); diff --git a/modules/v4d/samples/optflow-demo.cpp b/modules/v4d/samples/optflow-demo.cpp index fdb22d331..667cc32b2 100644 --- a/modules/v4d/samples/optflow-demo.cpp +++ b/modules/v4d/samples/optflow-demo.cpp @@ -423,8 +423,8 @@ static bool iteration() { //Detect trackable points in the motion mask detect_points(downMotionMaskGrey, detectedPoints); + v4d->clear(); v4d->nvg([=]() { - v4d->clear(); if (!downPrevGrey.empty()) { //We don't want the algorithm to get out of hand when there is a scene change, so we suppress it when we detect one. if (!detect_scene_change(downMotionMaskGrey, scene_change_thresh, scene_change_thresh_diff)) { diff --git a/modules/v4d/samples/pedestrian-demo.cpp b/modules/v4d/samples/pedestrian-demo.cpp index db9789bdc..64076417c 100644 --- a/modules/v4d/samples/pedestrian-demo.cpp +++ b/modules/v4d/samples/pedestrian-demo.cpp @@ -3,14 +3,12 @@ // of this distribution and at http://opencv.org/license.html. // Copyright Amir Hassan (kallaballa) -#include "opencv2/v4d/v4d.hpp" -#include "opencv2/v4d/util.hpp" - -#include - +#include #include #include +#include + constexpr unsigned int WIDTH = 1920; constexpr unsigned int HEIGHT = 1080; const unsigned long DIAG = hypot(double(WIDTH), double(HEIGHT)); @@ -31,7 +29,7 @@ using std::endl; using std::vector; using std::string; -static cv::Ptr v4d = cv::viz::V4D::make(cv::Size(WIDTH, HEIGHT), cv::Size(WIDTH, HEIGHT), OFFSCREEN, "Beauty Demo"); +static cv::Ptr v4d = cv::viz::V4D::make(cv::Size(WIDTH, HEIGHT), cv::Size(WIDTH, HEIGHT), OFFSCREEN, "Pedestrian Demo"); static cv::HOGDescriptor hog; //adapted from cv::dnn_objdetect::InferBbox @@ -178,10 +176,11 @@ static bool iteration() { } } + v4d->clear(); + v4d->nvg([&](const cv::Size& sz) { using namespace cv::viz::nvg; - v4d->clear(); beginPath(); strokeWidth(std::fmax(2.0, sz.width / 960.0)); strokeColor(cv::viz::colorConvert(cv::Scalar(0, 127, 255, 200), cv::COLOR_HLS2BGR)); diff --git a/modules/v4d/samples/shader-demo.cpp b/modules/v4d/samples/shader-demo.cpp index 109e56c32..43081f6eb 100644 --- a/modules/v4d/samples/shader-demo.cpp +++ b/modules/v4d/samples/shader-demo.cpp @@ -3,7 +3,8 @@ // of this distribution and at http://opencv.org/license.html. // Copyright Amir Hassan (kallaballa) -#include "opencv2/v4d/v4d.hpp" +#include +#include using std::cerr; using std::endl; @@ -49,10 +50,8 @@ GLint zoom_hdl; /** shader and program handle **/ GLuint shader_program_hdl; -#ifndef OPENCV_V4D_USE_ES3 //vertex array GLuint VAO; -#endif GLuint VBO, EBO; // vertex position, color @@ -68,10 +67,8 @@ unsigned int indices[] = { static void load_buffer_data() { -#ifndef OPENCV_V4D_USE_ES3 glGenVertexArrays(1, &VAO); glBindVertexArray(VAO); -#endif glGenBuffers(1, &VBO); glGenBuffers(1, &EBO); @@ -86,12 +83,10 @@ static void load_buffer_data() { glEnableVertexAttribArray(0); glBindBuffer(GL_ARRAY_BUFFER, 0); -#ifndef OPENCV_V4D_USE_ES3 glBindVertexArray(0); -#endif } -//workaround: required with emscripten + nanogui on every iteration before rendering +//workaround: required with emscripten on every iteration before rendering static void rebind_buffers() { glBindBuffer(GL_ARRAY_BUFFER, VBO); glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); @@ -103,6 +98,7 @@ static void rebind_buffers() { glEnableVertexAttribArray(0); glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindVertexArray(0); } //mandelbrot shader code adapted from my own project: https://github.com/kallaballa/FractalDive#after @@ -163,23 +159,21 @@ static void load_shader() { return iterations; } - vec4 return_color() + void determine_color() { int iter = get_iterations(); - if (iter == max_iterations) { - return vec4(0.0f, 0.0f, 0.0f, 0.0f); + if (iter != max_iterations) { + float iterations = float(iter) / float(max_iterations); + //convert to float + float cb = float(contrast_boost); + + outColor = vec4(base_color[0] * iterations * cb, base_color[1] * iterations * cb, base_color[2] * iterations * cb, base_color[3]); } - - float iterations = float(iter) / float(max_iterations); - //convert to float - float cb = float(contrast_boost); - - return vec4(base_color[0] * iterations * cb, base_color[1] * iterations * cb, base_color[2] * iterations * cb, base_color[3]); } void main() { - outColor = return_color(); + determine_color(); })"; cerr << "##### Vertex Shader #####" << endl; @@ -234,9 +228,7 @@ static void render_scene() { glUniform1f(zoom_hdl, zoom); } -#ifndef OPENCV_V4D_USE_ES3 glBindVertexArray(VAO); -#endif glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0); } @@ -335,9 +327,12 @@ static void setup_gui(cv::Ptr v4dMain) { static bool iteration() { //ignore failed capture attempts v4d->capture(); - + v4d->fb([](cv::UMat& frameBuffer) { + imshow("fb1", frameBuffer); + cv::waitKey(1); + }); #ifdef __EMSCRIPTEN__ - //required in conjunction with emscripten + nanovg + //required in conjunction with emscripten rebind_buffers(); #endif //Render using OpenGL diff --git a/modules/v4d/samples/video-demo.cpp b/modules/v4d/samples/video-demo.cpp index 28f16109d..619f3d8aa 100644 --- a/modules/v4d/samples/video-demo.cpp +++ b/modules/v4d/samples/video-demo.cpp @@ -140,8 +140,8 @@ static void init_scene() { } static void render_scene() { - glClearColor(0.1, 0.12, 0.2, 1); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); +// glClearColor(0.1, 0.12, 0.2, 1); + glClear(GL_DEPTH_BUFFER_BIT); glUseProgram(shader_program); @@ -194,6 +194,7 @@ static void glow_effect(const cv::UMat& src, cv::UMat& dst, const int ksize) { static bool iteration() { using namespace cv::viz; + v4d->capture(); //Render using OpenGL v4d->gl(render_scene); diff --git a/modules/v4d/src/detail/framebuffercontext.cpp b/modules/v4d/src/detail/framebuffercontext.cpp index b0f48316b..92b01bf8a 100644 --- a/modules/v4d/src/detail/framebuffercontext.cpp +++ b/modules/v4d/src/detail/framebuffercontext.cpp @@ -12,13 +12,13 @@ namespace cv { namespace viz { namespace detail { -FrameBufferContext::FrameBufferContext(const FrameBufferContext& other) : FrameBufferContext(other.frameBufferSize_, true, other.title_, other.major_, other.minor_, other.compat_, other.samples_, other.debug_) { +FrameBufferContext::FrameBufferContext(const FrameBufferContext& other) : FrameBufferContext(other.frameBufferSize_, true, other.title_, other.major_, other.minor_, other.compat_, other.samples_, other.debug_, other.glfwWindow_, other.textureID_) { } FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen, - const string& title, int major, int minor, bool compat, int samples, bool debug) : + const string& title, int major, int minor, bool compat, int samples, bool debug, GLFWwindow* sharedWindow, GLuint sharedTexture) : offscreen_(offscreen), title_(title), major_(major), minor_( - minor), compat_(compat), samples_(samples), debug_(debug), frameBufferSize_(frameBufferSize) { + minor), compat_(compat), samples_(samples), debug_(debug), frameBufferSize_(frameBufferSize), textureID_(sharedTexture) { if (glfwInit() != GLFW_TRUE) assert(false); @@ -62,14 +62,10 @@ FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool off #endif glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE); - /* I figure we don't need double buffering because the FBO (and the bound texture) is our backbuffer that - * we blit to the front on every iteration. - * On X11, wayland and in WASM it works and boosts performance a bit. - */ - glfwWindowHint(GLFW_DOUBLEBUFFER, GL_FALSE); +// glfwWindowHint(GLFW_DOUBLEBUFFER, GL_FALSE); glfwWindow_ = glfwCreateWindow(frameBufferSize.width, frameBufferSize.height, title_.c_str(), nullptr, - nullptr); + sharedWindow); if (glfwWindow_ == NULL) { assert(false); } @@ -93,27 +89,48 @@ FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool off #else clglSharing_ = false; #endif - frameBufferID_ = 0; - GL_CHECK(glGenFramebuffers(1, &frameBufferID_)); - GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_)); - GL_CHECK(glGenRenderbuffers(1, &renderBufferID_)); - textureID_ = 0; - GL_CHECK(glGenTextures(1, &textureID_)); - GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_)); - texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_); - GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1)); - GL_CHECK( - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0)); - - GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_)); - GL_CHECK( - glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height)); - GL_CHECK( - glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_)); - - GL_CHECK( - glFramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, textureID_, 0)); - assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE); + if(sharedWindow == nullptr) { + GL_CHECK(glGenFramebuffers(1, &frameBufferID_)); + GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_)); + GL_CHECK(glGenRenderbuffers(1, &renderBufferID_)); + + GL_CHECK(glGenTextures(1, &textureID_)); + GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_)); + texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_); + GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1)); + GL_CHECK( + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0)); + + GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_)); + GL_CHECK( + glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height)); + GL_CHECK( + glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_)); + + GL_CHECK( + glNamedFramebufferTexture(frameBufferID_, GL_COLOR_ATTACHMENT0, textureID_, 0)); + assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE); + } else { + GL_CHECK(glGenFramebuffers(1, &frameBufferID_)); + GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_)); + + GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_)); + texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_); + GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1)); + GL_CHECK( + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0)); + + GL_CHECK(glGenRenderbuffers(1, &renderBufferID_)); + GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_)); + GL_CHECK( + glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height)); + GL_CHECK( + glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_)); + + GL_CHECK( + glNamedFramebufferTexture(frameBufferID_, GL_COLOR_ATTACHMENT1, textureID_, 0)); + assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE); + } #ifndef __EMSCRIPTEN__ context_ = CLExecContext_t::getCurrent(); #endif diff --git a/modules/v4d/src/detail/framebuffercontext.hpp b/modules/v4d/src/detail/framebuffercontext.hpp index 628e22a48..1ba1a2919 100644 --- a/modules/v4d/src/detail/framebuffercontext.hpp +++ b/modules/v4d/src/detail/framebuffercontext.hpp @@ -44,6 +44,7 @@ typedef cv::ocl::OpenCLExecutionContextScope CLExecScope_t; */ class FrameBufferContext { friend class CLVAContext; + friend class GLContext; friend class NanoVGContext; friend class cv::viz::V4D; bool offscreen_; @@ -141,7 +142,7 @@ public: * @param frameBufferSize The frame buffer size. */ FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen, - const string& title, int major, int minor, bool compat, int samples, bool debug); + const string& title, int major, int minor, bool compat, int samples, bool debug, GLFWwindow* sharedWindow, GLuint sharedTexture); FrameBufferContext(const FrameBufferContext& other); diff --git a/modules/v4d/src/detail/glcontext.cpp b/modules/v4d/src/detail/glcontext.cpp new file mode 100644 index 000000000..9610c2efc --- /dev/null +++ b/modules/v4d/src/detail/glcontext.cpp @@ -0,0 +1,29 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright Amir Hassan (kallaballa) + +#include "glcontext.hpp" + +namespace cv { +namespace viz { +namespace detail { +GLContext::GLContext(FrameBufferContext& fbContext) : + mainFbContext_(fbContext), glFbContext_(fbContext) { +} + +void GLContext::render(std::function fn) { +#ifndef __EMSCRIPTEN__ + CLExecScope_t scope(glFbContext_.getCLExecContext()); +#endif + FrameBufferContext::GLScope glScope(glFbContext_); + fn(glFbContext_.getSize()); +} + +FrameBufferContext& GLContext::fbCtx() { + return glFbContext_; +} + +} +} +} diff --git a/modules/v4d/src/detail/glcontext.hpp b/modules/v4d/src/detail/glcontext.hpp new file mode 100644 index 000000000..e8802add3 --- /dev/null +++ b/modules/v4d/src/detail/glcontext.hpp @@ -0,0 +1,59 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright Amir Hassan (kallaballa) + +#ifndef SRC_OPENCV_GLCONTEXT_HPP_ +#define SRC_OPENCV_GLCONTEXT_HPP_ + +#include "framebuffercontext.hpp" +#include + +#ifdef __EMSCRIPTEN__ +#include +#endif + +#ifndef OPENCV_V4D_USE_ES3 +#define NANOGUI_USE_OPENGL +#else +#define NANOGUI_USE_GLES +#define NANOGUI_GLES_VERSION 3 +#endif + +#include +#include "opencv2/v4d/util.hpp" +#include "opencv2/v4d/nvg.hpp" + +namespace cv { +namespace viz { +namespace detail { +/*! + * Used to setup a nanovg context + */ +class GLContext { + FrameBufferContext& mainFbContext_; + FrameBufferContext glFbContext_; + cv::UMat preFB_; + cv::UMat fb_; + cv::UMat postFB_; +public: + /*! + * Creates a OpenGL Context + * @param fbContext The framebuffer context + */ + GLContext(FrameBufferContext& fbContext); + /*! + * Execute function object fn inside a gl context. + * The context takes care of setting up opengl states. + * @param fn A function that is passed the size of the framebuffer + * and performs drawing using opengl + */ + void render(std::function fn); + + FrameBufferContext& fbCtx(); +}; +} +} +} + +#endif /* SRC_OPENCV_GLCONTEXT_HPP_ */ diff --git a/modules/v4d/src/detail/nanovgcontext.cpp b/modules/v4d/src/detail/nanovgcontext.cpp index 2bcf549aa..7ab042b18 100644 --- a/modules/v4d/src/detail/nanovgcontext.cpp +++ b/modules/v4d/src/detail/nanovgcontext.cpp @@ -8,61 +8,32 @@ namespace cv { namespace viz { namespace detail { -NanoVGContext::NanoVGContext(NVGcontext* context, FrameBufferContext& fbContext) : - context_(context), mainFbContext_(fbContext), nvgFbContext_(fbContext) { - //FIXME workaround for first frame color glitch - FrameBufferContext::GLScope glScope(mainFbContext_); - FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_); +NanoVGContext::NanoVGContext(FrameBufferContext& fbContext) : + mainFbContext_(fbContext), nvgFbContext_(fbContext) { +#ifndef __EMSCRIPTEN__ + CLExecScope_t scope(nvgFbContext_.getCLExecContext()); +#endif + FrameBufferContext::GLScope nvgGlScope(nvgFbContext_); + FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_); + screen_ = new nanogui::Screen(); + screen_->initialize(nvgFbContext_.getGLFWWindow(), false); + context_ = screen_->nvg_context(); } void NanoVGContext::render(std::function fn) { - { -#ifndef __EMSCRIPTEN__ - CLExecScope_t scope(mainFbContext_.getCLExecContext()); -#endif - FrameBufferContext::GLScope mainGlScope(mainFbContext_); - FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_); - fb_.copyTo(preFB_); - } - { -#ifndef __EMSCRIPTEN__ - CLExecScope_t scope(nvgFbContext_.getCLExecContext()); -#endif - FrameBufferContext::GLScope nvgGlScope(nvgFbContext_); - FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_); - preFB_.copyTo(fb_); - } - { #ifndef __EMSCRIPTEN__ - CLExecScope_t scope(nvgFbContext_.getCLExecContext()); + CLExecScope_t scope(nvgFbContext_.getCLExecContext()); #endif - FrameBufferContext::GLScope nvgGlScope(nvgFbContext_); - NanoVGContext::Scope nvgScope(*this); - cv::viz::nvg::detail::NVG::initializeContext(context_); - fn(nvgFbContext_.getSize()); - } - { -#ifndef __EMSCRIPTEN__ - CLExecScope_t scope(nvgFbContext_.getCLExecContext()); -#endif - FrameBufferContext::GLScope nvgGlScope(nvgFbContext_); - FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_); - fb_.copyTo(postFB_); - } - { -#ifndef __EMSCRIPTEN__ - CLExecScope_t scope(mainFbContext_.getCLExecContext()); -#endif - FrameBufferContext::GLScope mainGlScope(mainFbContext_); - FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_); - postFB_.copyTo(fb_); - } + FrameBufferContext::GLScope nvgGlScope(nvgFbContext_); + NanoVGContext::Scope nvgScope(*this); + cv::viz::nvg::detail::NVG::initializeContext(context_); + fn(nvgFbContext_.getSize()); } void NanoVGContext::begin() { - float w = mainFbContext_.getSize().width; - float h = mainFbContext_.getSize().height; - float r = mainFbContext_.getXPixelRatio(); + float w = nvgFbContext_.getSize().width; + float h = nvgFbContext_.getSize().height; + float r = nvgFbContext_.getXPixelRatio(); nvgSave(context_); nvgBeginFrame(context_, w, h, r); @@ -77,6 +48,10 @@ void NanoVGContext::end() { nvgEndFrame(context_); nvgRestore(context_); } + +FrameBufferContext& NanoVGContext::fbCtx() { + return nvgFbContext_; +} } } } diff --git a/modules/v4d/src/detail/nanovgcontext.hpp b/modules/v4d/src/detail/nanovgcontext.hpp index b5a12e4d3..bc51b148e 100644 --- a/modules/v4d/src/detail/nanovgcontext.hpp +++ b/modules/v4d/src/detail/nanovgcontext.hpp @@ -7,7 +7,6 @@ #define SRC_OPENCV_NANOVGCONTEXT_HPP_ #include "framebuffercontext.hpp" -#include #ifdef __EMSCRIPTEN__ #include @@ -31,9 +30,10 @@ namespace detail { * Used to setup a nanovg context */ class NanoVGContext { + nanogui::Screen* screen_; NVGcontext* context_; FrameBufferContext& mainFbContext_; - FrameBufferContext& nvgFbContext_; + FrameBufferContext nvgFbContext_; cv::UMat preFB_; cv::UMat fb_; cv::UMat postFB_; @@ -65,7 +65,7 @@ public: * @param context The native NVGContext * @param fbContext The framebuffer context */ - NanoVGContext(NVGcontext* context, FrameBufferContext& fbContext); + NanoVGContext(FrameBufferContext& fbContext); /*! * Execute function object fn inside a nanovg context. * The context takes care of setting up opengl and nanovg states. @@ -74,6 +74,8 @@ public: * and performs drawing using cv::viz::nvg */ void render(std::function fn); + + FrameBufferContext& fbCtx(); private: /*! * Setup NanoVG context @@ -83,6 +85,7 @@ private: * Tear down NanoVG context */ void end(); + }; } } diff --git a/modules/v4d/src/v4d.cpp b/modules/v4d/src/v4d.cpp index b1e85ae52..c8b81675c 100644 --- a/modules/v4d/src/v4d.cpp +++ b/modules/v4d/src/v4d.cpp @@ -6,6 +6,7 @@ #include "opencv2/v4d/v4d.hpp" #include "detail/clvacontext.hpp" #include "detail/framebuffercontext.hpp" +#include "detail/glcontext.hpp" #include "detail/nanovgcontext.hpp" namespace cv { @@ -80,7 +81,7 @@ V4D::V4D(const cv::Size& size, const cv::Size& frameBufferSize, bool offscreen, assert( frameBufferSize_.width >= initialSize_.width && frameBufferSize_.height >= initialSize_.height); - mainFramebufferContext_ = new detail::FrameBufferContext(this->getFrameBufferSize(), offscreen, title, major, minor, compat, samples, debug); + mainFramebufferContext_ = new detail::FrameBufferContext(this->getFrameBufferSize(), offscreen, title, major, minor, compat, samples, debug, nullptr, 0); if(!initializeGUI()) assert(false); @@ -94,6 +95,8 @@ V4D::~V4D() { delete writer_; if (capture_) delete capture_; + if (glContext_) + delete glContext_; if (nvgContext_) delete nvgContext_; if (clvaContext_) @@ -156,7 +159,7 @@ bool V4D::initializeGUI() { std::vector widgets; find_widgets(&v4d->screen(), widgets); for (auto* w : widgets) { - auto mousePos = nanogui::Vector2i(v4d->getMousePosition()[0] / v4d->fb().getXPixelRatio(), v4d->getMousePosition()[1] / v4d->fb().getYPixelRatio()); + auto mousePos = nanogui::Vector2i(v4d->getMousePosition()[0] / v4d->fbCtx().getXPixelRatio(), v4d->getMousePosition()[1] / v4d->fbCtx().getYPixelRatio()); if(contains_absolute(w, mousePos)) { v4d->screen().scroll_callback_event(x, y); return; @@ -179,7 +182,8 @@ bool V4D::initializeGUI() { }); clvaContext_ = new detail::CLVAContext(*mainFramebufferContext_); - nvgContext_ = new detail::NanoVGContext(getNVGcontext(), *mainFramebufferContext_); + glContext_ = new detail::GLContext(*mainFramebufferContext_); + nvgContext_ = new detail::NanoVGContext(*mainFramebufferContext_); } catch(std::exception& ex) { cerr << "V4D initialization failed: " << ex.what() << endl; return false; @@ -207,63 +211,63 @@ bool V4D::keyboard_event(int key, int scancode, int action, int modifiers) { return screen().keyboard_event(key, scancode, action, modifiers); } -FrameBufferContext& V4D::fb() { +FrameBufferContext& V4D::fbCtx() { assert(mainFramebufferContext_ != nullptr); mainFramebufferContext_->makeCurrent(); return *mainFramebufferContext_; } -CLVAContext& V4D::clva() { +CLVAContext& V4D::clvaCtx() { assert(clvaContext_ != nullptr); return *clvaContext_; } -NanoVGContext& V4D::nvg() { +NanoVGContext& V4D::nvgCtx() { assert(nvgContext_ != nullptr); - fb().makeCurrent(); + nvgContext_->fbCtx().makeCurrent(); return *nvgContext_; } +GLContext& V4D::glCtx() { + assert(glContext_ != nullptr); + glContext_->fbCtx().makeCurrent(); + return *glContext_; +} + nanogui::Screen& V4D::screen() { assert(screen_ != nullptr); - fb().makeCurrent(); + fbCtx().makeCurrent(); return *screen_; } cv::Size V4D::getVideoFrameSize() { - return clva().getVideoFrameSize(); + return clvaCtx().getVideoFrameSize(); } void V4D::gl(std::function fn) { -#ifndef __EMSCRIPTEN__ - detail::CLExecScope_t scope(fb().getCLExecContext()); -#endif - detail::FrameBufferContext::GLScope glScope(fb()); - fn(); + glCtx().render([=](const cv::Size& sz){ + CV_UNUSED(sz); + fn(); + }); } void V4D::gl(std::function fn) { - auto fbSize = getFrameBufferSize(); -#ifndef __EMSCRIPTEN__ - detail::CLExecScope_t scope(fb().getCLExecContext()); -#endif - detail::FrameBufferContext::GLScope glScope(fb()); - fn(fbSize); + glCtx().render(fn); } void V4D::fb(std::function fn) { - fb().execute(fn); + fbCtx().execute(fn); } void V4D::nvg(std::function fn) { - nvg().render([=](const cv::Size& sz){ + nvgCtx().render([=](const cv::Size& sz){ CV_UNUSED(sz); fn(); }); } void V4D::nvg(std::function fn) { - nvg().render(fn); + nvgCtx().render(fn); } void V4D::nanogui(std::function fn) { @@ -292,8 +296,8 @@ void V4D::run(std::function fn) { } void V4D::setSource(const Source& src) { - if (!clva().hasContext()) - clva().copyContext(); + if (!clvaCtx().hasContext()) + clvaCtx().copyContext(); source_ = src; } @@ -316,20 +320,17 @@ bool V4D::capture(std::function fn) { return false; if(nextReaderFrame_.empty()) { - if(!clva().capture(fn, nextReaderFrame_)) + if(!clvaCtx().capture(fn, nextReaderFrame_)) return false; } currentReaderFrame_ = nextReaderFrame_; - { - FrameBufferContext::GLScope glScope(*mainFramebufferContext_); - FrameBufferContext::FrameBufferScope fbScope(*mainFramebufferContext_, readerFrameBuffer_); - - currentReaderFrame_.copyTo(readerFrameBuffer_); - } + fb([=,this](cv::UMat frameBuffer) { + currentReaderFrame_.copyTo(frameBuffer); + }); futureReader_ = pool.push([=,this](){ - return clva().capture(fn, nextReaderFrame_); + return clvaCtx().capture(fn, nextReaderFrame_); }); return captureSuccessful_; } @@ -339,8 +340,8 @@ bool V4D::isSourceReady() { } void V4D::setSink(const Sink& sink) { - if (!clva().hasContext()) - clva().copyContext(); + if (!clvaCtx().hasContext()) + clvaCtx().copyContext(); sink_ = sink; } @@ -355,14 +356,11 @@ void V4D::write(std::function fn) { if(futureWriter_.valid()) futureWriter_.get(); - { - FrameBufferContext::GLScope glScope(*mainFramebufferContext_); - FrameBufferContext::FrameBufferScope fbScope(*mainFramebufferContext_, writerFrameBuffer_); - - writerFrameBuffer_.copyTo(currentWriterFrame_); - } + fb([=, this](cv::UMat frameBuffer) { + frameBuffer.copyTo(currentWriterFrame_); + }); futureWriter_ = pool.push([=,this](){ - clva().write(fn, currentWriterFrame_); + clvaCtx().write(fn, currentWriterFrame_); }); } @@ -371,12 +369,14 @@ bool V4D::isSinkReady() { } void V4D::clear(const cv::Scalar& bgra) { - const float& b = bgra[0] / 255.0f; - const float& g = bgra[1] / 255.0f; - const float& r = bgra[2] / 255.0f; - const float& a = bgra[3] / 255.0f; - GL_CHECK(glClearColor(r, g, b, a)); - GL_CHECK(glClear(GL_COLOR_BUFFER_BIT)); + this->gl([&](){ + const float& b = bgra[0] / 255.0f; + const float& g = bgra[1] / 255.0f; + const float& r = bgra[2] / 255.0f; + const float& a = bgra[3] / 255.0f; + GL_CHECK(glClearColor(r, g, b, a)); + GL_CHECK(glClear(GL_COLOR_BUFFER_BIT)); + }); } void V4D::showGui(bool s) { @@ -465,7 +465,7 @@ void V4D::zoom(float factor) { } cv::Vec2f V4D::getPosition() { - fb().makeCurrent(); + fbCtx().makeCurrent(); int x, y; glfwGetWindowPos(getGLFWWindow(), &x, &y); return {float(x), float(y)}; @@ -488,7 +488,7 @@ cv::Rect V4D::getViewport() { } cv::Size V4D::getNativeFrameBufferSize() { - fb().makeCurrent(); + fbCtx().makeCurrent(); int w, h; glfwGetFramebufferSize(getGLFWWindow(), &w, &h); return {w, h}; @@ -499,7 +499,7 @@ cv::Size V4D::getFrameBufferSize() { } cv::Size V4D::getWindowSize() { - fb().makeCurrent(); + fbCtx().makeCurrent(); int w, h; glfwGetWindowSize(getGLFWWindow(), &w, &h); return {w, h}; @@ -510,17 +510,17 @@ cv::Size V4D::getInitialSize() { } void V4D::setWindowSize(const cv::Size& sz) { - fb().makeCurrent(); - screen().set_size(nanogui::Vector2i(sz.width / fb().getXPixelRatio(), sz.height / fb().getYPixelRatio())); + fbCtx().makeCurrent(); + screen().set_size(nanogui::Vector2i(sz.width / fbCtx().getXPixelRatio(), sz.height / fbCtx().getYPixelRatio())); } bool V4D::isFullscreen() { - fb().makeCurrent(); + fbCtx().makeCurrent(); return glfwGetWindowMonitor(getGLFWWindow()) != nullptr; } void V4D::setFullscreen(bool f) { - fb().makeCurrent(); + fbCtx().makeCurrent(); auto monitor = glfwGetPrimaryMonitor(); const GLFWvidmode* mode = glfwGetVideoMode(monitor); if (f) { @@ -535,22 +535,22 @@ void V4D::setFullscreen(bool f) { } bool V4D::isResizable() { - fb().makeCurrent(); + fbCtx().makeCurrent(); return glfwGetWindowAttrib(getGLFWWindow(), GLFW_RESIZABLE) == GLFW_TRUE; } void V4D::setResizable(bool r) { - fb().makeCurrent(); + fbCtx().makeCurrent(); glfwWindowHint(GLFW_RESIZABLE, r ? GLFW_TRUE : GLFW_FALSE); } bool V4D::isVisible() { - fb().makeCurrent(); + fbCtx().makeCurrent(); return glfwGetWindowAttrib(getGLFWWindow(), GLFW_VISIBLE) == GLFW_TRUE; } void V4D::setVisible(bool v) { - fb().makeCurrent(); + fbCtx().makeCurrent(); glfwWindowHint(GLFW_VISIBLE, v ? GLFW_TRUE : GLFW_FALSE); screen().set_visible(v); screen().perform_layout(); @@ -595,7 +595,7 @@ void V4D::setDefaultKeyboardEventCallback() { bool V4D::display() { bool result = true; if (!offscreen_) { - fb().makeCurrent(); + fbCtx().makeCurrent(); screen().draw_contents(); #ifndef __EMSCRIPTEN__ mainFramebufferContext_->blitFrameBufferToScreen(getViewport(), getWindowSize(), isStretching()); @@ -605,6 +605,7 @@ bool V4D::display() { screen().draw_widgets(); glfwSwapBuffers(getGLFWWindow()); glfwPollEvents(); + result = !glfwWindowShouldClose(getGLFWWindow()); } @@ -619,13 +620,8 @@ void V4D::close() { setVisible(false); closed_ = true; } - GLFWwindow* V4D::getGLFWWindow() { - return mainFramebufferContext_->getGLFWWindow(); -} - -NVGcontext* V4D::getNVGcontext() { - return screen().nvg_context(); + return fbCtx().getGLFWWindow(); } } }