redesigned context sharing through texture sharing

pull/3471/head
kallaballa 2 years ago
parent 09fef18a07
commit f6de87a6bc
  1. 10
      modules/v4d/include/opencv2/v4d/v4d.hpp
  2. 29
      modules/v4d/samples/cube-demo.cpp
  3. 2
      modules/v4d/samples/optflow-demo.cpp
  4. 13
      modules/v4d/samples/pedestrian-demo.cpp
  5. 39
      modules/v4d/samples/shader-demo.cpp
  6. 5
      modules/v4d/samples/video-demo.cpp
  7. 77
      modules/v4d/src/detail/framebuffercontext.cpp
  8. 3
      modules/v4d/src/detail/framebuffercontext.hpp
  9. 29
      modules/v4d/src/detail/glcontext.cpp
  10. 59
      modules/v4d/src/detail/glcontext.hpp
  11. 69
      modules/v4d/src/detail/nanovgcontext.cpp
  12. 9
      modules/v4d/src/detail/nanovgcontext.hpp
  13. 128
      modules/v4d/src/v4d.cpp

@ -53,6 +53,7 @@ namespace viz {
namespace detail { namespace detail {
class FrameBufferContext; class FrameBufferContext;
class CLVAContext; class CLVAContext;
class GLContext;
class NanoVGContext; class NanoVGContext;
/*! /*!
@ -125,6 +126,7 @@ class CV_EXPORTS V4D {
bool offscreen_; bool offscreen_;
FrameBufferContext* mainFramebufferContext_ = nullptr; FrameBufferContext* mainFramebufferContext_ = nullptr;
CLVAContext* clvaContext_ = nullptr; CLVAContext* clvaContext_ = nullptr;
GLContext* glContext_ = nullptr;
NanoVGContext* nvgContext_ = nullptr; NanoVGContext* nvgContext_ = nullptr;
cv::VideoCapture* capture_ = nullptr; cv::VideoCapture* capture_ = nullptr;
cv::VideoWriter* writer_ = nullptr; cv::VideoWriter* writer_ = nullptr;
@ -426,11 +428,11 @@ private:
void setMousePosition(int x, int y); void setMousePosition(int x, int y);
nanogui::Screen& screen(); nanogui::Screen& screen();
FormHelper& form(); FormHelper& form();
FrameBufferContext& fb(); FrameBufferContext& fbCtx();
CLVAContext& clva(); CLVAContext& clvaCtx();
NanoVGContext& nvg(); NanoVGContext& nvgCtx();
GLContext& glCtx();
GLFWwindow* getGLFWWindow(); GLFWwindow* getGLFWWindow();
NVGcontext* getNVGcontext();
}; };
} }
} /* namespace kb */ } /* namespace kb */

@ -72,15 +72,15 @@ static GLuint load_shader() {
return cv::viz::init_shader(vert.c_str(), frag.c_str(), "fragColor"); return cv::viz::init_shader(vert.c_str(), frag.c_str(), "fragColor");
} }
static void init_scene() { static void init_scene(const cv::Size& sz) {
glEnable (GL_DEPTH_TEST); glEnable (GL_DEPTH_TEST);
float vertices[] = { float vertices[] = {
// Front face // Front face
0.5, 0.5, 0.5, -0.5, 0.5, 0.5, -0.5, -0.5, 0.5, 0.5, -0.5, 0.5, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0,
// Back face // Back face
0.5, 0.5, -0.5, -0.5, 0.5, -0.5, -0.5, -0.5, -0.5, 0.5, -0.5, -0.5, }; 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, };
float vertex_colors[] = { 1.0, 0.4, 0.6, 1.0, 0.9, 0.2, 0.7, 0.3, 0.8, 0.5, 0.3, 1.0, float vertex_colors[] = { 1.0, 0.4, 0.6, 1.0, 0.9, 0.2, 0.7, 0.3, 0.8, 0.5, 0.3, 1.0,
@ -137,6 +137,7 @@ static void init_scene() {
shader_program = load_shader(); shader_program = load_shader();
uniform_transform = glGetUniformLocation(shader_program, "transform"); uniform_transform = glGetUniformLocation(shader_program, "transform");
glViewport(0,0, sz.width, sz.height);
} }
static void render_scene() { static void render_scene() {
@ -148,19 +149,29 @@ static void render_scene() {
float angle = fmod(double(cv::getTickCount()) / double(cv::getTickFrequency()), 2 * M_PI); float angle = fmod(double(cv::getTickCount()) / double(cv::getTickFrequency()), 2 * M_PI);
float scale = 0.25; float scale = 0.25;
cv::Matx44f scaleMat(scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, 0.0, scale, 0.0, 0.0, 0.0, cv::Matx44f scaleMat(
0.0, 1.0); scale, 0.0, 0.0, 0.0,
0.0, scale, 0.0, 0.0,
0.0, 0.0, scale, 0.0,
0.0, 0.0, 0.0, 1.0);
cv::Matx44f rotXMat(1.0, 0.0, 0.0, 0.0, 0.0, cos(angle), -sin(angle), 0.0, 0.0, sin(angle), cv::Matx44f rotXMat(
cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0); 1.0, 0.0, 0.0, 0.0,
0.0, cos(angle), -sin(angle), 0.0,
0.0, sin(angle), cos(angle), 0.0,
0.0, 0.0, 0.0, 1.0);
cv::Matx44f rotYMat(cos(angle), 0.0, sin(angle), 0.0, 0.0, 1.0, 0.0, 0.0, -sin(angle), 0.0, cv::Matx44f rotYMat(
cos(angle), 0.0, 0.0, 0.0, 0.0, 1.0); cos(angle), 0.0, sin(angle), 0.0,
0.0, 1.0, 0.0, 0.0,
-sin(angle), 0.0,cos(angle), 0.0,
0.0, 0.0, 0.0, 1.0);
cv::Matx44f rotZMat(cos(angle), -sin(angle), 0.0, 0.0, sin(angle), cos(angle), 0.0, 0.0, 0.0, cv::Matx44f rotZMat(cos(angle), -sin(angle), 0.0, 0.0, sin(angle), cos(angle), 0.0, 0.0, 0.0,
0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0); 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0);
cv::Matx44f transform = scaleMat * rotXMat * rotYMat * rotZMat; cv::Matx44f transform = scaleMat * rotXMat * rotYMat * rotZMat;
glUniformMatrix4fv(uniform_transform, 1, GL_FALSE, transform.val); glUniformMatrix4fv(uniform_transform, 1, GL_FALSE, transform.val);
glBindVertexArray(vao); glBindVertexArray(vao);
glDrawElements(GL_TRIANGLES, triangles * 3, GL_UNSIGNED_SHORT, NULL); glDrawElements(GL_TRIANGLES, triangles * 3, GL_UNSIGNED_SHORT, NULL);

@ -423,8 +423,8 @@ static bool iteration() {
//Detect trackable points in the motion mask //Detect trackable points in the motion mask
detect_points(downMotionMaskGrey, detectedPoints); detect_points(downMotionMaskGrey, detectedPoints);
v4d->clear();
v4d->nvg([=]() { v4d->nvg([=]() {
v4d->clear();
if (!downPrevGrey.empty()) { if (!downPrevGrey.empty()) {
//We don't want the algorithm to get out of hand when there is a scene change, so we suppress it when we detect one. //We don't want the algorithm to get out of hand when there is a scene change, so we suppress it when we detect one.
if (!detect_scene_change(downMotionMaskGrey, scene_change_thresh, scene_change_thresh_diff)) { if (!detect_scene_change(downMotionMaskGrey, scene_change_thresh, scene_change_thresh_diff)) {

@ -3,14 +3,12 @@
// of this distribution and at http://opencv.org/license.html. // of this distribution and at http://opencv.org/license.html.
// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org> // Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
#include "opencv2/v4d/v4d.hpp" #include <opencv2/v4d/v4d.hpp>
#include "opencv2/v4d/util.hpp"
#include <string>
#include <opencv2/tracking.hpp> #include <opencv2/tracking.hpp>
#include <opencv2/objdetect.hpp> #include <opencv2/objdetect.hpp>
#include <string>
constexpr unsigned int WIDTH = 1920; constexpr unsigned int WIDTH = 1920;
constexpr unsigned int HEIGHT = 1080; constexpr unsigned int HEIGHT = 1080;
const unsigned long DIAG = hypot(double(WIDTH), double(HEIGHT)); const unsigned long DIAG = hypot(double(WIDTH), double(HEIGHT));
@ -31,7 +29,7 @@ using std::endl;
using std::vector; using std::vector;
using std::string; using std::string;
static cv::Ptr<cv::viz::V4D> v4d = cv::viz::V4D::make(cv::Size(WIDTH, HEIGHT), cv::Size(WIDTH, HEIGHT), OFFSCREEN, "Beauty Demo"); static cv::Ptr<cv::viz::V4D> v4d = cv::viz::V4D::make(cv::Size(WIDTH, HEIGHT), cv::Size(WIDTH, HEIGHT), OFFSCREEN, "Pedestrian Demo");
static cv::HOGDescriptor hog; static cv::HOGDescriptor hog;
//adapted from cv::dnn_objdetect::InferBbox //adapted from cv::dnn_objdetect::InferBbox
@ -178,10 +176,11 @@ static bool iteration() {
} }
} }
v4d->clear();
v4d->nvg([&](const cv::Size& sz) { v4d->nvg([&](const cv::Size& sz) {
using namespace cv::viz::nvg; using namespace cv::viz::nvg;
v4d->clear();
beginPath(); beginPath();
strokeWidth(std::fmax(2.0, sz.width / 960.0)); strokeWidth(std::fmax(2.0, sz.width / 960.0));
strokeColor(cv::viz::colorConvert(cv::Scalar(0, 127, 255, 200), cv::COLOR_HLS2BGR)); strokeColor(cv::viz::colorConvert(cv::Scalar(0, 127, 255, 200), cv::COLOR_HLS2BGR));

@ -3,7 +3,8 @@
// of this distribution and at http://opencv.org/license.html. // of this distribution and at http://opencv.org/license.html.
// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org> // Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
#include "opencv2/v4d/v4d.hpp" #include <opencv2/v4d/v4d.hpp>
#include <opencv2/highgui.hpp>
using std::cerr; using std::cerr;
using std::endl; using std::endl;
@ -49,10 +50,8 @@ GLint zoom_hdl;
/** shader and program handle **/ /** shader and program handle **/
GLuint shader_program_hdl; GLuint shader_program_hdl;
#ifndef OPENCV_V4D_USE_ES3
//vertex array //vertex array
GLuint VAO; GLuint VAO;
#endif
GLuint VBO, EBO; GLuint VBO, EBO;
// vertex position, color // vertex position, color
@ -68,10 +67,8 @@ unsigned int indices[] = {
static void load_buffer_data() { static void load_buffer_data() {
#ifndef OPENCV_V4D_USE_ES3
glGenVertexArrays(1, &VAO); glGenVertexArrays(1, &VAO);
glBindVertexArray(VAO); glBindVertexArray(VAO);
#endif
glGenBuffers(1, &VBO); glGenBuffers(1, &VBO);
glGenBuffers(1, &EBO); glGenBuffers(1, &EBO);
@ -86,12 +83,10 @@ static void load_buffer_data() {
glEnableVertexAttribArray(0); glEnableVertexAttribArray(0);
glBindBuffer(GL_ARRAY_BUFFER, 0); glBindBuffer(GL_ARRAY_BUFFER, 0);
#ifndef OPENCV_V4D_USE_ES3
glBindVertexArray(0); glBindVertexArray(0);
#endif
} }
//workaround: required with emscripten + nanogui on every iteration before rendering //workaround: required with emscripten on every iteration before rendering
static void rebind_buffers() { static void rebind_buffers() {
glBindBuffer(GL_ARRAY_BUFFER, VBO); glBindBuffer(GL_ARRAY_BUFFER, VBO);
glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
@ -103,6 +98,7 @@ static void rebind_buffers() {
glEnableVertexAttribArray(0); glEnableVertexAttribArray(0);
glBindBuffer(GL_ARRAY_BUFFER, 0); glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
} }
//mandelbrot shader code adapted from my own project: https://github.com/kallaballa/FractalDive#after //mandelbrot shader code adapted from my own project: https://github.com/kallaballa/FractalDive#after
@ -163,23 +159,21 @@ static void load_shader() {
return iterations; return iterations;
} }
vec4 return_color() void determine_color()
{ {
int iter = get_iterations(); int iter = get_iterations();
if (iter == max_iterations) { if (iter != max_iterations) {
return vec4(0.0f, 0.0f, 0.0f, 0.0f); float iterations = float(iter) / float(max_iterations);
//convert to float
float cb = float(contrast_boost);
outColor = vec4(base_color[0] * iterations * cb, base_color[1] * iterations * cb, base_color[2] * iterations * cb, base_color[3]);
} }
float iterations = float(iter) / float(max_iterations);
//convert to float
float cb = float(contrast_boost);
return vec4(base_color[0] * iterations * cb, base_color[1] * iterations * cb, base_color[2] * iterations * cb, base_color[3]);
} }
void main() void main()
{ {
outColor = return_color(); determine_color();
})"; })";
cerr << "##### Vertex Shader #####" << endl; cerr << "##### Vertex Shader #####" << endl;
@ -234,9 +228,7 @@ static void render_scene() {
glUniform1f(zoom_hdl, zoom); glUniform1f(zoom_hdl, zoom);
} }
#ifndef OPENCV_V4D_USE_ES3
glBindVertexArray(VAO); glBindVertexArray(VAO);
#endif
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0); glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
} }
@ -335,9 +327,12 @@ static void setup_gui(cv::Ptr<cv::viz::V4D> v4dMain) {
static bool iteration() { static bool iteration() {
//ignore failed capture attempts //ignore failed capture attempts
v4d->capture(); v4d->capture();
v4d->fb([](cv::UMat& frameBuffer) {
imshow("fb1", frameBuffer);
cv::waitKey(1);
});
#ifdef __EMSCRIPTEN__ #ifdef __EMSCRIPTEN__
//required in conjunction with emscripten + nanovg //required in conjunction with emscripten
rebind_buffers(); rebind_buffers();
#endif #endif
//Render using OpenGL //Render using OpenGL

@ -140,8 +140,8 @@ static void init_scene() {
} }
static void render_scene() { static void render_scene() {
glClearColor(0.1, 0.12, 0.2, 1); // glClearColor(0.1, 0.12, 0.2, 1);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glClear(GL_DEPTH_BUFFER_BIT);
glUseProgram(shader_program); glUseProgram(shader_program);
@ -194,6 +194,7 @@ static void glow_effect(const cv::UMat& src, cv::UMat& dst, const int ksize) {
static bool iteration() { static bool iteration() {
using namespace cv::viz; using namespace cv::viz;
v4d->capture();
//Render using OpenGL //Render using OpenGL
v4d->gl(render_scene); v4d->gl(render_scene);

@ -12,13 +12,13 @@ namespace cv {
namespace viz { namespace viz {
namespace detail { namespace detail {
FrameBufferContext::FrameBufferContext(const FrameBufferContext& other) : FrameBufferContext(other.frameBufferSize_, true, other.title_, other.major_, other.minor_, other.compat_, other.samples_, other.debug_) { FrameBufferContext::FrameBufferContext(const FrameBufferContext& other) : FrameBufferContext(other.frameBufferSize_, true, other.title_, other.major_, other.minor_, other.compat_, other.samples_, other.debug_, other.glfwWindow_, other.textureID_) {
} }
FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen, FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen,
const string& title, int major, int minor, bool compat, int samples, bool debug) : const string& title, int major, int minor, bool compat, int samples, bool debug, GLFWwindow* sharedWindow, GLuint sharedTexture) :
offscreen_(offscreen), title_(title), major_(major), minor_( offscreen_(offscreen), title_(title), major_(major), minor_(
minor), compat_(compat), samples_(samples), debug_(debug), frameBufferSize_(frameBufferSize) { minor), compat_(compat), samples_(samples), debug_(debug), frameBufferSize_(frameBufferSize), textureID_(sharedTexture) {
if (glfwInit() != GLFW_TRUE) if (glfwInit() != GLFW_TRUE)
assert(false); assert(false);
@ -62,14 +62,10 @@ FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool off
#endif #endif
glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE); glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
/* I figure we don't need double buffering because the FBO (and the bound texture) is our backbuffer that // glfwWindowHint(GLFW_DOUBLEBUFFER, GL_FALSE);
* we blit to the front on every iteration.
* On X11, wayland and in WASM it works and boosts performance a bit.
*/
glfwWindowHint(GLFW_DOUBLEBUFFER, GL_FALSE);
glfwWindow_ = glfwCreateWindow(frameBufferSize.width, frameBufferSize.height, title_.c_str(), nullptr, glfwWindow_ = glfwCreateWindow(frameBufferSize.width, frameBufferSize.height, title_.c_str(), nullptr,
nullptr); sharedWindow);
if (glfwWindow_ == NULL) { if (glfwWindow_ == NULL) {
assert(false); assert(false);
} }
@ -93,27 +89,48 @@ FrameBufferContext::FrameBufferContext(const cv::Size& frameBufferSize, bool off
#else #else
clglSharing_ = false; clglSharing_ = false;
#endif #endif
frameBufferID_ = 0; if(sharedWindow == nullptr) {
GL_CHECK(glGenFramebuffers(1, &frameBufferID_)); GL_CHECK(glGenFramebuffers(1, &frameBufferID_));
GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_)); GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_));
GL_CHECK(glGenRenderbuffers(1, &renderBufferID_)); GL_CHECK(glGenRenderbuffers(1, &renderBufferID_));
textureID_ = 0;
GL_CHECK(glGenTextures(1, &textureID_)); GL_CHECK(glGenTextures(1, &textureID_));
GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_)); GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_));
texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_); texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_);
GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1)); GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1));
GL_CHECK( GL_CHECK(
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0)); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0));
GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_)); GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_));
GL_CHECK( GL_CHECK(
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height)); glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height));
GL_CHECK( GL_CHECK(
glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_)); glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_));
GL_CHECK( GL_CHECK(
glFramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, textureID_, 0)); glNamedFramebufferTexture(frameBufferID_, GL_COLOR_ATTACHMENT0, textureID_, 0));
assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE); assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE);
} else {
GL_CHECK(glGenFramebuffers(1, &frameBufferID_));
GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, frameBufferID_));
GL_CHECK(glBindTexture(GL_TEXTURE_2D, textureID_));
texture_ = new cv::ogl::Texture2D(frameBufferSize_, cv::ogl::Texture2D::RGBA, textureID_);
GL_CHECK(glPixelStorei(GL_UNPACK_ALIGNMENT, 1));
GL_CHECK(
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, frameBufferSize_.width, frameBufferSize_.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0));
GL_CHECK(glGenRenderbuffers(1, &renderBufferID_));
GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, renderBufferID_));
GL_CHECK(
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, frameBufferSize_.width, frameBufferSize_.height));
GL_CHECK(
glFramebufferRenderbuffer(GL_DRAW_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderBufferID_));
GL_CHECK(
glNamedFramebufferTexture(frameBufferID_, GL_COLOR_ATTACHMENT1, textureID_, 0));
assert(glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE);
}
#ifndef __EMSCRIPTEN__ #ifndef __EMSCRIPTEN__
context_ = CLExecContext_t::getCurrent(); context_ = CLExecContext_t::getCurrent();
#endif #endif

@ -44,6 +44,7 @@ typedef cv::ocl::OpenCLExecutionContextScope CLExecScope_t;
*/ */
class FrameBufferContext { class FrameBufferContext {
friend class CLVAContext; friend class CLVAContext;
friend class GLContext;
friend class NanoVGContext; friend class NanoVGContext;
friend class cv::viz::V4D; friend class cv::viz::V4D;
bool offscreen_; bool offscreen_;
@ -141,7 +142,7 @@ public:
* @param frameBufferSize The frame buffer size. * @param frameBufferSize The frame buffer size.
*/ */
FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen, FrameBufferContext(const cv::Size& frameBufferSize, bool offscreen,
const string& title, int major, int minor, bool compat, int samples, bool debug); const string& title, int major, int minor, bool compat, int samples, bool debug, GLFWwindow* sharedWindow, GLuint sharedTexture);
FrameBufferContext(const FrameBufferContext& other); FrameBufferContext(const FrameBufferContext& other);

@ -0,0 +1,29 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
#include "glcontext.hpp"
namespace cv {
namespace viz {
namespace detail {
GLContext::GLContext(FrameBufferContext& fbContext) :
mainFbContext_(fbContext), glFbContext_(fbContext) {
}
void GLContext::render(std::function<void(const cv::Size&)> fn) {
#ifndef __EMSCRIPTEN__
CLExecScope_t scope(glFbContext_.getCLExecContext());
#endif
FrameBufferContext::GLScope glScope(glFbContext_);
fn(glFbContext_.getSize());
}
FrameBufferContext& GLContext::fbCtx() {
return glFbContext_;
}
}
}
}

@ -0,0 +1,59 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright Amir Hassan (kallaballa) <amir@viel-zu.org>
#ifndef SRC_OPENCV_GLCONTEXT_HPP_
#define SRC_OPENCV_GLCONTEXT_HPP_
#include "framebuffercontext.hpp"
#include <nanogui/nanogui.h>
#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif
#ifndef OPENCV_V4D_USE_ES3
#define NANOGUI_USE_OPENGL
#else
#define NANOGUI_USE_GLES
#define NANOGUI_GLES_VERSION 3
#endif
#include <nanogui/opengl.h>
#include "opencv2/v4d/util.hpp"
#include "opencv2/v4d/nvg.hpp"
namespace cv {
namespace viz {
namespace detail {
/*!
* Used to setup a nanovg context
*/
class GLContext {
FrameBufferContext& mainFbContext_;
FrameBufferContext glFbContext_;
cv::UMat preFB_;
cv::UMat fb_;
cv::UMat postFB_;
public:
/*!
* Creates a OpenGL Context
* @param fbContext The framebuffer context
*/
GLContext(FrameBufferContext& fbContext);
/*!
* Execute function object fn inside a gl context.
* The context takes care of setting up opengl states.
* @param fn A function that is passed the size of the framebuffer
* and performs drawing using opengl
*/
void render(std::function<void(const cv::Size&)> fn);
FrameBufferContext& fbCtx();
};
}
}
}
#endif /* SRC_OPENCV_GLCONTEXT_HPP_ */

@ -8,61 +8,32 @@
namespace cv { namespace cv {
namespace viz { namespace viz {
namespace detail { namespace detail {
NanoVGContext::NanoVGContext(NVGcontext* context, FrameBufferContext& fbContext) : NanoVGContext::NanoVGContext(FrameBufferContext& fbContext) :
context_(context), mainFbContext_(fbContext), nvgFbContext_(fbContext) { mainFbContext_(fbContext), nvgFbContext_(fbContext) {
//FIXME workaround for first frame color glitch #ifndef __EMSCRIPTEN__
FrameBufferContext::GLScope glScope(mainFbContext_); CLExecScope_t scope(nvgFbContext_.getCLExecContext());
FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_); #endif
FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_);
screen_ = new nanogui::Screen();
screen_->initialize(nvgFbContext_.getGLFWWindow(), false);
context_ = screen_->nvg_context();
} }
void NanoVGContext::render(std::function<void(const cv::Size&)> fn) { void NanoVGContext::render(std::function<void(const cv::Size&)> fn) {
{
#ifndef __EMSCRIPTEN__
CLExecScope_t scope(mainFbContext_.getCLExecContext());
#endif
FrameBufferContext::GLScope mainGlScope(mainFbContext_);
FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_);
fb_.copyTo(preFB_);
}
{
#ifndef __EMSCRIPTEN__
CLExecScope_t scope(nvgFbContext_.getCLExecContext());
#endif
FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_);
preFB_.copyTo(fb_);
}
{
#ifndef __EMSCRIPTEN__ #ifndef __EMSCRIPTEN__
CLExecScope_t scope(nvgFbContext_.getCLExecContext()); CLExecScope_t scope(nvgFbContext_.getCLExecContext());
#endif #endif
FrameBufferContext::GLScope nvgGlScope(nvgFbContext_); FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
NanoVGContext::Scope nvgScope(*this); NanoVGContext::Scope nvgScope(*this);
cv::viz::nvg::detail::NVG::initializeContext(context_); cv::viz::nvg::detail::NVG::initializeContext(context_);
fn(nvgFbContext_.getSize()); fn(nvgFbContext_.getSize());
}
{
#ifndef __EMSCRIPTEN__
CLExecScope_t scope(nvgFbContext_.getCLExecContext());
#endif
FrameBufferContext::GLScope nvgGlScope(nvgFbContext_);
FrameBufferContext::FrameBufferScope fbScope(nvgFbContext_, fb_);
fb_.copyTo(postFB_);
}
{
#ifndef __EMSCRIPTEN__
CLExecScope_t scope(mainFbContext_.getCLExecContext());
#endif
FrameBufferContext::GLScope mainGlScope(mainFbContext_);
FrameBufferContext::FrameBufferScope fbScope(mainFbContext_, fb_);
postFB_.copyTo(fb_);
}
} }
void NanoVGContext::begin() { void NanoVGContext::begin() {
float w = mainFbContext_.getSize().width; float w = nvgFbContext_.getSize().width;
float h = mainFbContext_.getSize().height; float h = nvgFbContext_.getSize().height;
float r = mainFbContext_.getXPixelRatio(); float r = nvgFbContext_.getXPixelRatio();
nvgSave(context_); nvgSave(context_);
nvgBeginFrame(context_, w, h, r); nvgBeginFrame(context_, w, h, r);
@ -77,6 +48,10 @@ void NanoVGContext::end() {
nvgEndFrame(context_); nvgEndFrame(context_);
nvgRestore(context_); nvgRestore(context_);
} }
FrameBufferContext& NanoVGContext::fbCtx() {
return nvgFbContext_;
}
} }
} }
} }

@ -7,7 +7,6 @@
#define SRC_OPENCV_NANOVGCONTEXT_HPP_ #define SRC_OPENCV_NANOVGCONTEXT_HPP_
#include "framebuffercontext.hpp" #include "framebuffercontext.hpp"
#include <nanogui/nanogui.h>
#ifdef __EMSCRIPTEN__ #ifdef __EMSCRIPTEN__
#include <emscripten.h> #include <emscripten.h>
@ -31,9 +30,10 @@ namespace detail {
* Used to setup a nanovg context * Used to setup a nanovg context
*/ */
class NanoVGContext { class NanoVGContext {
nanogui::Screen* screen_;
NVGcontext* context_; NVGcontext* context_;
FrameBufferContext& mainFbContext_; FrameBufferContext& mainFbContext_;
FrameBufferContext& nvgFbContext_; FrameBufferContext nvgFbContext_;
cv::UMat preFB_; cv::UMat preFB_;
cv::UMat fb_; cv::UMat fb_;
cv::UMat postFB_; cv::UMat postFB_;
@ -65,7 +65,7 @@ public:
* @param context The native NVGContext * @param context The native NVGContext
* @param fbContext The framebuffer context * @param fbContext The framebuffer context
*/ */
NanoVGContext(NVGcontext* context, FrameBufferContext& fbContext); NanoVGContext(FrameBufferContext& fbContext);
/*! /*!
* Execute function object fn inside a nanovg context. * Execute function object fn inside a nanovg context.
* The context takes care of setting up opengl and nanovg states. * The context takes care of setting up opengl and nanovg states.
@ -74,6 +74,8 @@ public:
* and performs drawing using cv::viz::nvg * and performs drawing using cv::viz::nvg
*/ */
void render(std::function<void(const cv::Size&)> fn); void render(std::function<void(const cv::Size&)> fn);
FrameBufferContext& fbCtx();
private: private:
/*! /*!
* Setup NanoVG context * Setup NanoVG context
@ -83,6 +85,7 @@ private:
* Tear down NanoVG context * Tear down NanoVG context
*/ */
void end(); void end();
}; };
} }
} }

@ -6,6 +6,7 @@
#include "opencv2/v4d/v4d.hpp" #include "opencv2/v4d/v4d.hpp"
#include "detail/clvacontext.hpp" #include "detail/clvacontext.hpp"
#include "detail/framebuffercontext.hpp" #include "detail/framebuffercontext.hpp"
#include "detail/glcontext.hpp"
#include "detail/nanovgcontext.hpp" #include "detail/nanovgcontext.hpp"
namespace cv { namespace cv {
@ -80,7 +81,7 @@ V4D::V4D(const cv::Size& size, const cv::Size& frameBufferSize, bool offscreen,
assert( assert(
frameBufferSize_.width >= initialSize_.width frameBufferSize_.width >= initialSize_.width
&& frameBufferSize_.height >= initialSize_.height); && frameBufferSize_.height >= initialSize_.height);
mainFramebufferContext_ = new detail::FrameBufferContext(this->getFrameBufferSize(), offscreen, title, major, minor, compat, samples, debug); mainFramebufferContext_ = new detail::FrameBufferContext(this->getFrameBufferSize(), offscreen, title, major, minor, compat, samples, debug, nullptr, 0);
if(!initializeGUI()) if(!initializeGUI())
assert(false); assert(false);
@ -94,6 +95,8 @@ V4D::~V4D() {
delete writer_; delete writer_;
if (capture_) if (capture_)
delete capture_; delete capture_;
if (glContext_)
delete glContext_;
if (nvgContext_) if (nvgContext_)
delete nvgContext_; delete nvgContext_;
if (clvaContext_) if (clvaContext_)
@ -156,7 +159,7 @@ bool V4D::initializeGUI() {
std::vector<nanogui::Widget*> widgets; std::vector<nanogui::Widget*> widgets;
find_widgets(&v4d->screen(), widgets); find_widgets(&v4d->screen(), widgets);
for (auto* w : widgets) { for (auto* w : widgets) {
auto mousePos = nanogui::Vector2i(v4d->getMousePosition()[0] / v4d->fb().getXPixelRatio(), v4d->getMousePosition()[1] / v4d->fb().getYPixelRatio()); auto mousePos = nanogui::Vector2i(v4d->getMousePosition()[0] / v4d->fbCtx().getXPixelRatio(), v4d->getMousePosition()[1] / v4d->fbCtx().getYPixelRatio());
if(contains_absolute(w, mousePos)) { if(contains_absolute(w, mousePos)) {
v4d->screen().scroll_callback_event(x, y); v4d->screen().scroll_callback_event(x, y);
return; return;
@ -179,7 +182,8 @@ bool V4D::initializeGUI() {
}); });
clvaContext_ = new detail::CLVAContext(*mainFramebufferContext_); clvaContext_ = new detail::CLVAContext(*mainFramebufferContext_);
nvgContext_ = new detail::NanoVGContext(getNVGcontext(), *mainFramebufferContext_); glContext_ = new detail::GLContext(*mainFramebufferContext_);
nvgContext_ = new detail::NanoVGContext(*mainFramebufferContext_);
} catch(std::exception& ex) { } catch(std::exception& ex) {
cerr << "V4D initialization failed: " << ex.what() << endl; cerr << "V4D initialization failed: " << ex.what() << endl;
return false; return false;
@ -207,63 +211,63 @@ bool V4D::keyboard_event(int key, int scancode, int action, int modifiers) {
return screen().keyboard_event(key, scancode, action, modifiers); return screen().keyboard_event(key, scancode, action, modifiers);
} }
FrameBufferContext& V4D::fb() { FrameBufferContext& V4D::fbCtx() {
assert(mainFramebufferContext_ != nullptr); assert(mainFramebufferContext_ != nullptr);
mainFramebufferContext_->makeCurrent(); mainFramebufferContext_->makeCurrent();
return *mainFramebufferContext_; return *mainFramebufferContext_;
} }
CLVAContext& V4D::clva() { CLVAContext& V4D::clvaCtx() {
assert(clvaContext_ != nullptr); assert(clvaContext_ != nullptr);
return *clvaContext_; return *clvaContext_;
} }
NanoVGContext& V4D::nvg() { NanoVGContext& V4D::nvgCtx() {
assert(nvgContext_ != nullptr); assert(nvgContext_ != nullptr);
fb().makeCurrent(); nvgContext_->fbCtx().makeCurrent();
return *nvgContext_; return *nvgContext_;
} }
GLContext& V4D::glCtx() {
assert(glContext_ != nullptr);
glContext_->fbCtx().makeCurrent();
return *glContext_;
}
nanogui::Screen& V4D::screen() { nanogui::Screen& V4D::screen() {
assert(screen_ != nullptr); assert(screen_ != nullptr);
fb().makeCurrent(); fbCtx().makeCurrent();
return *screen_; return *screen_;
} }
cv::Size V4D::getVideoFrameSize() { cv::Size V4D::getVideoFrameSize() {
return clva().getVideoFrameSize(); return clvaCtx().getVideoFrameSize();
} }
void V4D::gl(std::function<void()> fn) { void V4D::gl(std::function<void()> fn) {
#ifndef __EMSCRIPTEN__ glCtx().render([=](const cv::Size& sz){
detail::CLExecScope_t scope(fb().getCLExecContext()); CV_UNUSED(sz);
#endif fn();
detail::FrameBufferContext::GLScope glScope(fb()); });
fn();
} }
void V4D::gl(std::function<void(const cv::Size&)> fn) { void V4D::gl(std::function<void(const cv::Size&)> fn) {
auto fbSize = getFrameBufferSize(); glCtx().render(fn);
#ifndef __EMSCRIPTEN__
detail::CLExecScope_t scope(fb().getCLExecContext());
#endif
detail::FrameBufferContext::GLScope glScope(fb());
fn(fbSize);
} }
void V4D::fb(std::function<void(cv::UMat&)> fn) { void V4D::fb(std::function<void(cv::UMat&)> fn) {
fb().execute(fn); fbCtx().execute(fn);
} }
void V4D::nvg(std::function<void()> fn) { void V4D::nvg(std::function<void()> fn) {
nvg().render([=](const cv::Size& sz){ nvgCtx().render([=](const cv::Size& sz){
CV_UNUSED(sz); CV_UNUSED(sz);
fn(); fn();
}); });
} }
void V4D::nvg(std::function<void(const cv::Size&)> fn) { void V4D::nvg(std::function<void(const cv::Size&)> fn) {
nvg().render(fn); nvgCtx().render(fn);
} }
void V4D::nanogui(std::function<void(FormHelper& form)> fn) { void V4D::nanogui(std::function<void(FormHelper& form)> fn) {
@ -292,8 +296,8 @@ void V4D::run(std::function<bool()> fn) {
} }
void V4D::setSource(const Source& src) { void V4D::setSource(const Source& src) {
if (!clva().hasContext()) if (!clvaCtx().hasContext())
clva().copyContext(); clvaCtx().copyContext();
source_ = src; source_ = src;
} }
@ -316,20 +320,17 @@ bool V4D::capture(std::function<void(cv::UMat&)> fn) {
return false; return false;
if(nextReaderFrame_.empty()) { if(nextReaderFrame_.empty()) {
if(!clva().capture(fn, nextReaderFrame_)) if(!clvaCtx().capture(fn, nextReaderFrame_))
return false; return false;
} }
currentReaderFrame_ = nextReaderFrame_; currentReaderFrame_ = nextReaderFrame_;
{ fb([=,this](cv::UMat frameBuffer) {
FrameBufferContext::GLScope glScope(*mainFramebufferContext_); currentReaderFrame_.copyTo(frameBuffer);
FrameBufferContext::FrameBufferScope fbScope(*mainFramebufferContext_, readerFrameBuffer_); });
currentReaderFrame_.copyTo(readerFrameBuffer_);
}
futureReader_ = pool.push([=,this](){ futureReader_ = pool.push([=,this](){
return clva().capture(fn, nextReaderFrame_); return clvaCtx().capture(fn, nextReaderFrame_);
}); });
return captureSuccessful_; return captureSuccessful_;
} }
@ -339,8 +340,8 @@ bool V4D::isSourceReady() {
} }
void V4D::setSink(const Sink& sink) { void V4D::setSink(const Sink& sink) {
if (!clva().hasContext()) if (!clvaCtx().hasContext())
clva().copyContext(); clvaCtx().copyContext();
sink_ = sink; sink_ = sink;
} }
@ -355,14 +356,11 @@ void V4D::write(std::function<void(const cv::UMat&)> fn) {
if(futureWriter_.valid()) if(futureWriter_.valid())
futureWriter_.get(); futureWriter_.get();
{ fb([=, this](cv::UMat frameBuffer) {
FrameBufferContext::GLScope glScope(*mainFramebufferContext_); frameBuffer.copyTo(currentWriterFrame_);
FrameBufferContext::FrameBufferScope fbScope(*mainFramebufferContext_, writerFrameBuffer_); });
writerFrameBuffer_.copyTo(currentWriterFrame_);
}
futureWriter_ = pool.push([=,this](){ futureWriter_ = pool.push([=,this](){
clva().write(fn, currentWriterFrame_); clvaCtx().write(fn, currentWriterFrame_);
}); });
} }
@ -371,12 +369,14 @@ bool V4D::isSinkReady() {
} }
void V4D::clear(const cv::Scalar& bgra) { void V4D::clear(const cv::Scalar& bgra) {
const float& b = bgra[0] / 255.0f; this->gl([&](){
const float& g = bgra[1] / 255.0f; const float& b = bgra[0] / 255.0f;
const float& r = bgra[2] / 255.0f; const float& g = bgra[1] / 255.0f;
const float& a = bgra[3] / 255.0f; const float& r = bgra[2] / 255.0f;
GL_CHECK(glClearColor(r, g, b, a)); const float& a = bgra[3] / 255.0f;
GL_CHECK(glClear(GL_COLOR_BUFFER_BIT)); GL_CHECK(glClearColor(r, g, b, a));
GL_CHECK(glClear(GL_COLOR_BUFFER_BIT));
});
} }
void V4D::showGui(bool s) { void V4D::showGui(bool s) {
@ -465,7 +465,7 @@ void V4D::zoom(float factor) {
} }
cv::Vec2f V4D::getPosition() { cv::Vec2f V4D::getPosition() {
fb().makeCurrent(); fbCtx().makeCurrent();
int x, y; int x, y;
glfwGetWindowPos(getGLFWWindow(), &x, &y); glfwGetWindowPos(getGLFWWindow(), &x, &y);
return {float(x), float(y)}; return {float(x), float(y)};
@ -488,7 +488,7 @@ cv::Rect V4D::getViewport() {
} }
cv::Size V4D::getNativeFrameBufferSize() { cv::Size V4D::getNativeFrameBufferSize() {
fb().makeCurrent(); fbCtx().makeCurrent();
int w, h; int w, h;
glfwGetFramebufferSize(getGLFWWindow(), &w, &h); glfwGetFramebufferSize(getGLFWWindow(), &w, &h);
return {w, h}; return {w, h};
@ -499,7 +499,7 @@ cv::Size V4D::getFrameBufferSize() {
} }
cv::Size V4D::getWindowSize() { cv::Size V4D::getWindowSize() {
fb().makeCurrent(); fbCtx().makeCurrent();
int w, h; int w, h;
glfwGetWindowSize(getGLFWWindow(), &w, &h); glfwGetWindowSize(getGLFWWindow(), &w, &h);
return {w, h}; return {w, h};
@ -510,17 +510,17 @@ cv::Size V4D::getInitialSize() {
} }
void V4D::setWindowSize(const cv::Size& sz) { void V4D::setWindowSize(const cv::Size& sz) {
fb().makeCurrent(); fbCtx().makeCurrent();
screen().set_size(nanogui::Vector2i(sz.width / fb().getXPixelRatio(), sz.height / fb().getYPixelRatio())); screen().set_size(nanogui::Vector2i(sz.width / fbCtx().getXPixelRatio(), sz.height / fbCtx().getYPixelRatio()));
} }
bool V4D::isFullscreen() { bool V4D::isFullscreen() {
fb().makeCurrent(); fbCtx().makeCurrent();
return glfwGetWindowMonitor(getGLFWWindow()) != nullptr; return glfwGetWindowMonitor(getGLFWWindow()) != nullptr;
} }
void V4D::setFullscreen(bool f) { void V4D::setFullscreen(bool f) {
fb().makeCurrent(); fbCtx().makeCurrent();
auto monitor = glfwGetPrimaryMonitor(); auto monitor = glfwGetPrimaryMonitor();
const GLFWvidmode* mode = glfwGetVideoMode(monitor); const GLFWvidmode* mode = glfwGetVideoMode(monitor);
if (f) { if (f) {
@ -535,22 +535,22 @@ void V4D::setFullscreen(bool f) {
} }
bool V4D::isResizable() { bool V4D::isResizable() {
fb().makeCurrent(); fbCtx().makeCurrent();
return glfwGetWindowAttrib(getGLFWWindow(), GLFW_RESIZABLE) == GLFW_TRUE; return glfwGetWindowAttrib(getGLFWWindow(), GLFW_RESIZABLE) == GLFW_TRUE;
} }
void V4D::setResizable(bool r) { void V4D::setResizable(bool r) {
fb().makeCurrent(); fbCtx().makeCurrent();
glfwWindowHint(GLFW_RESIZABLE, r ? GLFW_TRUE : GLFW_FALSE); glfwWindowHint(GLFW_RESIZABLE, r ? GLFW_TRUE : GLFW_FALSE);
} }
bool V4D::isVisible() { bool V4D::isVisible() {
fb().makeCurrent(); fbCtx().makeCurrent();
return glfwGetWindowAttrib(getGLFWWindow(), GLFW_VISIBLE) == GLFW_TRUE; return glfwGetWindowAttrib(getGLFWWindow(), GLFW_VISIBLE) == GLFW_TRUE;
} }
void V4D::setVisible(bool v) { void V4D::setVisible(bool v) {
fb().makeCurrent(); fbCtx().makeCurrent();
glfwWindowHint(GLFW_VISIBLE, v ? GLFW_TRUE : GLFW_FALSE); glfwWindowHint(GLFW_VISIBLE, v ? GLFW_TRUE : GLFW_FALSE);
screen().set_visible(v); screen().set_visible(v);
screen().perform_layout(); screen().perform_layout();
@ -595,7 +595,7 @@ void V4D::setDefaultKeyboardEventCallback() {
bool V4D::display() { bool V4D::display() {
bool result = true; bool result = true;
if (!offscreen_) { if (!offscreen_) {
fb().makeCurrent(); fbCtx().makeCurrent();
screen().draw_contents(); screen().draw_contents();
#ifndef __EMSCRIPTEN__ #ifndef __EMSCRIPTEN__
mainFramebufferContext_->blitFrameBufferToScreen(getViewport(), getWindowSize(), isStretching()); mainFramebufferContext_->blitFrameBufferToScreen(getViewport(), getWindowSize(), isStretching());
@ -605,6 +605,7 @@ bool V4D::display() {
screen().draw_widgets(); screen().draw_widgets();
glfwSwapBuffers(getGLFWWindow()); glfwSwapBuffers(getGLFWWindow());
glfwPollEvents(); glfwPollEvents();
result = !glfwWindowShouldClose(getGLFWWindow()); result = !glfwWindowShouldClose(getGLFWWindow());
} }
@ -619,13 +620,8 @@ void V4D::close() {
setVisible(false); setVisible(false);
closed_ = true; closed_ = true;
} }
GLFWwindow* V4D::getGLFWWindow() { GLFWwindow* V4D::getGLFWWindow() {
return mainFramebufferContext_->getGLFWWindow(); return fbCtx().getGLFWWindow();
}
NVGcontext* V4D::getNVGcontext() {
return screen().nvg_context();
} }
} }
} }

Loading…
Cancel
Save