From 3ddf77c7b80fbbcc8fafc8f81b7f6162c241f671 Mon Sep 17 00:00:00 2001 From: kallaballa Date: Sat, 8 Oct 2022 19:27:32 +0200 Subject: [PATCH] implemented on- and offscreen rendering --- src/tetra/subsystems.hpp | 97 +++++++++++++++++++++---- src/tetra/tetra-demo.cpp | 151 ++++++++++++++++++++++++--------------- 2 files changed, 176 insertions(+), 72 deletions(-) diff --git a/src/tetra/subsystems.hpp b/src/tetra/subsystems.hpp index 941c3ec4d..d9d6170b5 100644 --- a/src/tetra/subsystems.hpp +++ b/src/tetra/subsystems.hpp @@ -5,13 +5,15 @@ #include #include #include - #include #include #include #include #include "opencv2/core/va_intel.hpp" #include +#include +#include +#include #include #include #include @@ -285,6 +287,60 @@ std::string get_info() { } } // namespace va +namespace x11 { +Display* xdisplay; +Window xroot; +Window xwin; +XSetWindowAttributes swa; + +bool initialized = false; + +Display* get_x11_display() { + return xdisplay; +} + +Window get_x11_window() { + return xwin; +} + +bool is_initialized() { + return initialized; +} + +void init_x11() { + xdisplay = XOpenDisplay(nullptr); + if (xdisplay == nullptr) { + cerr << "Unable to open X11 display" << endl; + exit(3); + } + xroot = DefaultRootWindow(xdisplay); + swa.event_mask = ExposureMask; + xwin = XCreateWindow(xdisplay, xroot, 0, 0, WIDTH, HEIGHT, 0, + CopyFromParent, InputOutput, CopyFromParent, CWEventMask, &swa); + + XSetWindowAttributes xattr; + + xattr.override_redirect = False; + XChangeWindowAttributes(xdisplay, xwin, CWOverrideRedirect, &xattr); + + int one = 1; + XChangeProperty( + xdisplay, xwin, + XInternAtom ( xdisplay, "_HILDON_NON_COMPOSITED_WINDOW", False ), + XA_INTEGER, 32, PropModeReplace, + (unsigned char*) &one, 1); + + XWMHints hints; + hints.input = True; + hints.flags = InputHint; + XSetWMHints(xdisplay, xwin, &hints); + + XMapWindow(xdisplay, xwin); + XStoreName(xdisplay, xwin, "tetra-demo"); + initialized = true; +} +} + namespace egl { //code in the kb::egl namespace deals with setting up EGL EGLDisplay display; @@ -393,11 +449,17 @@ void debugMessageCallback(GLenum source, GLenum type, GLuint id, } void init_egl(bool debug = false) { + bool offscreen = !x11::is_initialized(); + eglCheck(eglBindAPI(EGL_OPENGL_API)); - eglCheck(display = eglGetDisplay(EGL_DEFAULT_DISPLAY)); + if(offscreen) { + eglCheck(display = eglGetDisplay(EGL_DEFAULT_DISPLAY)); + } else { + eglCheck(display = eglGetDisplay(x11::get_x11_display())); + } eglCheck(eglInitialize(display, nullptr, nullptr)); - const EGLint attributes[] = { + const EGLint egl_config_constraints[] = { EGL_BUFFER_SIZE, static_cast(24), EGL_DEPTH_SIZE, static_cast(24), EGL_STENCIL_SIZE, static_cast(0), @@ -413,14 +475,19 @@ void init_egl(bool debug = false) { EGLint configCount; EGLConfig configs[1]; - eglCheck(eglChooseConfig(display, attributes, configs, 1, &configCount)); + eglCheck(eglChooseConfig(display, egl_config_constraints, configs, 1, &configCount)); EGLint attrib_list[] = { EGL_WIDTH, WIDTH, EGL_HEIGHT, HEIGHT, EGL_NONE }; - eglCheck(surface = eglCreatePbufferSurface(display, configs[0], attrib_list)); + if(!offscreen) { + eglCheck(surface = eglCreateWindowSurface(display, configs[0], x11::get_x11_window(), nullptr)); + } else { + eglCheck(surface = eglCreatePbufferSurface(display, configs[0], attrib_list)); + } - const EGLint contextVersion[] = { EGL_CONTEXT_CLIENT_VERSION, 1, EGL_CONTEXT_OPENGL_DEBUG, debug ? EGL_TRUE : EGL_FALSE, EGL_NONE }; + const EGLint contextVersion[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_CONTEXT_OPENGL_DEBUG, debug ? EGL_TRUE : EGL_FALSE, EGL_NONE }; eglCheck(context = eglCreateContext(display, configs[0], EGL_NO_CONTEXT, contextVersion)); eglCheck(eglMakeCurrent(display, surface, surface, context)); + eglCheck(eglSwapInterval(display, 1)); if(debug) { glCheck(glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS)); auto glDebugMessageCallback = (void (*)(void *, void *))eglGetProcAddress("glDebugMessageCallback"); @@ -441,26 +508,26 @@ std::string get_info() { namespace gl { //code in the kb::gl namespace deals with OpenGL (and OpenCV/GL) internals cv::ogl::Texture2D *frame_buf_tex; -GLuint frame_buf_tex_name; +GLuint frame_buf; + void init_gl() { glewInit(); cv::ogl::ocl::initializeContextFromGL(); - frame_buf_tex_name = 0; - glCheck(glGenFramebuffers(1, &frame_buf_tex_name)); - glCheck(glBindFramebuffer(GL_FRAMEBUFFER, frame_buf_tex_name)); + frame_buf = 0; + glCheck(glGenFramebuffers(1, &frame_buf)); + glCheck(glBindFramebuffer(GL_FRAMEBUFFER, frame_buf)); frame_buf_tex = new cv::ogl::Texture2D(cv::Size(WIDTH, HEIGHT), cv::ogl::Texture2D::RGBA, false); frame_buf_tex->bind(); - + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glCheck(glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, frame_buf_tex->texId(), 0)); - GLenum drawBuffers[1] = { GL_COLOR_ATTACHMENT0 }; - glCheck(glDrawBuffers(1, drawBuffers)); + assert(glCheckFramebufferStatus(GL_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE); - glCheck(glBindFramebuffer(GL_FRAMEBUFFER, frame_buf_tex_name)); - glCheck(glClearColor(0.1, 0.39, 0.88, 1.0)); + glCheck(glViewport(0, 0, WIDTH, HEIGHT)); glCheck(glColor3f(1.0, 1.0, 1.0)); glCheck(glEnable(GL_CULL_FACE)); diff --git a/src/tetra/tetra-demo.cpp b/src/tetra/tetra-demo.cpp index 4693ed2ef..78b210c53 100644 --- a/src/tetra/tetra-demo.cpp +++ b/src/tetra/tetra-demo.cpp @@ -2,24 +2,86 @@ constexpr long unsigned int WIDTH = 1920; constexpr long unsigned int HEIGHT = 1080; -constexpr double FPS = 30; +constexpr double FPS = 24; #include "subsystems.hpp" using std::cerr; using std::endl; +cv::ocl::OpenCLExecutionContext VA_CONTEXT; +cv::ocl::OpenCLExecutionContext GL_CONTEXT; + +void render(cv::UMat& frameBuffer) { + glBindFramebuffer(GL_FRAMEBUFFER, kb::gl::frame_buf); + glViewport(0, 0, WIDTH , HEIGHT ); + glRotatef(1, 0, 1, 0); + glClearColor(0.0f, 0.0f, 1.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + + glColor3f(1.0, 1.0, 1.0); + glBegin(GL_LINES); + for (GLfloat i = -2.5; i <= 2.5; i += 0.25) { + glVertex3f(i, 0, 2.5); + glVertex3f(i, 0, -2.5); + glVertex3f(2.5, 0, i); + glVertex3f(-2.5, 0, i); + } + glEnd(); + + glBegin(GL_TRIANGLE_STRIP); + glColor3f(1, 1, 1); + glVertex3f(0, 2, 0); + glColor3f(1, 0, 0); + glVertex3f(-1, 0, 1); + glColor3f(0, 1, 0); + glVertex3f(1, 0, 1); + glColor3f(0, 0, 1); + glVertex3f(0, 0, -1.4); + glColor3f(1, 1, 1); + glVertex3f(0, 2, 0); + glColor3f(1, 0, 0); + glVertex3f(-1, 0, 1); + glEnd(); + glFlush(); + kb::gl::swapBuffers(); +} + +void blitFrameBufferToScreen() { + glBindFramebuffer(GL_READ_FRAMEBUFFER, kb::gl::frame_buf); + glReadBuffer(GL_COLOR_ATTACHMENT0); + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); + glViewport(0, 0, WIDTH, HEIGHT); + glBlitFramebuffer(0, 0, WIDTH, HEIGHT, 0, 0, WIDTH, HEIGHT, GL_COLOR_BUFFER_BIT, GL_NEAREST); +} + +void glow(cv::UMat &frameBuffer, cv::UMat &mask, double sigma = 50) { + cv::blur(frameBuffer, mask, cv::Size(sigma, sigma)); + cv::bitwise_not(mask, mask); + cv::bitwise_not(frameBuffer, frameBuffer); + mask.assignTo(mask, CV_16U); + frameBuffer.assignTo(frameBuffer, CV_16U); + cv::multiply(mask, frameBuffer, mask); + cv::divide(mask, cv::Scalar::all(255.0), mask); + mask.assignTo(mask, CV_8U); + cv::bitwise_not(mask, frameBuffer); +} + int main(int argc, char **argv) { using namespace kb; va::init_va(); cv::VideoWriter video("tetra-demo.mkv", cv::CAP_FFMPEG, cv::VideoWriter::fourcc('V', 'P', '9', '0'), FPS, cv::Size(WIDTH, HEIGHT), { cv::VIDEOWRITER_PROP_HW_DEVICE, 0, cv::VIDEOWRITER_PROP_HW_ACCELERATION, cv::VIDEO_ACCELERATION_VAAPI, cv::VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL, 1 }); - cv::ocl::OpenCLExecutionContext vaContext = cv::ocl::OpenCLExecutionContext::getCurrent(); + VA_CONTEXT = cv::ocl::OpenCLExecutionContext::getCurrent(); + //comment the next line for offscreen rendering + x11::init_x11(); + + //Passing true to init_egl will create a OpenGL debug context egl::init_egl(); gl::init_gl(); - cv::ocl::OpenCLExecutionContext glContext = cv::ocl::OpenCLExecutionContext::getCurrent(); + GL_CONTEXT = cv::ocl::OpenCLExecutionContext::getCurrent(); cerr << "VA Version: " << va::get_info() << endl; cerr << "EGL Version: " << egl::get_info() << endl; @@ -30,72 +92,47 @@ int main(int argc, char **argv) { cv::UMat mask; cv::UMat videoFrame; - double sigma = 50; int64 start = 0; uint64_t cnt = 0; while (true) { start = cv::getTickCount(); - //Draw a rotating tetrahedron - glContext.bind(); - glRotatef(1, 0, 1, 0); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - - glColor3f(1.0, 1.0, 1.0); - glBegin(GL_LINES); - for (GLfloat i = -2.5; i <= 2.5; i += 0.25) { - glVertex3f(i, 0, 2.5); - glVertex3f(i, 0, -2.5); - glVertex3f(2.5, 0, i); - glVertex3f(-2.5, 0, i); - } - glEnd(); - - glBegin(GL_TRIANGLE_STRIP); - glColor3f(1, 1, 1); - glVertex3f(0, 2, 0); - glColor3f(1, 0, 0); - glVertex3f(-1, 0, 1); - glColor3f(0, 1, 0); - glVertex3f(1, 0, 1); - glColor3f(0, 0, 1); - glVertex3f(0, 0, -1.4); - glColor3f(1, 1, 1); - glVertex3f(0, 2, 0); - glColor3f(1, 0, 0); - glVertex3f(-1, 0, 1); - glEnd(); - - glFlush(); - gl::swapBuffers(); - cl::fetch_frame_buffer(frameBuffer); //hand over the data (GPU 2 GPU) to OpenCV/OpenCL - - //Using OpenCL in the background - cv::flip(frameBuffer, frameBuffer, 0); // flip the image in the y-axis - - { - //Do a glow effect using blur - cv::blur(frameBuffer, mask, cv::Size(sigma, sigma)); - cv::bitwise_not(mask, mask); - cv::bitwise_not(frameBuffer, frameBuffer); - mask.assignTo(mask, CV_16U); - frameBuffer.assignTo(frameBuffer, CV_16U); - cv::multiply(mask, frameBuffer, mask); - cv::divide(mask, cv::Scalar::all(255.0), mask); - mask.assignTo(mask, CV_8U); - cv::bitwise_not(mask, frameBuffer); - } + GL_CONTEXT.bind(); + + //Using OpenGL, render a rotating tetrahedron + render(frameBuffer); + + //Transfer buffer ownership to OpenCL + cl::fetch_frame_buffer(frameBuffer); + + //Using OpenCL for a glow effect + glow(frameBuffer, mask); + + //Color-conversion from BGRA to RGB, also OpenCL. + cv::cvtColor(frameBuffer, videoFrame, cv::COLOR_BGRA2RGB); - cv::cvtColor(frameBuffer, videoFrame, cv::COLOR_BGRA2RGB); // Color-conversion from BGRA to RGB + VA_CONTEXT.bind(); + //Encode the frame using VAAPI on the GPU. + video.write(videoFrame); - vaContext.bind(); - video.write(videoFrame); //encode the frame using VAAPI on the GPU. + GL_CONTEXT.bind(); + if(x11::is_initialized()) { + //Transfer buffer ownership back to OpenGL + cl::return_frame_buffer(frameBuffer); + + //Blit the framebuffer we have been working on to screen + blitFrameBufferToScreen(); + } + + //Measure FPS int64 tick = cv::getTickCount(); double tickFreq = cv::getTickFrequency(); - if (cnt % int64(ceil(tickFreq / (FPS * 10000000))) == 0) + if (cnt % int64(ceil(tickFreq / (FPS * 10000000))) == 0) { cerr << "FPS : " << tickFreq / (tick - start + 1) << '\r'; + cnt = 0; + } ++cnt; }