implemented on- and offscreen rendering

src/tetra/subsystems.hpp

@@ -5,13 +5,15 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <filesystem>
-
 #include <va/va.h>
 #include <va/va_drm.h>
 #include <va/va_backend.h>
 #include <opencv2/opencv.hpp>
 #include "opencv2/core/va_intel.hpp"
 #include <opencv2/videoio.hpp>
+#include <X11/Xlib.h>
+#include <X11/Xatom.h>
+#include <X11/Xutil.h>
 #include <GL/glew.h>
 #include <EGL/egl.h>
 #include <CL/cl.h>
@@ -285,6 +287,60 @@ std::string get_info() {
 }
 } // namespace va
 
+namespace x11 {
+Display* xdisplay;
+Window xroot;
+Window xwin;
+XSetWindowAttributes swa;
+
+bool initialized = false;
+
+Display* get_x11_display() {
+    return xdisplay;
+}
+
+Window get_x11_window() {
+    return xwin;
+}
+
+bool is_initialized() {
+    return initialized;
+}
+
+void init_x11() {
+    xdisplay = XOpenDisplay(nullptr);
+    if (xdisplay == nullptr) {
+        cerr << "Unable to open X11 display" << endl;
+        exit(3);
+    }
+    xroot = DefaultRootWindow(xdisplay);
+    swa.event_mask  = ExposureMask;
+    xwin = XCreateWindow(xdisplay, xroot, 0, 0, WIDTH, HEIGHT, 0,
+    CopyFromParent, InputOutput, CopyFromParent, CWEventMask, &swa);
+
+    XSetWindowAttributes xattr;
+
+    xattr.override_redirect = False;
+    XChangeWindowAttributes(xdisplay, xwin, CWOverrideRedirect, &xattr);
+
+    int one = 1;
+     XChangeProperty(
+        xdisplay, xwin,
+        XInternAtom ( xdisplay, "_HILDON_NON_COMPOSITED_WINDOW", False ),
+        XA_INTEGER,  32,  PropModeReplace,
+        (unsigned char*) &one,  1);
+
+     XWMHints hints;
+       hints.input = True;
+       hints.flags = InputHint;
+       XSetWMHints(xdisplay, xwin, &hints);
+
+    XMapWindow(xdisplay, xwin);
+    XStoreName(xdisplay, xwin, "tetra-demo");
+    initialized = true;
+}
+}
+
 namespace egl {
 //code in the kb::egl namespace deals with setting up EGL
 EGLDisplay display;
@@ -393,11 +449,17 @@ void debugMessageCallback(GLenum source, GLenum type, GLuint id,
 }
 
 void init_egl(bool debug = false) {
+    bool offscreen = !x11::is_initialized();
+
     eglCheck(eglBindAPI(EGL_OPENGL_API));
-    eglCheck(display = eglGetDisplay(EGL_DEFAULT_DISPLAY));
+    if(offscreen) {
+        eglCheck(display = eglGetDisplay(EGL_DEFAULT_DISPLAY));
+    } else {
+        eglCheck(display = eglGetDisplay(x11::get_x11_display()));
+    }
     eglCheck(eglInitialize(display, nullptr, nullptr));
 
-    const EGLint attributes[] = {
+    const EGLint egl_config_constraints[] = {
     EGL_BUFFER_SIZE, static_cast<EGLint>(24),
     EGL_DEPTH_SIZE, static_cast<EGLint>(24),
     EGL_STENCIL_SIZE, static_cast<EGLint>(0),
@@ -413,14 +475,19 @@ void init_egl(bool debug = false) {
     EGLint configCount;
     EGLConfig configs[1];
 
-    eglCheck(eglChooseConfig(display, attributes, configs, 1, &configCount));
+    eglCheck(eglChooseConfig(display, egl_config_constraints, configs, 1, &configCount));
     EGLint attrib_list[] = { EGL_WIDTH, WIDTH, EGL_HEIGHT, HEIGHT, EGL_NONE };
 
-    eglCheck(surface = eglCreatePbufferSurface(display, configs[0], attrib_list));
+    if(!offscreen) {
+        eglCheck(surface = eglCreateWindowSurface(display, configs[0], x11::get_x11_window(), nullptr));
+    } else {
+        eglCheck(surface = eglCreatePbufferSurface(display, configs[0], attrib_list));
+    }
 
-    const EGLint contextVersion[] = { EGL_CONTEXT_CLIENT_VERSION, 1, EGL_CONTEXT_OPENGL_DEBUG, debug ? EGL_TRUE : EGL_FALSE, EGL_NONE };
+    const EGLint contextVersion[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_CONTEXT_OPENGL_DEBUG, debug ? EGL_TRUE : EGL_FALSE, EGL_NONE };
     eglCheck(context = eglCreateContext(display, configs[0], EGL_NO_CONTEXT, contextVersion));
     eglCheck(eglMakeCurrent(display, surface, surface, context));
+    eglCheck(eglSwapInterval(display, 1));
     if(debug) {
         glCheck(glEnable(GL_DEBUG_OUTPUT_SYNCHRONOUS));
         auto glDebugMessageCallback  = (void (*)(void *, void *))eglGetProcAddress("glDebugMessageCallback");
@@ -441,26 +508,26 @@ std::string get_info() {
 namespace gl {
 //code in the kb::gl namespace deals with OpenGL (and OpenCV/GL) internals
 cv::ogl::Texture2D *frame_buf_tex;
-GLuint frame_buf_tex_name;
+GLuint frame_buf;
+
 
 void init_gl() {
     glewInit();
 
     cv::ogl::ocl::initializeContextFromGL();
 
-    frame_buf_tex_name = 0;
-    glCheck(glGenFramebuffers(1, &frame_buf_tex_name));
-    glCheck(glBindFramebuffer(GL_FRAMEBUFFER, frame_buf_tex_name));
+    frame_buf = 0;
+    glCheck(glGenFramebuffers(1, &frame_buf));
+    glCheck(glBindFramebuffer(GL_FRAMEBUFFER, frame_buf));
     frame_buf_tex = new cv::ogl::Texture2D(cv::Size(WIDTH, HEIGHT), cv::ogl::Texture2D::RGBA, false);
     frame_buf_tex->bind();
-
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
     glCheck(glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, frame_buf_tex->texId(), 0));
-    GLenum drawBuffers[1] = { GL_COLOR_ATTACHMENT0 };
-    glCheck(glDrawBuffers(1, drawBuffers));
+
     assert(glCheckFramebufferStatus(GL_FRAMEBUFFER) == GL_FRAMEBUFFER_COMPLETE);
-    glCheck(glBindFramebuffer(GL_FRAMEBUFFER, frame_buf_tex_name));
 
-    glCheck(glClearColor(0.1, 0.39, 0.88, 1.0));
+    glCheck(glViewport(0, 0, WIDTH, HEIGHT));
     glCheck(glColor3f(1.0, 1.0, 1.0));
 
     glCheck(glEnable(GL_CULL_FACE));

src/tetra/tetra-demo.cpp

@@ -2,24 +2,86 @@
 
 constexpr long unsigned int WIDTH = 1920;
 constexpr long unsigned int HEIGHT = 1080;
-constexpr double FPS = 30;
+constexpr double FPS = 24;
 
 #include "subsystems.hpp"
 
 using std::cerr;
 using std::endl;
 
+cv::ocl::OpenCLExecutionContext VA_CONTEXT;
+cv::ocl::OpenCLExecutionContext GL_CONTEXT;
+
+void render(cv::UMat& frameBuffer) {
+    glBindFramebuffer(GL_FRAMEBUFFER, kb::gl::frame_buf);
+    glViewport(0, 0, WIDTH , HEIGHT );
+    glRotatef(1, 0, 1, 0);
+    glClearColor(0.0f, 0.0f, 1.0f, 1.0f);
+    glClear(GL_COLOR_BUFFER_BIT);
+
+    glColor3f(1.0, 1.0, 1.0);
+    glBegin(GL_LINES);
+    for (GLfloat i = -2.5; i <= 2.5; i += 0.25) {
+        glVertex3f(i, 0, 2.5);
+        glVertex3f(i, 0, -2.5);
+        glVertex3f(2.5, 0, i);
+        glVertex3f(-2.5, 0, i);
+    }
+    glEnd();
+
+    glBegin(GL_TRIANGLE_STRIP);
+        glColor3f(1, 1, 1);
+        glVertex3f(0, 2, 0);
+        glColor3f(1, 0, 0);
+        glVertex3f(-1, 0, 1);
+        glColor3f(0, 1, 0);
+        glVertex3f(1, 0, 1);
+        glColor3f(0, 0, 1);
+        glVertex3f(0, 0, -1.4);
+        glColor3f(1, 1, 1);
+        glVertex3f(0, 2, 0);
+        glColor3f(1, 0, 0);
+        glVertex3f(-1, 0, 1);
+    glEnd();
+    glFlush();
+    kb::gl::swapBuffers();
+}
+
+void blitFrameBufferToScreen() {
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, kb::gl::frame_buf);
+    glReadBuffer(GL_COLOR_ATTACHMENT0);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+    glViewport(0, 0, WIDTH, HEIGHT);
+    glBlitFramebuffer(0, 0, WIDTH, HEIGHT, 0, 0, WIDTH, HEIGHT, GL_COLOR_BUFFER_BIT, GL_NEAREST);
+}
+
+void glow(cv::UMat &frameBuffer, cv::UMat &mask, double sigma = 50) {
+    cv::blur(frameBuffer, mask, cv::Size(sigma, sigma));
+    cv::bitwise_not(mask, mask);
+    cv::bitwise_not(frameBuffer, frameBuffer);
+    mask.assignTo(mask, CV_16U);
+    frameBuffer.assignTo(frameBuffer, CV_16U);
+    cv::multiply(mask, frameBuffer, mask);
+    cv::divide(mask, cv::Scalar::all(255.0), mask);
+    mask.assignTo(mask, CV_8U);
+    cv::bitwise_not(mask, frameBuffer);
+}
+
 int main(int argc, char **argv) {
     using namespace kb;
 
     va::init_va();
     cv::VideoWriter video("tetra-demo.mkv", cv::CAP_FFMPEG, cv::VideoWriter::fourcc('V', 'P', '9', '0'), FPS, cv::Size(WIDTH, HEIGHT), { cv::VIDEOWRITER_PROP_HW_DEVICE, 0, cv::VIDEOWRITER_PROP_HW_ACCELERATION, cv::VIDEO_ACCELERATION_VAAPI, cv::VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL, 1 });
-    cv::ocl::OpenCLExecutionContext vaContext = cv::ocl::OpenCLExecutionContext::getCurrent();
+    VA_CONTEXT = cv::ocl::OpenCLExecutionContext::getCurrent();
 
+    //comment the next line for offscreen rendering
+    x11::init_x11();
+
+    //Passing true to init_egl will create a OpenGL debug context
     egl::init_egl();
     gl::init_gl();
 
-    cv::ocl::OpenCLExecutionContext glContext = cv::ocl::OpenCLExecutionContext::getCurrent();
+    GL_CONTEXT = cv::ocl::OpenCLExecutionContext::getCurrent();
 
     cerr << "VA Version: " << va::get_info() << endl;
     cerr << "EGL Version: " << egl::get_info() << endl;
@@ -30,72 +92,47 @@ int main(int argc, char **argv) {
     cv::UMat mask;
     cv::UMat videoFrame;
 
-    double sigma = 50;
     int64 start = 0;
     uint64_t cnt = 0;
 
     while (true) {
         start = cv::getTickCount();
 
-        //Draw a rotating tetrahedron
-        glContext.bind();
-        glRotatef(1, 0, 1, 0);
-        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
-
-        glColor3f(1.0, 1.0, 1.0);
-        glBegin(GL_LINES);
-        for (GLfloat i = -2.5; i <= 2.5; i += 0.25) {
-            glVertex3f(i, 0, 2.5);
-            glVertex3f(i, 0, -2.5);
-            glVertex3f(2.5, 0, i);
-            glVertex3f(-2.5, 0, i);
-        }
-        glEnd();
-
-        glBegin(GL_TRIANGLE_STRIP);
-            glColor3f(1, 1, 1);
-            glVertex3f(0, 2, 0);
-            glColor3f(1, 0, 0);
-            glVertex3f(-1, 0, 1);
-            glColor3f(0, 1, 0);
-            glVertex3f(1, 0, 1);
-            glColor3f(0, 0, 1);
-            glVertex3f(0, 0, -1.4);
-            glColor3f(1, 1, 1);
-            glVertex3f(0, 2, 0);
-            glColor3f(1, 0, 0);
-            glVertex3f(-1, 0, 1);
-        glEnd();
-
-        glFlush();
-        gl::swapBuffers();
-        cl::fetch_frame_buffer(frameBuffer); //hand over the data (GPU 2 GPU) to OpenCV/OpenCL
-
-        //Using OpenCL in the background
-        cv::flip(frameBuffer, frameBuffer, 0); //  flip the image in the y-axis
-
-        {
-            //Do a glow effect using blur
-            cv::blur(frameBuffer, mask, cv::Size(sigma, sigma));
-            cv::bitwise_not(mask, mask);
-            cv::bitwise_not(frameBuffer, frameBuffer);
-            mask.assignTo(mask, CV_16U);
-            frameBuffer.assignTo(frameBuffer, CV_16U);
-            cv::multiply(mask, frameBuffer, mask);
-            cv::divide(mask, cv::Scalar::all(255.0), mask);
-            mask.assignTo(mask, CV_8U);
-            cv::bitwise_not(mask, frameBuffer);
-        }
+        GL_CONTEXT.bind();
+
+        //Using OpenGL, render a rotating tetrahedron
+        render(frameBuffer);
+
+        //Transfer buffer ownership to OpenCL
+        cl::fetch_frame_buffer(frameBuffer);
+
+        //Using OpenCL for a glow effect
+        glow(frameBuffer, mask);
+
+        //Color-conversion from BGRA to RGB, also OpenCL.
+        cv::cvtColor(frameBuffer, videoFrame, cv::COLOR_BGRA2RGB);
 
-        cv::cvtColor(frameBuffer, videoFrame, cv::COLOR_BGRA2RGB); // Color-conversion from BGRA to RGB
+        VA_CONTEXT.bind();
+        //Encode the frame using VAAPI on the GPU.
+        video.write(videoFrame);
 
-        vaContext.bind();
-        video.write(videoFrame); //encode the frame using VAAPI on the GPU.
+        GL_CONTEXT.bind();
 
+        if(x11::is_initialized()) {
+            //Transfer buffer ownership back to OpenGL
+            cl::return_frame_buffer(frameBuffer);
+
+            //Blit the framebuffer we have been working on to screen
+            blitFrameBufferToScreen();
+        }
+
+        //Measure FPS
         int64 tick = cv::getTickCount();
         double tickFreq = cv::getTickFrequency();
-        if (cnt % int64(ceil(tickFreq / (FPS * 10000000))) == 0)
+        if (cnt % int64(ceil(tickFreq / (FPS * 10000000))) == 0) {
             cerr << "FPS : " << tickFreq / (tick - start + 1) << '\r';
+            cnt = 0;
+        }
 
         ++cnt;
     }