// This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // Copyright Amir Hassan (kallaballa) #include "opencv2/v4d/v4d.hpp" #include "detail/clvacontext.hpp" #include "detail/framebuffercontext.hpp" #include "detail/glcontext.hpp" #include "detail/nanovgcontext.hpp" #include "detail/nanoguicontext.hpp" #include "opencv2/v4d/dialog.hpp" #include "opencv2/v4d/formhelper.hpp" #include namespace cv { namespace v4d { namespace detail { void glfw_error_callback(int error, const char* description) { fprintf(stderr, "GLFW Error: (%d) %s\n", error, description); } } void gl_check_error(const std::filesystem::path& file, unsigned int line, const char* expression) { int errorCode = glGetError(); if (errorCode != 0) { std::stringstream ss; ss << "GL failed in " << file.filename() << " (" << line << ") : " << "\nExpression:\n " << expression << "\nError code:\n " << errorCode; throw std::runtime_error(ss.str()); } } cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code) { cv::Mat tmpIn(1, 1, CV_8UC3); cv::Mat tmpOut(1, 1, CV_8UC3); tmpIn.at(0, 0) = cv::Vec3b(src[0], src[1], src[2]); cvtColor(tmpIn, tmpOut, code); const cv::Vec3b& vdst = tmpOut.at(0, 0); cv::Scalar dst(vdst[0], vdst[1], vdst[2], src[3]); return dst; } cv::Ptr V4D::make(const cv::Size& size, const cv::Size& fbsize, const string& title, bool offscreen, bool debug, int major, int minor, bool compat, int samples) { cv::Ptr v4d = new V4D(size, fbsize, title, offscreen, debug, major, minor, false, 0); v4d->setVisible(true); return v4d; } V4D::V4D(const cv::Size& size, const cv::Size& fbsize, const string& title, bool offscreen, bool debug, int major, int minor, bool compat, int samples) : initialSize_(size), offscreen_(offscreen), title_(title), major_(major), minor_(minor), compat_( compat), samples_(samples), debug_(debug), viewport_(0, 0, size.width, size.height), scale_( 1), mousePos_(0, 0), stretch_(true), pool_(2) { #ifdef __EMSCRIPTEN__ printf(""); //makes sure we have FS as a dependency #endif mainFbContext_ = new detail::FrameBufferContext(*this, fbsize.empty() ? size : fbsize, offscreen_, title_, major_, minor_, compat_, samples_, debug_, nullptr, nullptr); nvgContext_ = new detail::NanoVGContext(*this, *mainFbContext_); nguiContext_ = new detail::NanoguiContext(*this, *mainFbContext_); clvaContext_ = new detail::CLVAContext(*this, *mainFbContext_); glContext_ = new detail::GLContext(*this, *mainFbContext_); } V4D::~V4D() { //don't delete form_. it is autmatically cleaned up by the base class (nanogui::Screen) if (glContext_) delete glContext_; if (nvgContext_) delete nvgContext_; if (nguiContext_) delete nguiContext_; if (clvaContext_) delete clvaContext_; if (mainFbContext_) delete mainFbContext_; } cv::ogl::Texture2D& V4D::texture() { return mainFbContext_->getTexture2D(); } void V4D::setKeyboardEventCallback( std::function fn) { keyEventCb_ = fn; } bool V4D::keyboard_event(int key, int scancode, int action, int modifiers) { if (keyEventCb_) return keyEventCb_(key, scancode, action, modifiers); return nguiCtx().screen().keyboard_event(key, scancode, action, modifiers); } FrameBufferContext& V4D::fbCtx() { assert(mainFbContext_ != nullptr); return *mainFbContext_; } CLVAContext& V4D::clvaCtx() { assert(clvaContext_ != nullptr); return *clvaContext_; } NanoVGContext& V4D::nvgCtx() { assert(nvgContext_ != nullptr); return *nvgContext_; } NanoguiContext& V4D::nguiCtx() { assert(nguiContext_ != nullptr); return *nguiContext_; } GLContext& V4D::glCtx() { assert(glContext_ != nullptr); return *glContext_; } bool V4D::hasFbCtx() { return mainFbContext_ != nullptr; } bool V4D::hasClvaCtx() { return clvaContext_ != nullptr; } bool V4D::hasNvgCtx() { return nvgContext_ != nullptr; } bool V4D::hasNguiCtx() { return nguiContext_ != nullptr; } bool V4D::hasGlCtx() { return glContext_ != nullptr; } cv::Size V4D::getVideoFrameSize() { return clvaCtx().getVideoFrameSize(); } void V4D::gl(std::function fn) { glCtx().render([=](const cv::Size& sz) { CV_UNUSED(sz); fn(); }); } void V4D::gl(std::function fn) { glCtx().render(fn); } void V4D::fb(std::function fn) { fbCtx().execute(fn); } void V4D::nvg(std::function fn) { nvgCtx().render([fn](const cv::Size& sz) { CV_UNUSED(sz); fn(); }); } void V4D::nvg(std::function fn) { nvgCtx().render(fn); } void V4D::nanogui(std::function fn) { nguiCtx().build(fn); } #ifdef __EMSCRIPTEN__ static void do_frame(void* void_fn_ptr) { auto* fn_ptr = reinterpret_cast*>(void_fn_ptr); if (fn_ptr) { auto& fn = *fn_ptr; fn(); } } #endif void V4D::run(std::function fn) { #ifndef __EMSCRIPTEN__ while (keepRunning() && fn()) ; #else emscripten_set_main_loop_arg(do_frame, &fn, -1, true); #endif } void V4D::setSource(const Source& src) { if (!clvaCtx().hasContext()) clvaCtx().copyContext(); source_ = src; } void V4D::feed(cv::InputArray& in) { cv::UMat frame; clvaCtx().capture([&](cv::UMat& videoFrame) { in.copyTo(videoFrame); }, frame); fb([frame](cv::UMat& frameBuffer){ cvtColor(frame,frameBuffer, cv::COLOR_RGB2BGRA); }); } bool V4D::capture() { return this->capture([&](cv::UMat& videoFrame) { if (source_.isReady()) source_().second.copyTo(videoFrame); }); } bool V4D::capture(std::function fn) { if (!source_.isReady() || !source_.isOpen()) { #ifndef __EMSCRIPTEN__ return false; #else return true; #endif } if (futureReader_.valid()) { if (!futureReader_.get()) { #ifndef __EMSCRIPTEN__ return false; #else return true; #endif } } if(nextReaderFrame_.empty()) { if (!clvaCtx().capture(fn, nextReaderFrame_)) { #ifndef __EMSCRIPTEN__ return false; #else return true; #endif } } currentReaderFrame_ = nextReaderFrame_.clone(); futureReader_ = pool_.enqueue( [](V4D* v, std::function fn, cv::UMat& frame) { return v->clvaCtx().capture(fn, frame); }, this, fn, nextReaderFrame_); fb([this](cv::UMat& frameBuffer){ cvtColor(currentReaderFrame_,frameBuffer, cv::COLOR_RGB2BGRA); }); return true; } bool V4D::isSourceReady() { return source_.isReady(); } void V4D::setSink(const Sink& sink) { if (!clvaCtx().hasContext()) clvaCtx().copyContext(); sink_ = sink; } void V4D::write() { this->write([&](const cv::UMat& videoFrame) { if (sink_.isReady()) sink_(videoFrame); }); } void V4D::write(std::function fn) { if (!sink_.isReady() || !sink_.isOpen()) return; if (futureWriter_.valid()) futureWriter_.get(); fb([this](cv::UMat& frameBuffer){ frameBuffer.copyTo(currentWriterFrame_); }); futureWriter_ = pool_.enqueue([](V4D* v, std::function fn, cv::UMat& frame) { v->clvaCtx().write(fn, frame); }, this, fn, currentWriterFrame_); } bool V4D::isSinkReady() { return sink_.isReady(); } void V4D::clear(const cv::Scalar& bgra) { this->gl([&]() { const float& b = bgra[0] / 255.0f; const float& g = bgra[1] / 255.0f; const float& r = bgra[2] / 255.0f; const float& a = bgra[3] / 255.0f; GL_CHECK(glClearColor(r, g, b, a)); GL_CHECK(glClear(GL_COLOR_BUFFER_BIT)); }); } void V4D::showGui(bool s) { auto children = nguiCtx().screen().children(); for (auto* child : children) { child->set_visible(s); } } void V4D::setMouseDrag(bool d) { mouseDrag_ = d; } bool V4D::isMouseDrag() { return mouseDrag_; } void V4D::pan(int x, int y) { viewport_.x += x * scale_; viewport_.y += y * scale_; } void V4D::zoom(float factor) { if (scale_ == 1 && viewport_.x == 0 && viewport_.y == 0 && factor > 1) return; double oldScale = scale_; double origW = getFrameBufferSize().width; double origH = getFrameBufferSize().height; scale_ *= factor; if (scale_ <= 0.025) { scale_ = 0.025; return; } else if (scale_ > 1) { scale_ = 1; viewport_.width = origW; viewport_.height = origH; if (factor > 1) { viewport_.x += log10(((viewport_.x * (1.0 - factor)) / viewport_.width) * 9 + 1.0) * viewport_.width; viewport_.y += log10(((viewport_.y * (1.0 - factor)) / viewport_.height) * 9 + 1.0) * viewport_.height; } else { viewport_.x += log10(((-viewport_.x * (1.0 - factor)) / viewport_.width) * 9 + 1.0) * viewport_.width; viewport_.y += log10(((-viewport_.y * (1.0 - factor)) / viewport_.height) * 9 + 1.0) * viewport_.height; } return; } cv::Vec2f offset; double oldW = (origW * oldScale); double oldH = (origH * oldScale); viewport_.width = std::min(scale_ * origW, origW); viewport_.height = std::min(scale_ * origH, origH); float delta_x; float delta_y; if (factor < 1.0) { offset = cv::Vec2f(viewport_.x, viewport_.y) - cv::Vec2f(mousePos_[0], origH - mousePos_[1]); delta_x = offset[0] / oldW; delta_y = offset[1] / oldH; } else { offset = cv::Vec2f(viewport_.x - (viewport_.width / 2.0), viewport_.y - (viewport_.height / 2.0)) - cv::Vec2f(viewport_.x, viewport_.y); delta_x = offset[0] / oldW; delta_y = offset[1] / oldH; } float x_offset; float y_offset; x_offset = delta_x * (viewport_.width - oldW); y_offset = delta_y * (viewport_.height - oldH); if (factor < 1.0) { viewport_.x += x_offset; viewport_.y += y_offset; } else { viewport_.x += x_offset; viewport_.y += y_offset; } } cv::Vec2f V4D::getPosition() { fbCtx().makeCurrent(); int x, y; glfwGetWindowPos(getGLFWWindow(), &x, &y); return cv::Vec2f(x, y); } cv::Vec2f V4D::getMousePosition() { return mousePos_; } void V4D::setMousePosition(int x, int y) { mousePos_ = { float(x), float(y) }; } float V4D::getScale() { return scale_; } cv::Rect& V4D::viewport() { return viewport_; } float V4D::getXPixelRatio() { fbCtx().makeCurrent(); #ifdef __EMSCRIPTEN__ float r = emscripten_get_device_pixel_ratio(); return r; #else float xscale, yscale; glfwGetWindowContentScale(getGLFWWindow(), &xscale, &yscale); return xscale; #endif } float V4D::getYPixelRatio() { fbCtx().makeCurrent(); #ifdef __EMSCRIPTEN__ float r = emscripten_get_device_pixel_ratio(); return r; #else float xscale, yscale; glfwGetWindowContentScale(getGLFWWindow(), &xscale, &yscale); return yscale; #endif } cv::Size V4D::getNativeFrameBufferSize() { fbCtx().makeCurrent(); int w, h; glfwGetFramebufferSize(getGLFWWindow(), &w, &h); return cv::Size{w, h}; } cv::Size V4D::getFrameBufferSize() { return fbCtx().size(); } cv::Size V4D::getInitialSize() { return initialSize_; } void V4D::resizeWindow(const cv::Size& sz) { fbCtx().setWindowSize(sz); } bool V4D::isFullscreen() { fbCtx().makeCurrent(); return glfwGetWindowMonitor(getGLFWWindow()) != nullptr; } void V4D::setFullscreen(bool f) { fbCtx().makeCurrent(); auto monitor = glfwGetPrimaryMonitor(); const GLFWvidmode* mode = glfwGetVideoMode(monitor); if (f) { glfwSetWindowMonitor(getGLFWWindow(), monitor, 0, 0, mode->width, mode->height, mode->refreshRate); resizeWindow(getNativeFrameBufferSize()); } else { glfwSetWindowMonitor(getGLFWWindow(), nullptr, 0, 0, getInitialSize().width, getInitialSize().height, 0); resizeWindow(getInitialSize()); } } bool V4D::isResizable() { return fbCtx().isResizable(); } void V4D::setResizable(bool r) { fbCtx().setResizable(r); } bool V4D::isVisible() { return fbCtx().isVisible(); } void V4D::setVisible(bool v) { fbCtx().setVisible(v); nguiCtx().screen().perform_layout(); } bool V4D::isOffscreen() { return offscreen_; } void V4D::setOffscreen(bool o) { offscreen_ = o; setVisible(!o); } void V4D::setStretching(bool s) { stretch_ = s; } bool V4D::isStretching() { return stretch_; } void V4D::setDefaultKeyboardEventCallback() { setKeyboardEventCallback([&](int key, int scancode, int action, int modifiers) { CV_UNUSED(scancode); CV_UNUSED(modifiers); if (key == GLFW_KEY_ESCAPE && action == GLFW_PRESS) { setOffscreen(!isOffscreen()); return true; } else if (key == GLFW_KEY_TAB && action == GLFW_PRESS) { auto children = nguiCtx().screen().children(); for (auto* child : children) { child->set_visible(!child->visible()); } return true; } return false; }); } bool V4D::display() { bool result = true; if (!offscreen_) { // run_sync_on_main<9>([this](){ // FrameBufferContext::GLScope glScope(clvaCtx().fbCtx()); // clvaCtx().fbCtx().blitFrameBufferToScreen(viewport(), clvaCtx().fbCtx().getWindowSize(), isStretching()); // clvaCtx().fbCtx().makeCurrent(); // glfwSwapBuffers(clvaCtx().fbCtx().getGLFWWindow()); // }); // run_sync_on_main<10>([this](){ // FrameBufferContext::GLScope glScope(glCtx().fbCtx()); // glCtx().fbCtx().blitFrameBufferToScreen(viewport(), glCtx().fbCtx().getWindowSize(), isStretching()); // glCtx().fbCtx().makeCurrent(); // glfwSwapBuffers(glCtx().fbCtx().getGLFWWindow()); // }); // run_sync_on_main<11>([this](){ // FrameBufferContext::GLScope glScope(nvgCtx().fbCtx()); // nvgCtx().fbCtx().blitFrameBufferToScreen(viewport(), nvgCtx().fbCtx().getWindowSize(), isStretching()); // nvgCtx().fbCtx().makeCurrent(); // glfwSwapBuffers(nvgCtx().fbCtx().getGLFWWindow()); // }); // run_sync_on_main<12>([this](){ // FrameBufferContext::GLScope glScope(nguiCtx().fbCtx()); // nguiCtx().fbCtx().blitFrameBufferToScreen(viewport(), nguiCtx().fbCtx().getWindowSize(), isStretching()); // nguiCtx().fbCtx().makeCurrent(); // glfwSwapBuffers(nguiCtx().fbCtx().getGLFWWindow()); // }); nguiCtx().render(); run_sync_on_main<6>([&, this](){ FrameBufferContext::GLScope glScope(fbCtx(), GL_READ_FRAMEBUFFER); fbCtx().blitFrameBufferToScreen(viewport(), fbCtx().getWindowSize(), isStretching()); #ifndef __EMSCRIPTEN__ glfwSwapBuffers(fbCtx().getGLFWWindow()); #else emscripten_webgl_commit_frame(); #endif glfwPollEvents(); result = !glfwWindowShouldClose(getGLFWWindow()); }); #ifdef __EMSCRIPTEN__ run_sync_on_main<7>([this](){ cv::UMat tmp; cv::v4d::detail::FrameBufferContext::GLScope glScope(fbCtx()); cv::v4d::detail::FrameBufferContext::FrameBufferScope fbScope(fbCtx(), tmp); }); #endif } if(frameCnt_ == (std::numeric_limits().max() - 1)) frameCnt_ = 0; else ++frameCnt_; return result; } uint64_t V4D::frameCount() { return frameCnt_; } bool V4D::isClosed() { return closed_; } void V4D::close() { setVisible(false); closed_ = true; } GLFWwindow* V4D::getGLFWWindow() { return fbCtx().getGLFWWindow(); } void V4D::printSystemInfo() { run_sync_on_main<8>([this](){ fbCtx().makeCurrent(); cerr << "OpenGL Version: " << getGlInfo() << endl; cerr << "OpenCL Platforms: " << getClInfo() << endl; }); } void V4D::updateFps(bool graphical) { if (frameCount() > 0) { tick_.stop(); if (tick_.getTimeMilli() > 50) { cerr << "FPS : " << (fps_ = tick_.getFPS()); #ifndef __EMSCRIPTEN__ cerr << '\r'; #else cerr << endl; #endif tick_.reset(); } if (graphical) { this->nvg([this]() { glClear(GL_DEPTH_BUFFER_BIT); using namespace cv::v4d::nvg; string txt = "FPS: " + std::to_string(fps_); beginPath(); roundedRect(5, 5, 15 * txt.size() + 5, 30, 5); fillColor(cv::Scalar(255, 255, 255, 180)); fill(); fontSize(30.0f); fontFace("mono"); fillColor(cv::Scalar(90, 90, 90, 255)); textAlign(NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE); text(10, 20, txt.c_str(), nullptr); }); } } tick_.start(); } } }