From 27f2d51f3fe72e5d03ea70a88318e81c42f094ad Mon Sep 17 00:00:00 2001 From: kallaballa Date: Fri, 30 Jun 2023 05:22:02 +0200 Subject: [PATCH] various fixes --- modules/v4d/include/opencv2/v4d/util.hpp | 11 ++++ modules/v4d/include/opencv2/v4d/v4d.hpp | 10 +--- modules/v4d/samples/beauty-demo.cpp | 21 +++---- modules/v4d/samples/optflow-demo.cpp | 60 +++++++++++++++---- modules/v4d/src/detail/framebuffercontext.cpp | 27 +++++---- modules/v4d/src/util.cpp | 12 ++++ modules/v4d/src/v4d.cpp | 17 +----- 7 files changed, 105 insertions(+), 53 deletions(-) diff --git a/modules/v4d/include/opencv2/v4d/util.hpp b/modules/v4d/include/opencv2/v4d/util.hpp index b9d00cb10..eb3bacb90 100644 --- a/modules/v4d/include/opencv2/v4d/util.hpp +++ b/modules/v4d/include/opencv2/v4d/util.hpp @@ -13,6 +13,8 @@ #include #include #include +#include + #ifdef __EMSCRIPTEN__ # include # include @@ -89,6 +91,15 @@ CV_EXPORTS size_t cnz(const cv::UMat& m); } using std::string; class V4D; + +/*! + * Convenience function to color convert from Scalar to Scalar + * @param src The scalar to color convert + * @param code The color converions code + * @return The color converted scalar + */ +CV_EXPORTS cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code); + #ifdef __EMSCRIPTEN__ CV_EXPORTS Mat read_embedded_image(const string &path); #endif diff --git a/modules/v4d/include/opencv2/v4d/v4d.hpp b/modules/v4d/include/opencv2/v4d/v4d.hpp index 6ab468194..e904c861d 100644 --- a/modules/v4d/include/opencv2/v4d/v4d.hpp +++ b/modules/v4d/include/opencv2/v4d/v4d.hpp @@ -84,14 +84,6 @@ template void find_widgets(const nanogui::Widget* parent, std::vecto } } -/*! - * Convenience function to color convert from Scalar to Scalar - * @param src The scalar to color convert - * @param code The color converions code - * @return The color converted scalar - */ -CV_EXPORTS cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code); - using namespace cv::v4d::detail; class CV_EXPORTS V4D { @@ -105,7 +97,7 @@ class CV_EXPORTS V4D { cv::Rect viewport_; float zoomScale_; cv::Vec2f mousePos_; - bool stretch_; + bool scaling_; FrameBufferContext* mainFbContext_ = nullptr; CLVAContext* clvaContext_ = nullptr; GLContext* glContext_ = nullptr; diff --git a/modules/v4d/samples/beauty-demo.cpp b/modules/v4d/samples/beauty-demo.cpp index 5cfae4bea..162a90686 100644 --- a/modules/v4d/samples/beauty-demo.cpp +++ b/modules/v4d/samples/beauty-demo.cpp @@ -200,13 +200,14 @@ static void adjust_saturation(const cv::UMat &srcBGR, cv::UMat &dstBGR, float fa cvtColor(hls, dstBGR, cv::COLOR_HLS2BGR); } +//Built the GUI static void setup_gui(cv::Ptr v) { v->nanogui([&](cv::v4d::FormHelper& form){ form.makeDialog(5, 30, "Effect"); form.makeGroup("Display"); form.makeFormVariable("Side by side", side_by_side, "Enable or disable side by side view"); - auto* scaleVar = form.makeFormVariable("Stretch", scale, "Enable or disable stetching to the window size"); + auto* scaleVar = form.makeFormVariable("Scale", scale, "Enable or disable scaling to the window size"); scaleVar->set_callback([=](const bool& b) { v->setScaling(b); scale = b; @@ -277,11 +278,12 @@ static bool iteration() { if (!window->capture()) return false; + //Save the video frame as BGR window->fb([&](cv::UMat &frameBuffer) { cvtColor(frameBuffer, input, cv::COLOR_BGRA2BGR); }); - //Downscale the input for face detection + //Downscale the video frame for face detection cv::resize(input, down, cv::Size(DOWNSIZE_WIDTH, DOWNSIZE_HEIGHT)); shapes.clear(); @@ -349,6 +351,7 @@ static bool iteration() { frameOutFloat.convertTo(frameOut, CV_8U, 1.0); if (side_by_side) { + //create side-by-side view with a result cv::resize(input, lhalf, cv::Size(0, 0), 0.5, 0.5); cv::resize(frameOut, rhalf, cv::Size(0, 0), 0.5, 0.5); @@ -356,12 +359,9 @@ static bool iteration() { lhalf.copyTo(frameOut(cv::Rect(0, 0, lhalf.size().width, lhalf.size().height))); rhalf.copyTo(frameOut(cv::Rect(rhalf.size().width, 0, rhalf.size().width, rhalf.size().height))); } - - window->fb([&](cv::UMat &frameBuffer) { - cvtColor(frameOut, frameBuffer, cv::COLOR_BGR2BGRA); - }); } else { if (side_by_side) { + //create side-by-side view without a result (using the input image for both sides) frameOut = cv::Scalar::all(0); cv::resize(input, lhalf, cv::Size(0, 0), 0.5, 0.5); lhalf.copyTo(frameOut(cv::Rect(0, 0, lhalf.size().width, lhalf.size().height))); @@ -369,12 +369,13 @@ static bool iteration() { } else { input.copyTo(frameOut); } - - window->fb([&](cv::UMat &frameBuffer) { - cvtColor(frameOut, frameBuffer, cv::COLOR_BGR2BGRA); - }); } + //write the result to the framebuffer + window->fb([&](cv::UMat &frameBuffer) { + cvtColor(frameOut, frameBuffer, cv::COLOR_BGR2BGRA); + }); + window->write(); return window->display(); diff --git a/modules/v4d/samples/optflow-demo.cpp b/modules/v4d/samples/optflow-demo.cpp index 55d9b5473..fe8bd428f 100644 --- a/modules/v4d/samples/optflow-demo.cpp +++ b/modules/v4d/samples/optflow-demo.cpp @@ -41,10 +41,13 @@ constexpr bool OFFSCREEN = false; cv::Ptr window; #ifndef __EMSCRIPTEN__ +//create a separate window in native builds. In WASM builds instead create a light-weight dialog cv::Ptr menuWindow; #endif /* Visualization parameters */ + +//How the background will be visualized enum BackgroundModes { GREY, COLOR, @@ -52,6 +55,7 @@ enum BackgroundModes { BLACK }; +//Post-processing modes for the foreground enum PostProcModes { GLOW, BLOOM, @@ -59,12 +63,8 @@ enum PostProcModes { }; // Generate the foreground at this scale. -#ifndef __EMSCRIPTEN__ -float fg_scale = 0.5f; -#else float fg_scale = 0.5f; -#endif -// On every frame the foreground loses on brightness. specifies the loss in percent. +// On every frame the foreground loses on brightness. Specifies the loss in percent. #ifndef __EMSCRIPTEN__ float fg_loss = 2.5; #else @@ -116,15 +116,18 @@ int bloom_thresh = 210; //The intensity of the bloom filter float bloom_gain = 3; +//Uses background subtraction to generate a "motion mask" static void prepare_motion_mask(const cv::UMat& srcGrey, cv::UMat& motionMaskGrey) { static cv::Ptr bg_subtrator = cv::createBackgroundSubtractorMOG2(100, 16.0, false); static int morph_size = 1; static cv::Mat element = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(2 * morph_size + 1, 2 * morph_size + 1), cv::Point(morph_size, morph_size)); bg_subtrator->apply(srcGrey, motionMaskGrey); + //Surpress speckles cv::morphologyEx(motionMaskGrey, motionMaskGrey, cv::MORPH_OPEN, element, cv::Point(element.cols >> 1, element.rows >> 1), 2, cv::BORDER_CONSTANT, cv::morphologyDefaultBorderValue()); } +//Detect points to track static void detect_points(const cv::UMat& srcMotionMaskGrey, vector& points) { static cv::Ptr detector = cv::FastFeatureDetector::create(1, false); static vector tmpKeyPoints; @@ -138,6 +141,7 @@ static void detect_points(const cv::UMat& srcMotionMaskGrey, vector } } +//Detect extrem changes in scene content and report it static bool detect_scene_change(const cv::UMat& srcMotionMaskGrey, const float thresh, const float theshDiff) { static float last_movement = 0; @@ -152,6 +156,7 @@ static bool detect_scene_change(const cv::UMat& srcMotionMaskGrey, const float t return result; } +//Visualize the sparse optical flow static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UMat &nextGrey, const vector &detectedPoints, const float scaleFactor, const int maxStrokeSize, const cv::Scalar color, const int maxPoints, const float pointLossPercent) { static vector hull, prevPoints, nextPoints, newPoints; static vector upPrevPoints, upNextPoints; @@ -160,25 +165,33 @@ static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UM static std::random_device rd; static std::mt19937 g(rd()); + //less then 5 points is a degenerate case (e.g. the corners of a video frame) if (detectedPoints.size() > 4) { cv::convexHull(detectedPoints, hull); float area = cv::contourArea(hull); + //make sure the area of the point cloud is positive if (area > 0) { float density = (detectedPoints.size() / area); + //stroke size is biased by the area of the point cloud float strokeSize = maxStrokeSize * pow(area / (nextGrey.cols * nextGrey.rows), 0.33f); + //max points is biased by the densitiy of the point cloud size_t currentMaxPoints = ceil(density * maxPoints); + //lose a number of random points specified by pointLossPercent std::shuffle(prevPoints.begin(), prevPoints.end(), g); prevPoints.resize(ceil(prevPoints.size() * (1.0f - (pointLossPercent / 100.0f)))); + //calculate how many newly detected points to add size_t copyn = std::min(detectedPoints.size(), (size_t(std::ceil(currentMaxPoints)) - prevPoints.size())); if (prevPoints.size() < currentMaxPoints) { std::copy(detectedPoints.begin(), detectedPoints.begin() + copyn, std::back_inserter(prevPoints)); } + //calculate the sparse optical flow cv::calcOpticalFlowPyrLK(prevGrey, nextGrey, prevPoints, nextPoints, status, err); newPoints.clear(); if (prevPoints.size() > 1 && nextPoints.size() > 1) { + //scale the points to original size upNextPoints.clear(); upPrevPoints.clear(); for (cv::Point2f pt : prevPoints) { @@ -190,20 +203,29 @@ static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UM } using namespace cv::v4d::nvg; + //start drawing beginPath(); strokeWidth(strokeSize); strokeColor(color); for (size_t i = 0; i < prevPoints.size(); i++) { - if (status[i] == 1 && err[i] < (1.0 / density) && upNextPoints[i].y >= 0 && upNextPoints[i].x >= 0 && upNextPoints[i].y < nextGrey.rows / scaleFactor && upNextPoints[i].x < nextGrey.cols / scaleFactor) { + if (status[i] == 1 //point was found in prev and new set + && err[i] < (1.0 / density) //with a higher density be more sensitive to the feature error + && upNextPoints[i].y >= 0 && upNextPoints[i].x >= 0 //check bounds + && upNextPoints[i].y < nextGrey.rows / scaleFactor && upNextPoints[i].x < nextGrey.cols / scaleFactor //check bounds + ) { float len = hypot(fabs(upPrevPoints[i].x - upNextPoints[i].x), fabs(upPrevPoints[i].y - upNextPoints[i].y)); + //upper and lower bound of the flow vector lengthss if (len > 0 && len < sqrt(area)) { + //collect new points newPoints.push_back(nextPoints[i]); + //the actual drawing operations moveTo(upNextPoints[i].x, upNextPoints[i].y); lineTo(upPrevPoints[i].x, upPrevPoints[i].y); } } } + //end drawing stroke(); } prevPoints = newPoints; @@ -211,6 +233,7 @@ static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UM } } +//Bloom post-processing effect static void bloom(const cv::UMat& src, cv::UMat &dst, int ksize = 3, int threshValue = 235, float gain = 4) { static cv::UMat bgr; static cv::UMat hls; @@ -219,21 +242,29 @@ static void bloom(const cv::UMat& src, cv::UMat &dst, int ksize = 3, int threshV static cv::UMat blur; static std::vector hlsChannels; + //remove alpha channel cv::cvtColor(src, bgr, cv::COLOR_BGRA2RGB); + //convert to hls cv::cvtColor(bgr, hls, cv::COLOR_BGR2HLS); + //split channels cv::split(hls, hlsChannels); + //invert lightness cv::bitwise_not(hlsChannels[2], hlsChannels[2]); - + //multiply lightness and saturation cv::multiply(hlsChannels[1], hlsChannels[2], ls16, 1, CV_16U); + //normalize cv::divide(ls16, cv::Scalar(255.0), ls, 1, CV_8U); + //binary threhold according to threshValue cv::threshold(ls, blur, threshValue, 255, cv::THRESH_BINARY); - + //blur cv::boxFilter(blur, blur, -1, cv::Size(ksize, ksize), cv::Point(-1,-1), true, cv::BORDER_REPLICATE); + //convert to BGRA cv::cvtColor(blur, blur, cv::COLOR_GRAY2BGRA); - + //add src and the blurred L-S-product according to gain addWeighted(src, 1.0, blur, gain, 0, dst); } +//Glow post-processing effect static void glow_effect(const cv::UMat &src, cv::UMat &dst, const int ksize) { static cv::UMat resize; static cv::UMat blur; @@ -256,15 +287,19 @@ static void glow_effect(const cv::UMat &src, cv::UMat &dst, const int ksize) { cv::bitwise_not(dst, dst); } +//Compose the different layers into the final image static void composite_layers(cv::UMat& background, const cv::UMat& foreground, const cv::UMat& frameBuffer, cv::UMat& dst, int kernelSize, float fgLossPercent, BackgroundModes bgMode, PostProcModes ppMode) { static cv::UMat tmp; static cv::UMat post; static cv::UMat backgroundGrey; static vector channels; + //Lose a bit of foreground brightness based on fgLossPercent cv::subtract(foreground, cv::Scalar::all(255.0f * (fgLossPercent / 100.0f)), foreground); + //Add foreground an the current framebuffer into foregound cv::add(foreground, frameBuffer, foreground); + //Dependin on bgMode prepare the background in different ways switch (bgMode) { case GREY: cv::cvtColor(background, backgroundGrey, cv::COLOR_BGRA2GRAY); @@ -285,6 +320,7 @@ static void composite_layers(cv::UMat& background, const cv::UMat& foreground, c break; } + //Depending on ppMode perform post-processing switch (ppMode) { case GLOW: glow_effect(foreground, post, kernelSize); @@ -299,9 +335,11 @@ static void composite_layers(cv::UMat& background, const cv::UMat& foreground, c break; } + //Add background and post-processed foreground into dst cv::add(background, post, dst); } +//Build the GUI static void setup_gui(cv::Ptr main, cv::Ptr menu) { main->nanogui([&](cv::v4d::FormHelper& form){ form.makeDialog(5, 30, "Effects"); @@ -379,7 +417,7 @@ static void setup_gui(cv::Ptr main, cv::Ptr menu) { form.makeGroup("Display"); form.makeFormVariable("Show FPS", show_fps, "Enable or disable the On-screen FPS display"); - form.makeFormVariable("Stetch", scale, "Stretch the frame buffer to the window size")->set_callback([=](const bool &s) { + form.makeFormVariable("Scale", scale, "Scale the frame buffer to the window size")->set_callback([=](const bool &s) { main->setScaling(s); }); @@ -409,7 +447,9 @@ static bool iteration() { return false; window->fb([&](cv::UMat& frameBuffer) { + //resize to foreground scale cv::resize(frameBuffer, down, cv::Size(window->framebufferSize().width * fg_scale, window->framebufferSize().height * fg_scale)); + //save video background frameBuffer.copyTo(background); }); diff --git a/modules/v4d/src/detail/framebuffercontext.cpp b/modules/v4d/src/detail/framebuffercontext.cpp index eecabf149..6237af540 100644 --- a/modules/v4d/src/detail/framebuffercontext.cpp +++ b/modules/v4d/src/detail/framebuffercontext.cpp @@ -663,26 +663,33 @@ CLExecContext_t& FrameBufferContext::getCLExecContext() { #endif void FrameBufferContext::blitFrameBufferToScreen(const cv::Rect& viewport, - const cv::Size& windowSize, bool stretch, GLuint drawFramebufferID) { + const cv::Size& windowSize, bool scale, GLuint drawFramebufferID) { this->makeCurrent(); double hf = double(windowSize.height) / frameBufferSize_.height; double wf = double(windowSize.width) / frameBufferSize_.width; - double f = std::min(hf, wf); + double f; + if (frameBufferSize_.width > frameBufferSize_.height) + f = wf; + else + f = hf; + + double fbws = frameBufferSize_.width * f; + double fbhs = frameBufferSize_.height * f; - double wn = frameBufferSize_.width * f; - double hn = frameBufferSize_.height * f; - double xn = windowSize.width - wn; - double yn = windowSize.height - hn; + double marginw = std::max((windowSize.width - frameBufferSize_.width) / 2.0, 0.0); + double marginh = std::max((windowSize.height - frameBufferSize_.height) / 2.0, 0.0); + double marginws = std::max((windowSize.width - fbws) / 2.0, 0.0); + double marginhs = std::max((windowSize.height - fbhs) / 2.0, 0.0); GLint srcX0 = viewport.x; GLint srcY0 = viewport.y; GLint srcX1 = viewport.x + viewport.width; GLint srcY1 = viewport.y + viewport.height; - GLint dstX0 = stretch ? xn : windowSize.width - frameBufferSize_.width; - GLint dstY0 = stretch ? yn : windowSize.height - frameBufferSize_.height; - GLint dstX1 = stretch ? wn : frameBufferSize_.width; - GLint dstY1 = stretch ? hn : frameBufferSize_.height; + GLint dstX0 = scale ? marginws : marginw; + GLint dstY0 = scale ? marginhs : marginh; + GLint dstX1 = scale ? marginws + fbws : marginw + frameBufferSize_.width; + GLint dstY1 = scale ? marginhs + fbhs : marginh + frameBufferSize_.height; GL_CHECK(glBindFramebuffer(GL_DRAW_FRAMEBUFFER, drawFramebufferID)); GL_CHECK(glBlitFramebuffer( srcX0, srcY0, srcX1, srcY1, diff --git a/modules/v4d/src/util.cpp b/modules/v4d/src/util.cpp index a64ed1e35..22078ff91 100644 --- a/modules/v4d/src/util.cpp +++ b/modules/v4d/src/util.cpp @@ -46,6 +46,18 @@ size_t cnz(const cv::UMat& m) { return cv::countNonZero(grey); } } + +cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code) { + cv::Mat tmpIn(1, 1, CV_8UC3); + cv::Mat tmpOut(1, 1, CV_8UC3); + + tmpIn.at(0, 0) = cv::Vec3b(src[0], src[1], src[2]); + cvtColor(tmpIn, tmpOut, code); + const cv::Vec3b& vdst = tmpOut.at(0, 0); + cv::Scalar dst(vdst[0], vdst[1], vdst[2], src[3]); + return dst; +} + #ifdef __EMSCRIPTEN__ Mat read_embedded_image(const string &path) { SDL_Surface *loadedSurface = IMG_Load(path.c_str()); diff --git a/modules/v4d/src/v4d.cpp b/modules/v4d/src/v4d.cpp index f4472e76b..807772cf1 100644 --- a/modules/v4d/src/v4d.cpp +++ b/modules/v4d/src/v4d.cpp @@ -16,17 +16,6 @@ namespace cv { namespace v4d { -cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code) { - cv::Mat tmpIn(1, 1, CV_8UC3); - cv::Mat tmpOut(1, 1, CV_8UC3); - - tmpIn.at(0, 0) = cv::Vec3b(src[0], src[1], src[2]); - cvtColor(tmpIn, tmpOut, code); - const cv::Vec3b& vdst = tmpOut.at(0, 0); - cv::Scalar dst(vdst[0], vdst[1], vdst[2], src[3]); - return dst; -} - cv::Ptr V4D::make(const cv::Size& size, const cv::Size& fbsize, const string& title, bool offscreen, bool debug, bool compat, int samples) { cv::Ptr v4d = new V4D(size, fbsize, title, offscreen, debug, compat, samples); v4d->setVisible(!offscreen); @@ -36,7 +25,7 @@ cv::Ptr V4D::make(const cv::Size& size, const cv::Size& fbsize, const strin V4D::V4D(const cv::Size& size, const cv::Size& fbsize, const string& title, bool offscreen, bool debug, bool compat, int samples) : initialSize_(size), title_(title), compat_( compat), samples_(samples), debug_(debug), viewport_(0, 0, size.width, size.height), zoomScale_( - 1), mousePos_(0, 0), stretch_(true), pool_(2) { + 1), mousePos_(0, 0), scaling_(true), pool_(2) { #ifdef __EMSCRIPTEN__ printf(""); //makes sure we have FS as a dependency #endif @@ -460,11 +449,11 @@ void V4D::setVisible(bool v) { } void V4D::setScaling(bool s) { - stretch_ = s; + scaling_ = s; } bool V4D::isScaling() { - return stretch_; + return scaling_; } void V4D::setDefaultKeyboardEventCallback() {