various fixes

modules/v4d/include/opencv2/v4d/util.hpp

@@ -13,6 +13,8 @@
 #include <string>
 #include <iostream>
 #include <opencv2/core/ocl.hpp>
+#include <opencv2/imgproc.hpp>
+
 #ifdef __EMSCRIPTEN__
 #  include <emscripten.h>
 #  include <emscripten/bind.h>
@@ -89,6 +91,15 @@ CV_EXPORTS size_t cnz(const cv::UMat& m);
 }
 using std::string;
 class V4D;
+
+/*!
+ * Convenience function to color convert from Scalar to Scalar
+ * @param src The scalar to color convert
+ * @param code The color converions code
+ * @return The color converted scalar
+ */
+CV_EXPORTS cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code);
+
 #ifdef __EMSCRIPTEN__
 CV_EXPORTS Mat read_embedded_image(const string &path);
 #endif

modules/v4d/include/opencv2/v4d/v4d.hpp

@@ -84,14 +84,6 @@ template<typename T> void find_widgets(const nanogui::Widget* parent, std::vecto
 }
 }
 
-/*!
- * Convenience function to color convert from Scalar to Scalar
- * @param src The scalar to color convert
- * @param code The color converions code
- * @return The color converted scalar
- */
-CV_EXPORTS cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code);
-
 using namespace cv::v4d::detail;
 
 class CV_EXPORTS V4D {
@@ -105,7 +97,7 @@ class CV_EXPORTS V4D {
     cv::Rect viewport_;
     float zoomScale_;
     cv::Vec2f mousePos_;
-    bool stretch_;
+    bool scaling_;
     FrameBufferContext* mainFbContext_ = nullptr;
     CLVAContext* clvaContext_ = nullptr;
     GLContext* glContext_ = nullptr;

modules/v4d/samples/beauty-demo.cpp

@@ -200,13 +200,14 @@ static void adjust_saturation(const cv::UMat &srcBGR, cv::UMat &dstBGR, float fa
     cvtColor(hls, dstBGR, cv::COLOR_HLS2BGR);
 }
 
+//Built the GUI
 static void setup_gui(cv::Ptr<cv::v4d::V4D> v) {
     v->nanogui([&](cv::v4d::FormHelper& form){
         form.makeDialog(5, 30, "Effect");
 
         form.makeGroup("Display");
         form.makeFormVariable("Side by side", side_by_side, "Enable or disable side by side view");
-        auto* scaleVar = form.makeFormVariable("Stretch", scale, "Enable or disable stetching to the window size");
+        auto* scaleVar = form.makeFormVariable("Scale", scale, "Enable or disable scaling to the window size");
         scaleVar->set_callback([=](const bool& b) {
             v->setScaling(b);
             scale = b;
@@ -277,11 +278,12 @@ static bool iteration() {
         if (!window->capture())
             return false;
 
+        //Save the video frame as BGR
         window->fb([&](cv::UMat &frameBuffer) {
             cvtColor(frameBuffer, input, cv::COLOR_BGRA2BGR);
         });
 
-        //Downscale the input for face detection
+        //Downscale the video frame for face detection
         cv::resize(input, down, cv::Size(DOWNSIZE_WIDTH, DOWNSIZE_HEIGHT));
 
         shapes.clear();
@@ -349,6 +351,7 @@ static bool iteration() {
             frameOutFloat.convertTo(frameOut, CV_8U, 1.0);
 
             if (side_by_side) {
+                //create side-by-side view with a result
                 cv::resize(input, lhalf, cv::Size(0, 0), 0.5, 0.5);
                 cv::resize(frameOut, rhalf, cv::Size(0, 0), 0.5, 0.5);
 
@@ -356,12 +359,9 @@ static bool iteration() {
                 lhalf.copyTo(frameOut(cv::Rect(0, 0, lhalf.size().width, lhalf.size().height)));
                 rhalf.copyTo(frameOut(cv::Rect(rhalf.size().width, 0, rhalf.size().width, rhalf.size().height)));
             }
-
-            window->fb([&](cv::UMat &frameBuffer) {
-                cvtColor(frameOut, frameBuffer, cv::COLOR_BGR2BGRA);
-            });
         } else {
             if (side_by_side) {
+                //create side-by-side view without a result (using the input image for both sides)
                 frameOut = cv::Scalar::all(0);
                 cv::resize(input, lhalf, cv::Size(0, 0), 0.5, 0.5);
                 lhalf.copyTo(frameOut(cv::Rect(0, 0, lhalf.size().width, lhalf.size().height)));
@@ -369,12 +369,13 @@ static bool iteration() {
             } else {
                 input.copyTo(frameOut);
             }
-
-            window->fb([&](cv::UMat &frameBuffer) {
-                cvtColor(frameOut, frameBuffer, cv::COLOR_BGR2BGRA);
-            });
         }
 
+        //write the result to the framebuffer
+        window->fb([&](cv::UMat &frameBuffer) {
+            cvtColor(frameOut, frameBuffer, cv::COLOR_BGR2BGRA);
+        });
+
         window->write();
 
         return window->display();

modules/v4d/samples/optflow-demo.cpp

@@ -41,10 +41,13 @@ constexpr bool OFFSCREEN = false;
 
 cv::Ptr<cv::v4d::V4D> window;
 #ifndef __EMSCRIPTEN__
+//create a separate window in native builds. In WASM builds instead create a light-weight dialog
 cv::Ptr<cv::v4d::V4D> menuWindow;
 #endif
 
 /* Visualization parameters */
+
+//How the background will be visualized
 enum BackgroundModes {
     GREY,
     COLOR,
@@ -52,6 +55,7 @@ enum BackgroundModes {
     BLACK
 };
 
+//Post-processing modes for the foreground
 enum PostProcModes {
     GLOW,
     BLOOM,
@@ -59,12 +63,8 @@ enum PostProcModes {
 };
 
 // Generate the foreground at this scale.
-#ifndef __EMSCRIPTEN__
-float fg_scale = 0.5f;
-#else
 float fg_scale = 0.5f;
-#endif
-// On every frame the foreground loses on brightness. specifies the loss in percent.
+// On every frame the foreground loses on brightness. Specifies the loss in percent.
 #ifndef __EMSCRIPTEN__
 float fg_loss = 2.5;
 #else
@@ -116,15 +116,18 @@ int bloom_thresh = 210;
 //The intensity of the bloom filter
 float bloom_gain = 3;
 
+//Uses background subtraction to generate a "motion mask"
 static void prepare_motion_mask(const cv::UMat& srcGrey, cv::UMat& motionMaskGrey) {
     static cv::Ptr<cv::BackgroundSubtractor> bg_subtrator = cv::createBackgroundSubtractorMOG2(100, 16.0, false);
     static int morph_size = 1;
     static cv::Mat element = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(2 * morph_size + 1, 2 * morph_size + 1), cv::Point(morph_size, morph_size));
 
     bg_subtrator->apply(srcGrey, motionMaskGrey);
+    //Surpress speckles
     cv::morphologyEx(motionMaskGrey, motionMaskGrey, cv::MORPH_OPEN, element, cv::Point(element.cols >> 1, element.rows >> 1), 2, cv::BORDER_CONSTANT, cv::morphologyDefaultBorderValue());
 }
 
+//Detect points to track
 static void detect_points(const cv::UMat& srcMotionMaskGrey, vector<cv::Point2f>& points) {
     static cv::Ptr<cv::FastFeatureDetector> detector = cv::FastFeatureDetector::create(1, false);
     static vector<cv::KeyPoint> tmpKeyPoints;
@@ -138,6 +141,7 @@ static void detect_points(const cv::UMat& srcMotionMaskGrey, vector<cv::Point2f>
     }
 }
 
+//Detect extrem changes in scene content and report it
 static bool detect_scene_change(const cv::UMat& srcMotionMaskGrey, const float thresh, const float theshDiff) {
     static float last_movement = 0;
 
@@ -152,6 +156,7 @@ static bool detect_scene_change(const cv::UMat& srcMotionMaskGrey, const float t
     return result;
 }
 
+//Visualize the sparse optical flow
 static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UMat &nextGrey, const vector<cv::Point2f> &detectedPoints, const float scaleFactor, const int maxStrokeSize, const cv::Scalar color, const int maxPoints, const float pointLossPercent) {
     static vector<cv::Point2f> hull, prevPoints, nextPoints, newPoints;
     static vector<cv::Point2f> upPrevPoints, upNextPoints;
@@ -160,25 +165,33 @@ static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UM
     static std::random_device rd;
     static std::mt19937 g(rd());
 
+    //less then 5 points is a degenerate case (e.g. the corners of a video frame)
     if (detectedPoints.size() > 4) {
         cv::convexHull(detectedPoints, hull);
         float area = cv::contourArea(hull);
+        //make sure the area of the point cloud is positive
         if (area > 0) {
             float density = (detectedPoints.size() / area);
+            //stroke size is biased by the area of the point cloud
             float strokeSize = maxStrokeSize * pow(area / (nextGrey.cols * nextGrey.rows), 0.33f);
+            //max points is biased by the densitiy of the point cloud
             size_t currentMaxPoints = ceil(density * maxPoints);
 
+            //lose a number of random points specified by pointLossPercent
             std::shuffle(prevPoints.begin(), prevPoints.end(), g);
             prevPoints.resize(ceil(prevPoints.size() * (1.0f - (pointLossPercent / 100.0f))));
 
+            //calculate how many newly detected points to add
             size_t copyn = std::min(detectedPoints.size(), (size_t(std::ceil(currentMaxPoints)) - prevPoints.size()));
             if (prevPoints.size() < currentMaxPoints) {
                 std::copy(detectedPoints.begin(), detectedPoints.begin() + copyn, std::back_inserter(prevPoints));
             }
 
+            //calculate the sparse optical flow
             cv::calcOpticalFlowPyrLK(prevGrey, nextGrey, prevPoints, nextPoints, status, err);
             newPoints.clear();
             if (prevPoints.size() > 1 && nextPoints.size() > 1) {
+                //scale the points to original size
                 upNextPoints.clear();
                 upPrevPoints.clear();
                 for (cv::Point2f pt : prevPoints) {
@@ -190,20 +203,29 @@ static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UM
                 }
 
                 using namespace cv::v4d::nvg;
+                //start drawing
                 beginPath();
                 strokeWidth(strokeSize);
                 strokeColor(color);
 
                 for (size_t i = 0; i < prevPoints.size(); i++) {
-                    if (status[i] == 1 && err[i] < (1.0 / density) && upNextPoints[i].y >= 0 && upNextPoints[i].x >= 0 && upNextPoints[i].y < nextGrey.rows / scaleFactor && upNextPoints[i].x < nextGrey.cols / scaleFactor) {
+                    if (status[i] == 1 //point was found in prev and new set
+                            && err[i] < (1.0 / density) //with a higher density be more sensitive to the feature error
+                            && upNextPoints[i].y >= 0 && upNextPoints[i].x >= 0 //check bounds
+                            && upNextPoints[i].y < nextGrey.rows / scaleFactor && upNextPoints[i].x < nextGrey.cols / scaleFactor //check bounds
+                            ) {
                         float len = hypot(fabs(upPrevPoints[i].x - upNextPoints[i].x), fabs(upPrevPoints[i].y - upNextPoints[i].y));
+                        //upper and lower bound of the flow vector lengthss
                         if (len > 0 && len < sqrt(area)) {
+                            //collect new points
                             newPoints.push_back(nextPoints[i]);
+                            //the actual drawing operations
                             moveTo(upNextPoints[i].x, upNextPoints[i].y);
                             lineTo(upPrevPoints[i].x, upPrevPoints[i].y);
                         }
                     }
                 }
+                //end drawing
                 stroke();
             }
             prevPoints = newPoints;
@@ -211,6 +233,7 @@ static void visualize_sparse_optical_flow(const cv::UMat &prevGrey, const cv::UM
     }
 }
 
+//Bloom post-processing effect
 static void bloom(const cv::UMat& src, cv::UMat &dst, int ksize = 3, int threshValue = 235, float gain = 4) {
     static cv::UMat bgr;
     static cv::UMat hls;
@@ -219,21 +242,29 @@ static void bloom(const cv::UMat& src, cv::UMat &dst, int ksize = 3, int threshV
     static cv::UMat blur;
     static std::vector<cv::UMat> hlsChannels;
 
+    //remove alpha channel
     cv::cvtColor(src, bgr, cv::COLOR_BGRA2RGB);
+    //convert to hls
     cv::cvtColor(bgr, hls, cv::COLOR_BGR2HLS);
+    //split channels
     cv::split(hls, hlsChannels);
+    //invert lightness
     cv::bitwise_not(hlsChannels[2], hlsChannels[2]);
-
+    //multiply lightness and saturation
     cv::multiply(hlsChannels[1], hlsChannels[2], ls16, 1, CV_16U);
+    //normalize
     cv::divide(ls16, cv::Scalar(255.0), ls, 1, CV_8U);
+    //binary threhold according to threshValue
     cv::threshold(ls, blur, threshValue, 255, cv::THRESH_BINARY);
-
+    //blur
     cv::boxFilter(blur, blur, -1, cv::Size(ksize, ksize), cv::Point(-1,-1), true, cv::BORDER_REPLICATE);
+    //convert to BGRA
     cv::cvtColor(blur, blur, cv::COLOR_GRAY2BGRA);
-
+    //add src and the blurred L-S-product according to gain
     addWeighted(src, 1.0, blur, gain, 0, dst);
 }
 
+//Glow post-processing effect
 static void glow_effect(const cv::UMat &src, cv::UMat &dst, const int ksize) {
     static cv::UMat resize;
     static cv::UMat blur;
@@ -256,15 +287,19 @@ static void glow_effect(const cv::UMat &src, cv::UMat &dst, const int ksize) {
     cv::bitwise_not(dst, dst);
 }
 
+//Compose the different layers into the final image
 static void composite_layers(cv::UMat& background, const cv::UMat& foreground, const cv::UMat& frameBuffer, cv::UMat& dst, int kernelSize, float fgLossPercent, BackgroundModes bgMode, PostProcModes ppMode) {
     static cv::UMat tmp;
     static cv::UMat post;
     static cv::UMat backgroundGrey;
     static vector<cv::UMat> channels;
 
+    //Lose a bit of foreground brightness based on fgLossPercent
     cv::subtract(foreground, cv::Scalar::all(255.0f * (fgLossPercent / 100.0f)), foreground);
+    //Add foreground an the current framebuffer into foregound
     cv::add(foreground, frameBuffer, foreground);
 
+    //Dependin on bgMode prepare the background in different ways
     switch (bgMode) {
     case GREY:
         cv::cvtColor(background, backgroundGrey, cv::COLOR_BGRA2GRAY);
@@ -285,6 +320,7 @@ static void composite_layers(cv::UMat& background, const cv::UMat& foreground, c
         break;
     }
 
+    //Depending on ppMode perform post-processing
     switch (ppMode) {
     case GLOW:
         glow_effect(foreground, post, kernelSize);
@@ -299,9 +335,11 @@ static void composite_layers(cv::UMat& background, const cv::UMat& foreground, c
         break;
     }
 
+    //Add background and post-processed foreground into dst
     cv::add(background, post, dst);
 }
 
+//Build the GUI
 static void setup_gui(cv::Ptr<cv::v4d::V4D> main, cv::Ptr<cv::v4d::V4D> menu) {
     main->nanogui([&](cv::v4d::FormHelper& form){
         form.makeDialog(5, 30, "Effects");
@@ -379,7 +417,7 @@ static void setup_gui(cv::Ptr<cv::v4d::V4D> main, cv::Ptr<cv::v4d::V4D> menu) {
 
         form.makeGroup("Display");
         form.makeFormVariable("Show FPS", show_fps, "Enable or disable the On-screen FPS display");
-        form.makeFormVariable("Stetch", scale, "Stretch the frame buffer to the window size")->set_callback([=](const bool &s) {
+        form.makeFormVariable("Scale", scale, "Scale the frame buffer to the window size")->set_callback([=](const bool &s) {
             main->setScaling(s);
         });
 
@@ -409,7 +447,9 @@ static bool iteration() {
         return false;
 
     window->fb([&](cv::UMat& frameBuffer) {
+        //resize to foreground scale
         cv::resize(frameBuffer, down, cv::Size(window->framebufferSize().width * fg_scale, window->framebufferSize().height * fg_scale));
+        //save video background
         frameBuffer.copyTo(background);
     });
 

modules/v4d/src/detail/framebuffercontext.cpp

@@ -663,26 +663,33 @@ CLExecContext_t& FrameBufferContext::getCLExecContext() {
 #endif
 
 void FrameBufferContext::blitFrameBufferToScreen(const cv::Rect& viewport,
-        const cv::Size& windowSize, bool stretch, GLuint drawFramebufferID) {
+        const cv::Size& windowSize, bool scale, GLuint drawFramebufferID) {
     this->makeCurrent();
 
     double hf = double(windowSize.height) / frameBufferSize_.height;
     double wf = double(windowSize.width) / frameBufferSize_.width;
-    double f = std::min(hf, wf);
+    double f;
+    if (frameBufferSize_.width > frameBufferSize_.height)
+        f = wf;
+    else
+        f = hf;
+
+    double fbws = frameBufferSize_.width * f;
+    double fbhs = frameBufferSize_.height * f;
 
-    double wn = frameBufferSize_.width * f;
-    double hn = frameBufferSize_.height * f;
-    double xn = windowSize.width - wn;
-    double yn = windowSize.height - hn;
+    double marginw = std::max((windowSize.width - frameBufferSize_.width) / 2.0, 0.0);
+    double marginh = std::max((windowSize.height - frameBufferSize_.height) / 2.0, 0.0);
+    double marginws = std::max((windowSize.width - fbws) / 2.0, 0.0);
+    double marginhs = std::max((windowSize.height - fbhs) / 2.0, 0.0);
 
     GLint srcX0 = viewport.x;
     GLint srcY0 = viewport.y;
     GLint srcX1 = viewport.x + viewport.width;
     GLint srcY1 = viewport.y + viewport.height;
-    GLint dstX0 = stretch ? xn : windowSize.width - frameBufferSize_.width;
-    GLint dstY0 = stretch ? yn : windowSize.height - frameBufferSize_.height;
-    GLint dstX1 = stretch ? wn : frameBufferSize_.width;
-    GLint dstY1 = stretch ? hn : frameBufferSize_.height;
+    GLint dstX0 = scale ? marginws : marginw;
+    GLint dstY0 = scale ? marginhs : marginh;
+    GLint dstX1 = scale ? marginws + fbws : marginw + frameBufferSize_.width;
+    GLint dstY1 = scale ? marginhs + fbhs : marginh + frameBufferSize_.height;
 
     GL_CHECK(glBindFramebuffer(GL_DRAW_FRAMEBUFFER, drawFramebufferID));
     GL_CHECK(glBlitFramebuffer( srcX0, srcY0, srcX1, srcY1,

modules/v4d/src/util.cpp

@@ -46,6 +46,18 @@ size_t cnz(const cv::UMat& m) {
     return cv::countNonZero(grey);
 }
 }
+
+cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code) {
+    cv::Mat tmpIn(1, 1, CV_8UC3);
+    cv::Mat tmpOut(1, 1, CV_8UC3);
+
+    tmpIn.at<cv::Vec3b>(0, 0) = cv::Vec3b(src[0], src[1], src[2]);
+    cvtColor(tmpIn, tmpOut, code);
+    const cv::Vec3b& vdst = tmpOut.at<cv::Vec3b>(0, 0);
+    cv::Scalar dst(vdst[0], vdst[1], vdst[2], src[3]);
+    return dst;
+}
+
 #ifdef __EMSCRIPTEN__
 Mat read_embedded_image(const string &path) {
     SDL_Surface *loadedSurface = IMG_Load(path.c_str());

modules/v4d/src/v4d.cpp

@@ -16,17 +16,6 @@
 namespace cv {
 namespace v4d {
 
-cv::Scalar colorConvert(const cv::Scalar& src, cv::ColorConversionCodes code) {
-    cv::Mat tmpIn(1, 1, CV_8UC3);
-    cv::Mat tmpOut(1, 1, CV_8UC3);
-
-    tmpIn.at<cv::Vec3b>(0, 0) = cv::Vec3b(src[0], src[1], src[2]);
-    cvtColor(tmpIn, tmpOut, code);
-    const cv::Vec3b& vdst = tmpOut.at<cv::Vec3b>(0, 0);
-    cv::Scalar dst(vdst[0], vdst[1], vdst[2], src[3]);
-    return dst;
-}
-
 cv::Ptr<V4D> V4D::make(const cv::Size& size, const cv::Size& fbsize, const string& title, bool offscreen, bool debug, bool compat, int samples) {
     cv::Ptr<V4D> v4d = new V4D(size, fbsize, title, offscreen, debug, compat, samples);
     v4d->setVisible(!offscreen);
@@ -36,7 +25,7 @@ cv::Ptr<V4D> V4D::make(const cv::Size& size, const cv::Size& fbsize, const strin
 V4D::V4D(const cv::Size& size, const cv::Size& fbsize, const string& title, bool offscreen, bool debug, bool compat, int samples) :
         initialSize_(size), title_(title), compat_(
                 compat), samples_(samples), debug_(debug), viewport_(0, 0, size.width, size.height), zoomScale_(
-                1), mousePos_(0, 0), stretch_(true), pool_(2) {
+                1), mousePos_(0, 0), scaling_(true), pool_(2) {
 #ifdef __EMSCRIPTEN__
     printf(""); //makes sure we have FS as a dependency
 #endif
@@ -460,11 +449,11 @@ void V4D::setVisible(bool v) {
 }
 
 void V4D::setScaling(bool s) {
-    stretch_ = s;
+    scaling_ = s;
 }
 
 bool V4D::isScaling() {
-    return stretch_;
+    return scaling_;
 }
 
 void V4D::setDefaultKeyboardEventCallback() {