diff --git a/modules/v4d/CMakeLists.txt b/modules/v4d/CMakeLists.txt
index d8e74d80e..edf176db6 100644
--- a/modules/v4d/CMakeLists.txt
+++ b/modules/v4d/CMakeLists.txt
@@ -49,7 +49,7 @@ macro(add_binary_sample sample)
endmacro()
if(EMSCRIPTEN)
- set(EM_LINKER_FLAGS "-sENVIRONMENT=web,worker -sOFFSCREENCANVAS_SUPPORT -sOFFSCREENCANVASES_TO_PTHREAD=#offscreenCanvas -sEXPORTED_FUNCTIONS=_malloc,_main,_v4dSetVideoFramePointer -sEXPORTED_RUNTIME_METHODS=ccall,setValue -sPROXY_TO_PTHREAD=1 --use-preload-plugins --preload-file doc/lena.png -sINITIAL_MEMORY=128MB -sALLOW_MEMORY_GROWTH=1 -sUSE_GLFW=3 -sMIN_WEBGL_VERSION=2 -sMAX_WEBGL_VERSION=2 --bind")
+ set(EM_LINKER_FLAGS "-sENVIRONMENT=web,worker -sOFFSCREENCANVAS_SUPPORT -sSTANDALONE_WASM -sOFFSCREENCANVASES_TO_PTHREAD=#offscreenCanvas -sEXPORTED_FUNCTIONS=_main,_v4dInitCapture -sEXPORTED_RUNTIME_METHODS=ccall -sPROXY_TO_PTHREAD=1 --use-preload-plugins --preload-file doc/lena.png -sINITIAL_MEMORY=128MB -sALLOW_MEMORY_GROWTH=1 -sUSE_GLFW=3 -sMIN_WEBGL_VERSION=2 -sMAX_WEBGL_VERSION=2 --bind")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${EM_LINKER_FLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${EM_LINKER_FLAGS}")
@@ -118,23 +118,23 @@ if(BUILD_EXAMPLES)
set(NANOGUI_BUILD_GLFW OFF)
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/third/nanogui/ext/glfw/include/")
- add_emscripten_sample(example_v4d_display_image samples/display_image.cpp)
- add_emscripten_sample(example_v4d_display_image_fb samples/display_image_fb.cpp)
- add_emscripten_sample(example_v4d_vector_graphics samples/vector_graphics.cpp)
- add_emscripten_sample(example_v4d_vector_graphics_and_fb samples/vector_graphics_and_fb.cpp)
- add_emscripten_sample(example_v4d_render_opengl samples/render_opengl.cpp)
- add_emscripten_sample(example_v4d_custom_source_and_sink samples/custom_source_and_sink.cpp)
- add_emscripten_sample(example_v4d_font_rendering samples/font_rendering.cpp)
- add_emscripten_sample(example_v4d_font_with_gui samples/font_with_gui.cpp)
- add_emscripten_sample(example_v4d_video_editing samples/video_editing.cpp)
- add_emscripten_sample(example_v4d_cube-demo samples/cube-demo.cpp)
+ # add_emscripten_sample(example_v4d_display_image samples/display_image.cpp)
+ # add_emscripten_sample(example_v4d_display_image_fb samples/display_image_fb.cpp)
+ # add_emscripten_sample(example_v4d_vector_graphics samples/vector_graphics.cpp)
+ # add_emscripten_sample(example_v4d_vector_graphics_and_fb samples/vector_graphics_and_fb.cpp)
+ # add_emscripten_sample(example_v4d_render_opengl samples/render_opengl.cpp)
+ # add_emscripten_sample(example_v4d_custom_source_and_sink samples/custom_source_and_sink.cpp)
+ # add_emscripten_sample(example_v4d_font_rendering samples/font_rendering.cpp)
+ # add_emscripten_sample(example_v4d_font_with_gui samples/font_with_gui.cpp)
+ # add_emscripten_sample(example_v4d_video_editing samples/video_editing.cpp)
+ # add_emscripten_sample(example_v4d_cube-demo samples/cube-demo.cpp)
add_emscripten_sample(example_v4d_video-demo samples/video-demo.cpp)
- add_emscripten_sample(example_v4d_nanovg-demo samples/nanovg-demo.cpp)
- add_emscripten_sample(example_v4d_font-demo samples/font-demo.cpp)
- add_emscripten_sample(example_v4d_shader-demo samples/shader-demo.cpp)
- add_emscripten_sample(example_v4d_pedestrian-demo samples/pedestrian-demo.cpp)
- add_emscripten_sample(example_v4d_optflow-demo samples/optflow-demo.cpp)
- add_emscripten_sample(example_v4d_beauty-demo samples/beauty-demo.cpp)
+ # add_emscripten_sample(example_v4d_nanovg-demo samples/nanovg-demo.cpp)
+ # add_emscripten_sample(example_v4d_font-demo samples/font-demo.cpp)
+ # add_emscripten_sample(example_v4d_shader-demo samples/shader-demo.cpp)
+ # add_emscripten_sample(example_v4d_pedestrian-demo samples/pedestrian-demo.cpp)
+ # add_emscripten_sample(example_v4d_optflow-demo samples/optflow-demo.cpp)
+ # add_emscripten_sample(example_v4d_beauty-demo samples/beauty-demo.cpp)
else()
add_binary_sample(example_v4d_display_image)
add_binary_sample(example_v4d_custom_source_and_sink)
diff --git a/modules/v4d/samples/example_v4d_beauty-demo.html b/modules/v4d/samples/example_v4d_beauty-demo.html
index c4f543b5c..fdb3ef08b 100644
--- a/modules/v4d/samples/example_v4d_beauty-demo.html
+++ b/modules/v4d/samples/example_v4d_beauty-demo.html
@@ -128,8 +128,8 @@
var statusElement = document.getElementById('status');
var progressElement = document.getElementById('progress');
var fsButton = document.querySelector("#fullscreenBtn");
- var cameraBtn = document.querySelector("#captureBtn");
- var videoElement = document.querySelector("#video");
+ var cameraBtn = document.querySelector("captureBtn");
+ var videoElement = document.querySelector("#v4dVideoElement");
var cameraCanvas = document.querySelector("#cameraCanvas");
function fixCanvasSize() {
diff --git a/modules/v4d/samples/example_v4d_video-demo.html b/modules/v4d/samples/example_v4d_video-demo.html
index 8ac1e3d92..84efeaa9d 100644
--- a/modules/v4d/samples/example_v4d_video-demo.html
+++ b/modules/v4d/samples/example_v4d_video-demo.html
@@ -110,7 +110,7 @@
-
+
Downloading...
@@ -128,8 +128,8 @@
var statusElement = document.getElementById('status');
var progressElement = document.getElementById('progress');
var fsButton = document.querySelector("#fullscreenBtn");
- var cameraBtn = document.querySelector("#captureBtn");
- var videoElement = document.querySelector("#video");
+ var captureBtn = document.querySelector("#captureBtn");
+ var videoElement = document.querySelector("#v4dVideoElement");
var cameraCanvas = document.querySelector("#cameraCanvas");
function fixCanvasSize() {
@@ -142,9 +142,7 @@
var Module = {
onRuntimeInitialized: function() {
fixCanvasSize();
- Module.videoBuffer = Module._malloc(1280 * 720 * 4);
- Module.cameraCtx = null;
- Module.ccall('v4dSetVideoFramePointer', 'void', ['number', 'number', 'number'], [Module.videoBuffer, 1280, 720]);
+ Module._v4dInitCapture(1280, 720);
},
preRun: [],
postRun: [],
@@ -207,26 +205,16 @@
};
Module.setStatus('Downloading...');
- window.onerror = function(event) {
- // TODO: do not warn on ok events like simulating an infinite loop or exitStatus
- Module.setStatus('Exception thrown, see JavaScript console');
- //spinnerElement.style.display = 'none';
- Module.setStatus = function(text) {
- if (text) Module.printErr('[post-exception status] ' + text);
- };
- };
-
var playing = false;
var timeupdate = false;
function checkReady() {
if (playing && timeupdate) {
- Module.doCapture = true;
+ globalThis.doCapture = true;
}
}
- cameraBtn.addEventListener('click', async function() {
- let stream = await navigator.mediaDevices.getUserMedia({ video: { width: 1280, height: 720 }, audio: false});
-
+ captureBtn.addEventListener('click', async function() {
+ let stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
videoElement.addEventListener(
"playing",
() => {
@@ -247,6 +235,15 @@
videoElement.srcObject = stream;
});
+ window.onerror = function(event) {
+ // TODO: do not warn on ok events like simulating an infinite loop or exitStatus
+ Module.setStatus('Exception thrown, see JavaScript console');
+ //spinnerElement.style.display = 'none';
+ Module.setStatus = function(text) {
+ if (text) Module.printErr('[post-exception status] ' + text);
+ };
+ };
+
fsButton.addEventListener('click', async function () {
Module.requestFullscreen(false, false)
});
diff --git a/modules/v4d/src/detail/framebuffercontext.cpp b/modules/v4d/src/detail/framebuffercontext.cpp
index 8462e86f8..29ba3b99d 100644
--- a/modules/v4d/src/detail/framebuffercontext.cpp
+++ b/modules/v4d/src/detail/framebuffercontext.cpp
@@ -78,7 +78,7 @@ void FrameBufferContext::init() {
glfwWindowHint(GLFW_STENCIL_BITS, 8);
glfwWindowHint(GLFW_DEPTH_BITS, 24);
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);
- glfwWindowHint(GLFW_VISIBLE, offscreen_ ? GLFW_FALSE : GLFW_TRUE);
+ glfwWindowHint(GLFW_VISIBLE, GLFW_TRUE);
glfwWindowHint(GLFW_DOUBLEBUFFER, GLFW_TRUE);
glfwWindow_ = glfwCreateWindow(frameBufferSize_.width, frameBufferSize_.height, title_.c_str(), nullptr,
diff --git a/modules/v4d/src/util.cpp b/modules/v4d/src/util.cpp
index 41047303f..093d537b0 100644
--- a/modules/v4d/src/util.cpp
+++ b/modules/v4d/src/util.cpp
@@ -312,120 +312,140 @@ Source makeCaptureSource(const string& inputFilename) {
#else
using namespace emscripten;
-uint8_t* current_frame = nullptr;
-extern "C" {
-
-EMSCRIPTEN_KEEPALIVE
-void v4dSetVideoFramePointer(uint8_t* frame, int width, int height) {
- assert(current_frame == nullptr);
- current_frame = frame;
-// memset(current_frame, 127, width * height * 4);
-}
-}
-
-GLuint framebuffer = 0;
-GLuint texture = 0;
-
-bool captureVideoFrameGPU(int width, int height) {
- int ret = EM_ASM_INT(
- if(typeof Module.ctx !== 'undefined' && Module.ctx !== null && Module.doCapture) {
- globalThis.gl = Module.ctx;
- globalThis.v4dMainFrameBuffer = globalThis.gl.getParameter(globalThis.gl.FRAMEBUFFER_BINDING);
- globalThis.v4dMainTexture = globalThis.gl.getFramebufferAttachmentParameter(globalThis.gl.FRAMEBUFFER, globalThis.gl.COLOR_ATTACHMENT0, globalThis.gl.FRAMEBUFFER_ATTACHMENT_OBJECT_NAME);
- return 1;
- } else {
- return 0;
- }
- );
+class HTML5Capture {
+private:
+ cv::Ptr window_;
+ int width_;
+ int height_;
+ UMat tmp_;
+ GLuint framebuffer = 0;
+ GLuint texture = 0;
+public:
+ HTML5Capture(cv::Ptr window, int width, int height) :
+ window_(window), width_(width), height_(height), tmp_(cv::Size(width, height), CV_8UC4) {
+ cerr << "start constr" << endl;
+ EM_ASM({
+ globalThis.playing = false;
+ globalThis.timeupdate = false;
+ globalThis.v4dVideoElement = document.querySelector("#v4dVideoElement");
+ globalThis.v4dCopyCanvasElement = document.createElement("canvas");
+ globalThis.v4dCopyCanvasElement.id = "v4dCopyCanvasElement0";
+ globalThis.v4dCopyCanvasElement.width = $0;
+ globalThis.v4dCopyCanvasElement.height = $1;
+ globalThis.v4dCopyCanvasElement.style.display = "none";
+ }, width, height);
+ cerr << "end constr" << endl;
+ }
- if(ret) {
- EM_ASM(
- if(typeof globalThis.v4dVideoElement === 'undefined' || globalThis.v4dVideoElement === null) {
- globalThis.v4dVideoElement = document.querySelector("#video");
+ bool captureGPU(UMat& dst) {
+ cerr << "start capture" << endl;
+ FrameBufferContext::GLScope scope(window_->fbCtx());
+ cerr << "start em" << endl;
+
+ int ret = EM_ASM_INT(
+ if(typeof Module.ctx !== 'undefined' && Module.ctx != null && globalThis.doCapture) {
+ globalThis.gl = Module.ctx;
+ globalThis.v4dMainFrameBuffer = globalThis.gl.getParameter(globalThis.gl.FRAMEBUFFER_BINDING);
+ globalThis.v4dMainTexture = globalThis.gl.getFramebufferAttachmentParameter(globalThis.gl.FRAMEBUFFER, globalThis.gl.COLOR_ATTACHMENT0, globalThis.gl.FRAMEBUFFER_ATTACHMENT_OBJECT_NAME);
+ return 1;
+ } else {
+ return 0;
}
);
+ cerr << "en em: " << ret << endl;
+ if(ret) {
+ cerr << "1" << endl;
+ if(framebuffer == 0) {
+ GL_CHECK(glGenFramebuffers(1, &framebuffer));
+ }
- if(framebuffer == 0) {
- GL_CHECK(glGenFramebuffers(1, &framebuffer));
- }
-
- GL_CHECK(glBindFramebuffer(GL_READ_FRAMEBUFFER, framebuffer));
-
- if(texture == 0) {
- GL_CHECK(glGenTextures(1, &texture));
- }
-
- GL_CHECK(glBindTexture(GL_TEXTURE_2D, texture));
+ GL_CHECK(glBindFramebuffer(GL_READ_FRAMEBUFFER, framebuffer));
+ cerr << "2" << endl;
+ if(texture == 0) {
+ GL_CHECK(glGenTextures(1, &texture));
+ }
- EM_ASM(
- const level = 0;
- const internalFormat = globalThis.gl.RGBA;
- const border = 0;
- const srcFormat = globalThis.gl.RGBA;
- const srcType = globalThis.gl.UNSIGNED_BYTE;
- globalThis.gl.texImage2D(
- globalThis.gl.TEXTURE_2D,
- level,
- internalFormat,
- srcFormat,
- srcType,
- globalThis.v4dVideoElement
+ GL_CHECK(glBindTexture(GL_TEXTURE_2D, texture));
+ cerr << "3" << endl;
+ EM_ASM(
+ const level = 0;
+ const internalFormat = globalThis.gl.RGBA;
+ const border = 0;
+ const srcFormat = globalThis.gl.RGBA;
+ const srcType = globalThis.gl.UNSIGNED_BYTE;
+ globalThis.gl.texImage2D(
+ globalThis.gl.TEXTURE_2D,
+ level,
+ internalFormat,
+ srcFormat,
+ srcType,
+ globalThis.v4dVideoElement
+ );
);
- );
+ cerr << "4" << endl;
+ GL_CHECK(glFramebufferTexture2D(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0));
+ EM_ASM(
+ globalThis.gl.bindFramebuffer(globalThis.gl.DRAW_FRAMEBUFFER, globalThis.v4dMainFrameBuffer);
+ globalThis.gl.bindTexture(globalThis.gl.TEXTURE_2D, globalThis.v4dMainTexture);
+ globalThis.gl.pixelStorei(globalThis.gl.UNPACK_FLIP_Y_WEBGL, true);
+ globalThis.gl.framebufferTexture2D(globalThis.gl.DRAW_FRAMEBUFFER, globalThis.gl.COLOR_ATTACHMENT0, globalThis.gl.TEXTURE_2D, globalThis.v4dMainTexture, 0);
+ );
+ cerr << "5" << endl;
+ FrameBufferContext::FrameBufferScope fbScope(window_->fbCtx(), tmp_);
+ cvtColor(tmp_, dst, COLOR_BGRA2RGB);
+ cerr << "captured" << endl;
+ return true;
+ }
+ cerr << "not captured" << endl;
+ return false;
+ }
- GL_CHECK(glFramebufferTexture2D(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0));
+ void captureCPU() {
EM_ASM(
- globalThis.gl.bindFramebuffer(globalThis.gl.DRAW_FRAMEBUFFER, globalThis.v4dMainFrameBuffer);
- globalThis.gl.bindTexture(globalThis.gl.TEXTURE_2D, globalThis.v4dMainTexture);
- globalThis.gl.pixelStorei(globalThis.gl.UNPACK_FLIP_Y_WEBGL, true);
- globalThis.gl.framebufferTexture2D(globalThis.gl.DRAW_FRAMEBUFFER, globalThis.gl.COLOR_ATTACHMENT0, globalThis.gl.TEXTURE_2D, globalThis.v4dMainTexture, 0);
+ if(globalThis.doCapture) {
+ if(typeof globalThis.v4dCopyCanvasContext === 'undefined' || globalThis.v4dCopyCanvasContext === null)
+ globalThis.v4dCopyCanvasContext = globalThis.v4dCopyCanvasElement.getContext('2d', { willReadFrequently: true });
+ if(typeof globalThis.v4dFrameData === 'undefined' || globalThis.v4dFrameData === null)
+ globalThis.v4dFrameData = Module._malloc(width_ * height_ * 4);
+
+ globalThis.v4dCopyCanvasElement.drawImage(globalThis.v4dVideoElement, 0, 0, 1280, 720);
+ var cameraArrayBuffer = globalThis.v4dCopyCanvasContext.getImageData(0, 0, 1280, 720);
+ Module.HEAPU8.set(cameraArrayBuffer.data, globalThis.v4dFrameData);
+ }
);
- return true;
}
- return false;
-}
+};
-EM_JS(void,copyVideoFrameCPU,(int p), {
- if(Module.doCapture) {
- if(typeof Module.cameraCtx === 'undefined' || Module.cameraCtx === null)
- Module.cameraCtx = document.querySelector("#cameraCanvas").getContext('2d', { willReadFrequently: true });
- if(typeof Module.videoElement === 'undefined' || Module.videoElement === null)
- Module.videoElement = document.querySelector("#video");
+cv::Ptr capture = nullptr;
+int capture_width = 0;
+int capture_height = 0;
- Module.cameraCtx.drawImage(Module.videoElement, 0, 0, 1280, 720);
- var cameraArrayBuffer = Module.cameraCtx.getImageData(0, 0, 1280, 720);
+extern "C" {
- Module.HEAPU8.set(cameraArrayBuffer.data, p);
- }
-});
+EMSCRIPTEN_KEEPALIVE
+void v4dInitCapture(int width, int height) {
+ capture_width = width;
+ capture_height = height;
+}
+
+}
Source makeCaptureSource(int width, int height, cv::Ptr window) {
using namespace std;
return Source([=](cv::UMat& frame) {
- //FIXME
- static cv::UMat tmp(cv::Size(width, height), CV_8UC4);
+ if(capture == nullptr && capture_width > 0 && capture_height > 0)
+// run_sync_on_main<16>([&](){
+// capture = new HTML5Capture(window, capture_width, capture_height);
+// });
try {
if(frame.empty())
frame.create(cv::Size(width, height), CV_8UC3);
- if (current_frame != nullptr) {
- run_sync_on_main<17>([&](){
- FrameBufferContext::GLScope scope(window->fbCtx());
- if(captureVideoFrameGPU(width, height)) {
- FrameBufferContext::FrameBufferScope fbScope(window->fbCtx(), tmp);
- cvtColor(tmp, frame, COLOR_BGRA2RGB);
- }
- });
-
-// run_sync_on_main<16>([&](){
-// copyVideoFrameCPU(reinterpret_cast(current_frame));
-// cv::Mat tmp(cv::Size(width, height), CV_8UC4, current_frame);
-// cv::UMat utmp = tmp.getUMat(ACCESS_READ);
-// cvtColor(utmp, frame, cv::COLOR_BGRA2RGB);
-// utmp.release();
-// tmp.release();
+ if(capture != nullptr) {
+// run_sync_on_main<17>([&](){
+// capture->captureGPU(frame);
// });
} else {
std::cerr << "Nothing captured" << endl;