diff --git a/.gitignore b/.gitignore index d66c2014c..ee7336110 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ src/*/*.o src/*/*.dep src/*/*.d src/tetra/tetra-demo -src/camera/camera-demo -camera-demo.mkv -tetra-demo.mkv +src/video/video-demo +src/nanovg/nanovg-demo +src/optflow/optflow-demo diff --git a/Makefile b/Makefile index be5cd9788..095151245 100644 --- a/Makefile +++ b/Makefile @@ -39,22 +39,22 @@ dirs: ${MAKE} -C src/video/ ${MAKEFLAGS} CXX=${CXX} ${MAKECMDGOALS} ${MAKE} -C src/nanovg/ ${MAKEFLAGS} CXX=${CXX} ${MAKECMDGOALS} ${MAKE} -C src/optflow/ ${MAKEFLAGS} CXX=${CXX} ${MAKECMDGOALS} - ${MAKE} -C src/optflow_mt/ ${MAKEFLAGS} CXX=${CXX} ${MAKECMDGOALS} - + ${MAKE} -C src/pedestrian/ ${MAKEFLAGS} CXX=${CXX} ${MAKECMDGOALS} + debian-release: ${MAKE} -C src/tetra/ ${MAKEFLAGS} CXX=${CXX} release ${MAKE} -C src/video/ ${MAKEFLAGS} CXX=${CXX} release ${MAKE} -C src/nanovg/ ${MAKEFLAGS} CXX=${CXX} release ${MAKE} -C src/optflow/ ${MAKEFLAGS} CXX=${CXX} release - ${MAKE} -C src/optflow_mt/ ${MAKEFLAGS} CXX=${CXX} release - + ${MAKE} -C src/pedestrian/ ${MAKEFLAGS} CXX=${CXX} release + debian-clean: ${MAKE} -C src/tetra/ ${MAKEFLAGS} CXX=${CXX} clean ${MAKE} -C src/video/ ${MAKEFLAGS} CXX=${CXX} clean ${MAKE} -C src/nanovg/ ${MAKEFLAGS} CXX=${CXX} clean ${MAKE} -C src/optflow/ ${MAKEFLAGS} CXX=${CXX} clean - ${MAKE} -C src/optflow_mt/ ${MAKEFLAGS} CXX=${CXX} clean - + ${MAKE} -C src/pedestrian/ ${MAKEFLAGS} CXX=${CXX} clean + install: ${TARGET} true diff --git a/README.md b/README.md index f9b549cc5..eb6a42477 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ # GCV -OpenGL/OpenCL/VAAPI interop demos (aka. run it on the GPU!) using the 4.x branch of OpenCV (https://github.com/opencv/opencv/tree/4.x) +OpenGL/OpenCL/VAAPI interop demos (aka. run it on the GPU!) using my 4.x branch of OpenCV (https://github.com/kallaballa/opencv/tree/GCV) The goal of the demos is to show how to use OpenCL interop in conjunction with OpenCV on Linux to create programs that run mostly (the part the matters) on the GPU. The author of the example video (which is also used for the demos videos in this README) is **(c) copyright Blender Foundation | www.bigbuckbunny.org**. # Hardware requirements +* Support for OpenCL 1.2 * Support for cl_khr_gl_sharing and cl_intel_va_api_media_sharing OpenCL extensions. -* Support for OpenCL 2.0 * If you are on a recent Intel Platform (Gen8 - Gen12) you probably need to install an alternative [compute-runtime](https://github.com/kallaballa/compute-runtime) -There are currently five demos (**the preview videos are scaled down and highly compressed**): +There are currently four demos (**the preview videos are scaled down and highly compressed**): ## tetra-demo Renders a rainbow tetrahedron on blue background using OpenGL and encodes to VP9 @@ -26,8 +26,8 @@ Renders a color wheel on top of a input-video using nanovg (OpenGL) and encodes https://user-images.githubusercontent.com/287266/200169216-1ff25db5-f5e0-49d1-92ba-ab7903168754.mp4 -## optflow-demo and optflow_mt-demo -My take on a optical flow visualization on top of a video. Encoded to VP9. "optflow_mt" is an example on how to use threads to maximize use of both GPU and CPU. +## optflow-demo +My take on a optical flow visualization on top of a video. Encoded to VP9. https://user-images.githubusercontent.com/287266/200512662-8251cf2c-23b3-4376-b664-d3a85b42d187.mp4 @@ -37,7 +37,7 @@ You need to build the most recent 4.x branch of OpenCV. ## Build OpenCV ```bash -git clone --branch 4.x https://github.com/opencv/opencv.git +git clone --branch GCV https://github.com/kallaballa/opencv.git cd opencv mkdir build cd build @@ -81,9 +81,4 @@ src/nanovg/nanovg-demo bunny.webm src/optflow/optflow-demo bunny.webm ``` -## Run the optflow_mt-demo: - -```bash -src/optflow_mt/optflow-demo_mt bunny.webm -``` diff --git a/src/optflow_mt/Makefile b/src/optflow_mt/Makefile deleted file mode 100644 index e0f829706..000000000 --- a/src/optflow_mt/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -TARGET := optflow_mt-demo - -SRCS := optflow_mt-demo.cpp - -#precompiled headers -HEADERS := -OBJS := ${SRCS:.cpp=.o} -DEPS := ${SRCS:.cpp=.dep} - -CXXFLAGS += -fpic `pkg-config --cflags egl opencv4 libva-drm glew` -LDFLAGS += -LIBS += -lm `pkg-config --libs egl opencv4 libva-drm glew` -lnanovg -lopencv_optflow -lOpenCL -.PHONY: all release debug clean distclean - -all: release -release: ${TARGET} -debug: ${TARGET} -info: ${TARGET} -profile: ${TARGET} -unsafe: ${TARGET} -asan: ${TARGET} - -${TARGET}: ${OBJS} - ${CXX} ${LDFLAGS} -o $@ $^ ${LIBS} - -${OBJS}: %.o: %.cpp %.dep ${GCH} - ${CXX} ${CXXFLAGS} -o $@ -c $< - -${DEPS}: %.dep: %.cpp Makefile - ${CXX} ${CXXFLAGS} -MM $< > $@ - -${GCH}: %.gch: ${HEADERS} - ${CXX} ${CXXFLAGS} -o $@ -c ${@:.gch=.hpp} - -install: - mkdir -p ${DESTDIR}/${PREFIX} - cp ${TARGET} ${DESTDIR}/${PREFIX} - -uninstall: - rm ${DESTDIR}/${PREFIX}/${TARGET} - -clean: - rm -f *~ ${DEPS} ${OBJS} ${CUO} ${GCH} ${TARGET} - -distclean: clean - diff --git a/src/optflow_mt/optflow_mt-demo.cpp b/src/optflow_mt/optflow_mt-demo.cpp deleted file mode 100644 index 659f98d25..000000000 --- a/src/optflow_mt/optflow_mt-demo.cpp +++ /dev/null @@ -1,198 +0,0 @@ -#define CL_TARGET_OPENCL_VERSION 120 - -constexpr unsigned long WIDTH = 1920; -constexpr unsigned long HEIGHT = 1080; -constexpr bool OFFSCREEN = false; -constexpr const int VA_HW_DEVICE_INDEX = 0; - -#include "../common/subsystems.hpp" -#include "../common/tsafe_queue.hpp" -#include -#include -#include -#include -#include -#include - -#include - -using std::cerr; -using std::endl; -using std::vector; -using std::string; - -SafeQueue, cv::UMat>> queue; - -static bool done = false; -static void finish(int ignore) { - std::cerr << endl; - done = true; -} - -int main(int argc, char **argv) { - signal(SIGINT, finish); - using namespace kb; - - if (argc != 2) { - std::cerr << "Usage: optflow " << endl; - exit(1); - } - - va::init(); - cv::VideoCapture cap(argv[1], cv::CAP_FFMPEG, { cv::CAP_PROP_HW_DEVICE, VA_HW_DEVICE_INDEX, cv::CAP_PROP_HW_ACCELERATION, cv::VIDEO_ACCELERATION_VAAPI, cv::CAP_PROP_HW_ACCELERATION_USE_OPENCL, 1 }); - if (!cap.isOpened()) { - cerr << "ERROR! Unable to open camera" << endl; - return -1; - } - - double fps = cap.get(cv::CAP_PROP_FPS); - - cv::VideoWriter encoder("optflow.mkv", cv::CAP_FFMPEG, cv::VideoWriter::fourcc('V', 'P', '9', '0'), fps, cv::Size(WIDTH, HEIGHT), { cv::VIDEOWRITER_PROP_HW_ACCELERATION, cv::VIDEO_ACCELERATION_VAAPI, cv::VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL, 1 }); - - if (!OFFSCREEN) - x11::init(); - egl::init(); - gl::init(); - nvg::init(); - - cerr << "VA Version: " << va::get_info() << endl; - cerr << "EGL Version: " << egl::get_info() << endl; - cerr << "OpenGL Version: " << gl::get_info() << endl; - cerr << "OpenCL Platforms: " << endl << cl::get_info() << endl; - - std::thread producer([&](){ - cv::Ptr bgSubtractor = cv::createBackgroundSubtractorMOG2(100, 32.0, false); - vector fgPoints1; - vector > contours; - vector hierarchy; - cv::UMat videoFrame1, foregroundMaskGrey; - while (!done) { - va::bind(); - cap >> videoFrame1; - if (videoFrame1.empty()) - break; - - cv::resize(videoFrame1, videoFrame1, cv::Size(WIDTH, HEIGHT)); - - bgSubtractor->apply(videoFrame1, foregroundMaskGrey); - - int morph_size = 1; - cv::Mat element = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(2 * morph_size + 1, 2 * morph_size + 1), cv::Point(morph_size, morph_size)); - cv::morphologyEx(foregroundMaskGrey, foregroundMaskGrey, cv::MORPH_OPEN, element, cv::Point(-1, -1), 2); - cv::morphologyEx(foregroundMaskGrey, foregroundMaskGrey, cv::MORPH_CLOSE, element, cv::Point(-1, -1), 2); - findContours(foregroundMaskGrey, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); - - fgPoints1.clear(); - for (const auto &c : contours) { - for (const auto &pt : c) { - fgPoints1.push_back(pt); - } - } - - queue.enqueue({fgPoints1, videoFrame1.clone()}); - } - queue.enqueue({{}, {}}); - }); - - cv::UMat frameBuffer; - cv::UMat background; - cv::UMat foreground(HEIGHT, WIDTH, CV_8UC4, cv::Scalar::all(0)); - cv::UMat videoFrame2, nextVideoFrameGray, prevVideoFrameGray; - vector fgPoints2, prevPoints, nextPoints, newPoints; - double avgLength = 1; - - uint64_t cnt = 1; - int64 start = cv::getTickCount(); - double tickFreq = cv::getTickFrequency(); - double lastFps = fps; - - while (!done) { - auto p = queue.dequeue(); - fgPoints2 = p.first; - videoFrame2 = p.second; - - if(videoFrame2.empty()) - break; - - cv::cvtColor(videoFrame2, background, cv::COLOR_RGB2BGRA); - cvtColor(videoFrame2, nextVideoFrameGray, cv::COLOR_RGB2GRAY); - - if (fgPoints2.size() > 4) { - prevPoints = fgPoints2; - if (prevVideoFrameGray.empty()) { - prevVideoFrameGray = nextVideoFrameGray.clone(); - } - - std::vector status; - std::vector err; - cv::TermCriteria criteria = cv::TermCriteria((cv::TermCriteria::COUNT) + (cv::TermCriteria::EPS), 10, 0.03); - cv::calcOpticalFlowPyrLK(prevVideoFrameGray, nextVideoFrameGray, prevPoints, nextPoints, status, err, cv::Size(15, 15), 2, criteria); - - nvg::begin(); - nvg::clear(); - newPoints.clear(); - using kb::nvg::vg; - nvgBeginPath(vg); - nvgStrokeWidth(vg, std::fmax(3.0, WIDTH/960.0)); - nvgStrokeColor(vg, nvgHSLA(0.1, 1, 0.5, 64)); - - for (size_t i = 0; i < prevPoints.size(); i++) { - if (status[i] == 1 && nextPoints[i].y >= 0 && nextPoints[i].x >= 0 && nextPoints[i].y < nextVideoFrameGray.rows && nextPoints[i].x < nextVideoFrameGray.cols) { - double len = hypot(fabs(nextPoints[i].x - prevPoints[i].x), fabs(nextPoints[i].y - prevPoints[i].y)); - avgLength = ((avgLength * 0.95) + (len * 0.05)); - if (len > 0 && len < avgLength) { - newPoints.push_back(nextPoints[i]); - - nvgMoveTo(vg, nextPoints[i].x, nextPoints[i].y); - nvgLineTo(vg, prevPoints[i].x, prevPoints[i].y); - } - } - } - nvgStroke(vg); - nvg::end(); - - prevVideoFrameGray = nextVideoFrameGray.clone(); - prevPoints = nextPoints; - } - gl::acquire_from_gl(frameBuffer); - - cv::flip(frameBuffer, frameBuffer, 0); - cv::addWeighted(foreground, 0.9, frameBuffer, 1.1, 0.0, foreground); - cv::addWeighted(background, 1.0, foreground, 1.0, 0.0, frameBuffer); - cv::flip(frameBuffer, frameBuffer, 0); - cv::cvtColor(frameBuffer, videoFrame2, cv::COLOR_BGRA2RGB); - - gl::release_to_gl(frameBuffer); - - va::bind(); - cv::flip(videoFrame2, videoFrame2, 0); - encoder.write(videoFrame2); - - if (x11::is_initialized()) { - gl::bind(); - gl::blit_frame_buffer_to_screen(); - - if (x11::window_closed()) { - finish(0); - break; - } - - gl::swap_buffers(); - } - - //Measure FPS - if (cnt % uint64(ceil(lastFps)) == 0) { - int64 tick = cv::getTickCount(); - lastFps = tickFreq / ((tick - start + 1) / cnt); - cerr << "FPS : " << lastFps << '\r'; - start = tick; - cnt = 1; - } - - ++cnt; - } - - producer.join(); - - return 0; -}