Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #21658 from sivanov-work:vpp_core_add_roi

Browse Source 
G-API: Add ROI processing in VPP preproc

* Add ROI in VPP prepro

* Apply comments
pull/21690/head
Sergey Ivanov 3 years ago committed by GitHub
parent 8fa429caef
commit 44c2c77548
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 206 additions and 101 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp
  2. 48
      modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
  3. 3
      modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp
  4. 18
      modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp
  5. 12
      modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp
  6. 3
      modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp
  7. 217
      modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp

6
modules/gapi/perf/streaming/gapi_streaming_source_perf_tests.cpp
Unescape View File

@ -205,6 +205,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance)
}
VPPPreprocEngine preproc_engine(std::move(policy));
cv::gapi::wip::Data out;
cv::util::optional<cv::Rect> empty_roi;
TEST_CYCLE()
{
source_ptr->pull(out);
@ -212,7 +213,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Test, TestPerformance)
cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);
pp_session sess = preproc_engine.initialize_preproc(param.value(),
required_frame_param);
(void)preproc_engine.run_sync(sess, frame);
(void)preproc_engine.run_sync(sess, frame, empty_roi);
}
SANITY_CHECK_NOTHING();
@ -269,6 +270,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance)
}
VPPPreprocEngine preproc_engine(std::move(policy));
cv::gapi::wip::Data out;
cv::util::optional<cv::Rect> empty_roi;
TEST_CYCLE()
{
source_ptr->pull(out);
@ -276,7 +278,7 @@ PERF_TEST_P_(OneVPLSourcePerf_PP_Engine_Bypass_Test, TestPerformance)
cv::util::optional<pp_params> param = preproc_engine.is_applicable(frame);
pp_session sess = preproc_engine.initialize_preproc(param.value(),
required_frame_param);
(void)preproc_engine.run_sync(sess, frame);
(void)preproc_engine.run_sync(sess, frame, empty_roi);
}
SANITY_CHECK_NOTHING();

48
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
Unescape View File

@ -34,6 +34,21 @@ bool FrameInfoComparator::equal_to(const mfxFrameInfo& lhs, const mfxFrameInfo&
return lhs == rhs;
}
void apply_roi(mfxFrameSurface1* surface_handle,
const cv::util::optional<cv::Rect> &opt_roi) {
if (opt_roi.has_value()) {
const cv::Rect &roi = opt_roi.value();
surface_handle->Info.CropX = static_cast<mfxU16>(roi.x);
surface_handle->Info.CropY = static_cast<mfxU16>(roi.y);
surface_handle->Info.CropW = static_cast<mfxU16>(roi.width);
surface_handle->Info.CropH = static_cast<mfxU16>(roi.height);
GAPI_LOG_DEBUG(nullptr, "applied ROI {" << surface_handle->Info.CropX <<
", " << surface_handle->Info.CropY << "}, "
"{ " << surface_handle->Info.CropX + surface_handle->Info.CropW <<
", " << surface_handle->Info.CropY + surface_handle->Info.CropH << "}");
}
}
VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& accel) :
ProcessingEngineBase(std::move(accel)) {
GAPI_LOG_INFO(nullptr, "Create VPP preprocessing engine");
@ -57,31 +72,25 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& acce
my_sess.sync_in_queue.pop();
auto *vpp_suface = my_sess.processing_surface_ptr.lock()->get_handle();
/* TODO: consider CROP/ROI here
static int x_offset = 0;
static int y_offset = 0;
dec_surface->Info.CropX = x_offset;
dec_surface->Info.CropY = y_offset;
dec_surface->Info.CropW = 100 + x_offset++;
dec_surface->Info.CropH = 100 + y_offset++;
*/
session_type::outgoing_task vpp_pending_op {pending_op.sync_handle, nullptr};
apply_roi(pending_op.decoded_surface_ptr, pending_op.roi);
mfxSyncPoint vpp_sync_handle{};
my_sess.last_status = MFXVideoVPP_RunFrameVPPAsync(my_sess.session,
pending_op.decoded_surface_ptr,
vpp_suface,
nullptr, &vpp_pending_op.sync_handle);
vpp_pending_op.vpp_surface_ptr = vpp_suface;
nullptr, &vpp_sync_handle);
session_type::outgoing_task vpp_pending_op {vpp_sync_handle,
vpp_suface,
std::move(pending_op) };
GAPI_LOG_DEBUG(nullptr, "Got VPP async operation" <<
", sync id: " <<
vpp_pending_op.sync_handle <<
", dec surface: " <<
pending_op.decoded_surface_ptr <<
vpp_pending_op.original_surface_ptr <<
", trans surface: " <<
vpp_pending_op.vpp_surface_ptr <<
", status: " <<
mfxstatus_to_string(my_sess.last_status));
// NB: process status
if (my_sess.last_status == MFX_ERR_MORE_SURFACE ||
my_sess.last_status == MFX_ERR_NONE) {
@ -131,6 +140,7 @@ VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& acce
// put frames in ready queue on success
if (MFX_ERR_NONE == sess.last_status) {
pending_op.release_frame();
on_frame_ready(my_sess, pending_op.vpp_surface_ptr);
}
}
@ -327,8 +337,8 @@ VPPPreprocEngine::initialize_session(mfxSession,
return {};
}
cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame) {
cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &roi) {
std::shared_ptr<EngineSession> pp_sess_impl = sess.get<EngineSession>();
if (!pp_sess_impl) {
// bypass case
@ -347,8 +357,10 @@ cv::MediaFrame VPPPreprocEngine::run_sync(const pp_session& sess, const cv::Medi
// schedule decoded surface into preproc queue
session_type::incoming_task in_preproc_request {nullptr,
vpl_adapter->get_surface()->get_handle(),
in_frame};
vpl_adapter->get_surface()->get_handle(),
vpl_adapter->get_surface()->get_info(),
in_frame,
roi};
s->sync_in_queue.emplace(in_preproc_request);
// invoke pipeline to transform decoded surface into preprocessed surface

3
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.hpp
Unescape View File

@ -47,7 +47,8 @@ public:
const GFrameDesc& required_frame_descr) override;
cv::MediaFrame run_sync(const pp_session &session_handle,
const cv::MediaFrame& in_frame) override;
const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &opt_roi) override;
private:
std::map<mfxFrameInfo, session_ptr_type, FrameInfoComparator> preproc_session_map;

18
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.cpp
Unescape View File

@ -60,6 +60,24 @@ void VPPPreprocSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key)
const mfxFrameInfo& VPPPreprocSession::get_video_param() const {
return mfx_vpp_out_param.vpp.Out;
}
VPPPreprocSession::outgoing_task::outgoing_task(mfxSyncPoint acquired_sync_handle,
mfxFrameSurface1* acquired_surface_ptr,
VPPPreprocSession::incoming_task &&in) :
sync_handle(acquired_sync_handle),
vpp_surface_ptr(acquired_surface_ptr),
original_surface_ptr(in.decoded_surface_ptr),
original_frame_info(std::move(in.decoded_frame_info)),
original_frame(in.decoded_frame_copy) {
}
void VPPPreprocSession::outgoing_task::release_frame() {
// restore initial surface params
memcpy(&(original_surface_ptr->Info),
&original_frame_info, sizeof(Surface::info_t));
// release references on frame adapter
original_frame = cv::MediaFrame();
}
} // namespace onevpl
} // namespace wip
} // namespace gapi

12
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_session.hpp
Unescape View File

@ -41,12 +41,24 @@ private:
struct incoming_task {
mfxSyncPoint sync_handle;
mfxFrameSurface1* decoded_surface_ptr;
Surface::info_t decoded_frame_info;
cv::MediaFrame decoded_frame_copy;
cv::util::optional<cv::Rect> roi;
};
struct outgoing_task {
outgoing_task() = default;
outgoing_task(mfxSyncPoint acquired_sync_handle,
mfxFrameSurface1* acquired_surface_ptr,
incoming_task &&in);
mfxSyncPoint sync_handle;
mfxFrameSurface1* vpp_surface_ptr;
mfxFrameSurface1* original_surface_ptr;
void release_frame();
private:
Surface::info_t original_frame_info;
cv::MediaFrame original_frame;
};
std::queue<incoming_task> sync_in_queue;

3
modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp
Unescape View File

@ -27,7 +27,8 @@ struct IPreprocEngine {
initialize_preproc(const pp_params& initial_frame_param,
const GFrameDesc& required_frame_descr) = 0;
virtual cv::MediaFrame
run_sync(const pp_session &sess, const cv::MediaFrame& in_frame) = 0;
run_sync(const pp_session &sess, const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &opt_roi = {}) = 0;
};
} // namespace wip
} // namespace gapi

217
modules/gapi/test/streaming/gapi_streaming_vpp_preproc_test.cpp
Unescape View File

@ -181,6 +181,8 @@ using acceleration_t = int;
using out_frame_info_t = cv::GFrameDesc;
using preproc_args_t = std::tuple<source_t, decoder_t, acceleration_t, out_frame_info_t>;
static cv::util::optional<cv::Rect> empty_roi;
class VPPPreprocParams : public ::testing::TestWithParam<preproc_args_t> {};
preproc_args_t files[] = {
@ -246,7 +248,9 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread)
required_frame_param);
// 2) make preproc using incoming decoded frame & preproc session
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, first_decoded_frame);
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess,
first_decoded_frame,
empty_roi);
cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc();
ASSERT_FALSE(first_frame_decoded_desc == first_outcome_pp_desc);
@ -278,7 +282,9 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread)
ASSERT_EQ(pp_sess.get<EngineSession>().get(),
first_pp_sess.get<EngineSession>().get());
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame);
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess,
decoded_frame,
empty_roi);
cv::GFrameDesc pp_desc = pp_frame.desc();
ASSERT_TRUE(pp_desc == first_outcome_pp_desc);
in_progress = false;
@ -291,8 +297,95 @@ TEST(OneVPL_Source_PreprocEngine, functional_single_thread)
ASSERT_NE(frames_processed_count, 1);
}
void decode_function(cv::gapi::wip::onevpl::VPLLegacyDecodeEngine &decode_engine,
cv::gapi::wip::onevpl::ProcessingEngineBase::session_ptr sess_ptr,
SafeQueue &queue, size_t &decoded_number) {
// decode first frame
{
cv::MediaFrame decoded_frame;
ASSERT_NO_THROW(decoded_frame = extract_decoded_frame(sess_ptr->session, decode_engine));
queue.push(std::move(decoded_frame));
}
// launch pipeline
try {
while(true) {
queue.push(extract_decoded_frame(sess_ptr->session, decode_engine));
decoded_number++;
}
} catch (...) {}
// send stop
queue.push_stop();
}
void preproc_function(cv::gapi::wip::onevpl::VPPPreprocEngine &preproc_engine, SafeQueue&queue,
size_t &preproc_number, const out_frame_info_t &required_frame_param,
const cv::util::optional<cv::Rect> &roi_rect = {}) {
using namespace cv::gapi::wip;
using namespace cv::gapi::wip::onevpl;
// create preproc session based on frame description & network info
cv::MediaFrame first_decoded_frame = queue.pop();
cv::util::optional<pp_params> first_pp_params = preproc_engine.is_applicable(first_decoded_frame);
ASSERT_TRUE(first_pp_params.has_value());
pp_session first_pp_sess =
preproc_engine.initialize_preproc(first_pp_params.value(),
required_frame_param);
// make preproc using incoming decoded frame & preproc session
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess,
first_decoded_frame,
roi_rect);
cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc();
// do not hold media frames because they share limited DX11 surface pool resources
first_decoded_frame = cv::MediaFrame();
first_pp_frame = cv::MediaFrame();
// launch pipeline
bool in_progress = false;
// let's allow counting of preprocessed frames to check this value later:
// Currently, it looks redundant to implement any kind of gracefull shutdown logic
// in this test - so let's apply agreement that media source is processed
// succesfully when preproc_number != 1 in result.
// Specific validation logic which adhere to explicit counter value may be implemented
// in particular test scope
preproc_number = 1;
try {
while(true) {
cv::MediaFrame decoded_frame = queue.pop();
if (SafeQueue::is_stop(decoded_frame)) {
break;
}
in_progress = true;
cv::util::optional<pp_params> params = preproc_engine.is_applicable(decoded_frame);
ASSERT_TRUE(params.has_value());
ASSERT_TRUE(0 == memcmp(&params.value(), &first_pp_params.value(), sizeof(pp_params)));
pp_session pp_sess = preproc_engine.initialize_preproc(params.value(),
required_frame_param);
ASSERT_EQ(pp_sess.get<EngineSession>().get(),
first_pp_sess.get<EngineSession>().get());
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame, empty_roi);
cv::GFrameDesc pp_desc = pp_frame.desc();
ASSERT_TRUE(pp_desc == first_outcome_pp_desc);
in_progress = false;
preproc_number++;
}
} catch (...) {}
// test if interruption has happened
ASSERT_FALSE(in_progress);
ASSERT_NE(preproc_number, 1);
}
TEST_P(VPPPreprocParams, functional_different_threads)
using roi_t = cv::util::optional<cv::Rect>;
using preproc_roi_args_t = decltype(std::tuple_cat(std::declval<preproc_args_t>(),
std::declval<std::tuple<roi_t>>()));
class VPPPreprocROIParams : public ::testing::TestWithParam<preproc_roi_args_t> {};
TEST_P(VPPPreprocROIParams, functional_roi_different_threads)
{
using namespace cv::gapi::wip;
using namespace cv::gapi::wip::onevpl;
@ -300,7 +393,8 @@ TEST_P(VPPPreprocParams, functional_different_threads)
decoder_t decoder_id;
acceleration_t accel;
out_frame_info_t required_frame_param;
std::tie(file_path, decoder_id, accel, required_frame_param) = GetParam();
roi_t opt_roi;
std::tie(file_path, decoder_id, accel, required_frame_param, opt_roi) = GetParam();
file_path = findDataFile(file_path);
@ -338,87 +432,52 @@ TEST_P(VPPPreprocParams, functional_different_threads)
size_t decoded_number = 1;
size_t preproc_number = 0;
std::thread decode_thread([&decode_engine, sess_ptr,
&queue, &decoded_number] () {
// decode first frame
{
cv::MediaFrame decoded_frame;
ASSERT_NO_THROW(decoded_frame = extract_decoded_frame(sess_ptr->session, decode_engine));
queue.push(std::move(decoded_frame));
}
// launch pipeline
try {
while(true) {
queue.push(extract_decoded_frame(sess_ptr->session, decode_engine));
decoded_number++;
}
} catch (...) {}
// send stop
queue.push_stop();
});
std::thread preproc_thread([&preproc_engine, &queue, &preproc_number, required_frame_param] () {
// create preproc session based on frame description & network info
cv::MediaFrame first_decoded_frame = queue.pop();
cv::util::optional<pp_params> first_pp_params = preproc_engine.is_applicable(first_decoded_frame);
ASSERT_TRUE(first_pp_params.has_value());
pp_session first_pp_sess =
preproc_engine.initialize_preproc(first_pp_params.value(), required_frame_param);
// make preproc using incoming decoded frame & preproc session
cv::MediaFrame first_pp_frame = preproc_engine.run_sync(first_pp_sess, first_decoded_frame);
cv::GFrameDesc first_outcome_pp_desc = first_pp_frame.desc();
// do not hold media frames because they share limited DX11 surface pool resources
first_decoded_frame = cv::MediaFrame();
first_pp_frame = cv::MediaFrame();
// launch pipeline
bool in_progress = false;
// let's allow counting of preprocessed frames to check this value later:
// Currently, it looks redundant to implement any kind of gracefull shutdown logic
// in this test - so let's apply agreement that media source is processed
// succesfully when preproc_number != 1 in result
preproc_number = 1;
try {
while(true) {
cv::MediaFrame decoded_frame = queue.pop();
if (SafeQueue::is_stop(decoded_frame)) {
break;
}
in_progress = true;
cv::util::optional<pp_params> params = preproc_engine.is_applicable(decoded_frame);
ASSERT_TRUE(params.has_value());
ASSERT_TRUE(0 == memcmp(&params.value(), &first_pp_params.value(), sizeof(pp_params)));
pp_session pp_sess = preproc_engine.initialize_preproc(params.value(),
required_frame_param);
ASSERT_EQ(pp_sess.get<EngineSession>().get(),
first_pp_sess.get<EngineSession>().get());
cv::MediaFrame pp_frame = preproc_engine.run_sync(pp_sess, decoded_frame);
cv::GFrameDesc pp_desc = pp_frame.desc();
ASSERT_TRUE(pp_desc == first_outcome_pp_desc);
in_progress = false;
preproc_number++;
}
} catch (...) {}
// test if interruption has happened
ASSERT_FALSE(in_progress);
ASSERT_NE(preproc_number, 1);
});
std::thread decode_thread(decode_function, std::ref(decode_engine), sess_ptr,
std::ref(queue), std::ref(decoded_number));
std::thread preproc_thread(preproc_function, std::ref(preproc_engine),
std::ref(queue), std::ref(preproc_number),
std::cref(required_frame_param),
std::cref(opt_roi));
decode_thread.join();
preproc_thread.join();
ASSERT_EQ(preproc_number, decoded_number);
}
INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngine, VPPPreprocParams,
testing::ValuesIn(files));
preproc_roi_args_t files_w_roi[] = {
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{cv::Rect{0,0,50,50}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{cv::Rect{0,0,100,100}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h264",
MFX_CODEC_AVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1080}}},
roi_t{cv::Rect{100,100,200,200}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h265",
MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}},
roi_t{cv::Rect{0,0,100,100}}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h265",
MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}},
roi_t{}},
preproc_roi_args_t {"highgui/video/big_buck_bunny.h265",
MFX_CODEC_HEVC, MFX_ACCEL_MODE_VIA_D3D11,
out_frame_info_t{cv::GFrameDesc {cv::MediaFormat::NV12, {1920, 1280}}},
roi_t{cv::Rect{100,100,200,200}}}
};
INSTANTIATE_TEST_CASE_P(OneVPL_Source_PreprocEngineROI, VPPPreprocROIParams,
testing::ValuesIn(files_w_roi));
using VPPInnerPreprocParams = VPPPreprocParams;
TEST_P(VPPInnerPreprocParams, functional_inner_preproc_size)

Write Preview
Loading…
Cancel
Save