From 762045648614691240e950bcfbd141d19fdc334b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 6 Dec 2021 09:54:48 +0300 Subject: [PATCH 1/8] videoio(MSMF): add queue for async ReadSample() --- modules/videoio/src/cap_msmf.cpp | 87 +++++++++++++++++++--------- modules/videoio/test/test_camera.cpp | 15 ++++- 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index fe3d261d34..fcc9260584 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -31,6 +31,7 @@ #endif #include #include +#include #include #include #include @@ -308,8 +309,10 @@ private: class SourceReaderCB : public IMFSourceReaderCallback { public: + static const size_t MSMF_READER_MAX_QUEUE_SIZE = 3; + SourceReaderCB() : - m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0), m_lastSampleTimestamp(0) + m_nRefCount(0), m_hEvent(CreateEvent(NULL, FALSE, FALSE, NULL)), m_bEOS(FALSE), m_hrStatus(S_OK), m_reader(NULL), m_dwStreamIndex(0) { } @@ -354,12 +357,19 @@ public: if (pSample) { CV_LOG_DEBUG(NULL, "videoio(MSMF): got frame at " << llTimestamp); - if (m_lastSample.Get()) + if (m_capturedFrames.size() >= MSMF_READER_MAX_QUEUE_SIZE) { - CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed)"); +#if 0 + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop frame (not processed). Timestamp=" << m_capturedFrames.front().timestamp); + m_capturedFrames.pop(); +#else + // this branch reduces latency if we drop frames due to slow processing. + // avoid fetching of already outdated frames from the queue's front. + CV_LOG_DEBUG(NULL, "videoio(MSMF): drop previous frames (not processed): " << m_capturedFrames.size()); + std::queue().swap(m_capturedFrames); // similar to missing m_capturedFrames.clean(); +#endif } - m_lastSampleTimestamp = llTimestamp; - m_lastSample = pSample; + m_capturedFrames.emplace(CapturedFrameInfo{ llTimestamp, _ComPtr(pSample), hrStatus }); } } else @@ -396,32 +406,45 @@ public: return S_OK; } - HRESULT Wait(DWORD dwMilliseconds, _ComPtr& videoSample, BOOL& pbEOS) + HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, LONGLONG& sampleTimestamp, BOOL& pbEOS) { pbEOS = FALSE; - DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); - if (dwResult == WAIT_TIMEOUT) + for (;;) { - return E_PENDING; - } - else if (dwResult != WAIT_OBJECT_0) - { - return HRESULT_FROM_WIN32(GetLastError()); - } + { + cv::AutoLock lock(m_mutex); - pbEOS = m_bEOS; - if (!pbEOS) - { - cv::AutoLock lock(m_mutex); - videoSample = m_lastSample; - CV_Assert(videoSample); - m_lastSample.Release(); - ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. - } + pbEOS = m_bEOS && m_capturedFrames.empty(); + if (pbEOS) + return m_hrStatus; + + if (!m_capturedFrames.empty()) + { + CV_Assert(!m_capturedFrames.empty()); + CapturedFrameInfo frameInfo = m_capturedFrames.front(); m_capturedFrames.pop(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): handle frame at " << frameInfo.timestamp); + mediaSample = frameInfo.sample; + CV_Assert(mediaSample); + sampleTimestamp = frameInfo.timestamp; + ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. + return frameInfo.hrStatus; + } + } - return m_hrStatus; + CV_LOG_DEBUG(NULL, "videoio(MSMF): waiting for frame... "); + DWORD dwResult = WaitForSingleObject(m_hEvent, dwMilliseconds); + if (dwResult == WAIT_TIMEOUT) + { + return E_PENDING; + } + else if (dwResult != WAIT_OBJECT_0) + { + return HRESULT_FROM_WIN32(GetLastError()); + } + } } + private: // Destructor is private. Caller should call Release. virtual ~SourceReaderCB() @@ -438,8 +461,14 @@ public: IMFSourceReader *m_reader; DWORD m_dwStreamIndex; - LONGLONG m_lastSampleTimestamp; - _ComPtr m_lastSample; + + struct CapturedFrameInfo { + LONGLONG timestamp; + _ComPtr sample; + HRESULT hrStatus; + }; + + std::queue m_capturedFrames; }; //================================================================================================== @@ -902,7 +931,7 @@ bool CvCapture_MSMF::grabFrame() } } BOOL bEOS = false; - if (FAILED(hr = reader->Wait(10000, videoSample, bEOS))) // 10 sec + if (FAILED(hr = reader->Wait(10000, videoSample, sampleTime, bEOS))) // 10 sec { CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); return false; @@ -912,7 +941,7 @@ bool CvCapture_MSMF::grabFrame() CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost"); return false; } - sampleTime = reader->m_lastSampleTimestamp; + CV_LOG_DEBUG(NULL, "videoio(MSMF): grabbed frame " << sampleTime); return true; } else if (isOpen) @@ -991,6 +1020,7 @@ bool CvCapture_MSMF::grabFrame() bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) { CV_TRACE_FUNCTION(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame start..."); do { if (!videoSample) @@ -1082,6 +1112,7 @@ bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) buffer2d->Unlock2D(); else buf->Unlock(); + CV_LOG_DEBUG(NULL, "videoio(MSMF): retrieve video frame done!"); return !frame.empty(); } while (0); diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp index e82285ad5e..135eb8eea5 100644 --- a/modules/videoio/test/test_camera.cpp +++ b/modules/videoio/test/test_camera.cpp @@ -25,15 +25,26 @@ static void test_readFrames(/*const*/ VideoCapture& capture, const int N = 100, const bool validTickAndFps = cvTickFreq != 0 && fps != 0.; testTimestamps &= validTickAndFps; + double frame0ts = 0; + for (int i = 0; i < N; i++) { SCOPED_TRACE(cv::format("frame=%d", i)); capture >> frame; - const int64 sysTimeCurr = cv::getTickCount(); - const double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); ASSERT_FALSE(frame.empty()); + const int64 sysTimeCurr = cv::getTickCount(); + double camTimeCurr = capture.get(cv::CAP_PROP_POS_MSEC); + if (i == 0) + frame0ts = camTimeCurr; + camTimeCurr -= frame0ts; // normalized timestamp based on the first frame + + if (cvtest::debugLevel > 0) + { + std::cout << i << ": " << camTimeCurr << std::endl; + } + // Do we have a previous frame? if (i > 0 && testTimestamps) { From 542b3e8a64a1516657ec84fd9b2acaaa2baf6440 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Mon, 13 Dec 2021 23:43:49 +0100 Subject: [PATCH 2/8] Fix harmless signed integer overflow. When computing: t1 = (bayer[1] + bayer[bayer_step] + bayer[bayer_step+2] + bayer[bayer_step*2+1])*G2Y; there is a T (unsigned short or char) multiplied by an int which can overflow. Then again, it is stored to t1 which is unsigned so the overflow disappears. Keeping all unsigned is safer. --- modules/imgproc/src/demosaicing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 03bc781046..27dfc1520c 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -603,7 +603,7 @@ public: virtual void operator ()(const Range& range) const CV_OVERRIDE { SIMDInterpolator vecOp; - const int G2Y = 9617; + const unsigned G2Y = 9617; const int SHIFT = 14; const T* bayer0 = srcmat.ptr(); From 52c423a42398af9d40e65e8becc33a4fccccbe83 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 13 Dec 2021 19:10:02 +0000 Subject: [PATCH 3/8] build: winpack_dldt with dldt 2021.4.2 --- cmake/OpenCVDetectInferenceEngine.cmake | 4 +- ...-dldt-disable-multidevice-autoplugin.patch | 16 ++ ...20210630-dldt-disable-unused-targets.patch | 219 ++++++++++++++++++ .../2021.4.2/20210630-dldt-pdb.patch | 15 ++ .../2021.4.2/20210630-dldt-vs-version.patch | 16 ++ .../winpack_dldt/2021.4.2/build.config.py | 1 + .../winpack_dldt/2021.4.2/patch.config.py | 4 + .../winpack_dldt/2021.4.2/sysroot.config.py | 56 +++++ platforms/winpack_dldt/build_package.py | 2 +- 9 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch create mode 100644 platforms/winpack_dldt/2021.4.2/build.config.py create mode 100644 platforms/winpack_dldt/2021.4.2/patch.config.py create mode 100644 platforms/winpack_dldt/2021.4.2/sysroot.config.py diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 35640fa719..128883b128 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -112,8 +112,8 @@ if(DEFINED InferenceEngine_VERSION) endif() endif() if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) - message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE_INIT "2021040100") + message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.2 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040200") elseif(DEFINED INF_ENGINE_RELEASE) set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") endif() diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 0000000000..f1e7487442 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..9d44cdadc6 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch new file mode 100644 index 0000000000..65e6f84dc8 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch new file mode 100644 index 0000000000..36b0068775 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4.2/build.config.py b/platforms/winpack_dldt/2021.4.2/build.config.py new file mode 100644 index 0000000000..708dcdddfb --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.2-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4.2/patch.config.py b/platforms/winpack_dldt/2021.4.2/patch.config.py new file mode 100644 index 0000000000..7f8715aae2 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4.2/sysroot.config.py b/platforms/winpack_dldt/2021.4.2/sysroot.config.py new file mode 100644 index 0000000000..fa4281107d --- /dev/null +++ b/platforms/winpack_dldt/2021.4.2/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index 0292b028b8..7875f11d31 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -469,7 +469,7 @@ class Builder: def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.4.1' + dldt_src_commit = '2021.4.2' dldt_config = None dldt_release = None From 3cc83ce024d252aa7306bccaf92fef8aeef2764c Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Tue, 14 Dec 2021 21:06:02 +0800 Subject: [PATCH 4/8] docs: correct normalize factor in gaussian pyramid tutorial --- doc/tutorials/imgproc/pyramids/pyramids.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/imgproc/pyramids/pyramids.markdown b/doc/tutorials/imgproc/pyramids/pyramids.markdown index 2c9a7ec8d3..94dd36e83d 100644 --- a/doc/tutorials/imgproc/pyramids/pyramids.markdown +++ b/doc/tutorials/imgproc/pyramids/pyramids.markdown @@ -47,7 +47,7 @@ Theory - To produce layer \f$(i+1)\f$ in the Gaussian pyramid, we do the following: - Convolve \f$G_{i}\f$ with a Gaussian kernel: - \f[\frac{1}{16} \begin{bmatrix} 1 & 4 & 6 & 4 & 1 \\ 4 & 16 & 24 & 16 & 4 \\ 6 & 24 & 36 & 24 & 6 \\ 4 & 16 & 24 & 16 & 4 \\ 1 & 4 & 6 & 4 & 1 \end{bmatrix}\f] + \f[\frac{1}{256} \begin{bmatrix} 1 & 4 & 6 & 4 & 1 \\ 4 & 16 & 24 & 16 & 4 \\ 6 & 24 & 36 & 24 & 6 \\ 4 & 16 & 24 & 16 & 4 \\ 1 & 4 & 6 & 4 & 1 \end{bmatrix}\f] - Remove every even-numbered row and column. From 72c55b56f25422d00cbb43baebe3a204d64b019d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 11 Dec 2021 17:28:21 +0000 Subject: [PATCH 5/8] imgproc: catch NaNs in clip(), use table index debug check - no NaN propagation guarantee --- modules/imgproc/src/color_lab.cpp | 7 ++++++- modules/imgproc/test/test_color.cpp | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index 6b03be6195..14df6473f4 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -978,8 +978,9 @@ static struct LUVLUT_T { const long long int *LvToVpl_b; } LUVLUT = {0, 0, 0}; +/* NB: no NaN propagation guarantee */ #define clip(value) \ - value < 0.0f ? 0.0f : value > 1.0f ? 1.0f : value; + value < 0.0f ? 0.0f : value <= 1.0f ? value : 1.0f; //all constants should be presented through integers to keep bit-exactness static const softdouble gammaThreshold = softdouble(809)/softdouble(20000); // 0.04045 @@ -1330,6 +1331,10 @@ static inline void trilinearInterpolate(int cx, int cy, int cz, const int16_t* L int ty = cy >> (lab_base_shift - lab_lut_shift); int tz = cz >> (lab_base_shift - lab_lut_shift); + CV_DbgCheck(tx, tx >= 0 && tx < LAB_LUT_DIM, ""); + CV_DbgCheck(ty, ty >= 0 && ty < LAB_LUT_DIM, ""); + CV_DbgCheck(tz, tz >= 0 && tz < LAB_LUT_DIM, ""); + const int16_t* baseLUT = &LUT[3*8*tx + (3*8*LAB_LUT_DIM)*ty + (3*8*LAB_LUT_DIM*LAB_LUT_DIM)*tz]; int aa[8], bb[8], cc[8]; for(int i = 0; i < 8; i++) diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index f828dac18a..0c4fdfb2ca 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -3117,5 +3117,25 @@ TEST(ImgProc_cvtColorTwoPlane, y_plane_padding_differs_from_uv_plane_padding_170 EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_uv_padded_mat, NORM_INF), .0); } +TEST(ImgProc_RGB2Lab, NaN_21111) +{ + const float kNaN = std::numeric_limits::quiet_NaN(); + cv::Mat3f src(1, 111, Vec3f::all(kNaN)), dst; + // Make some entries with only one NaN. + src(0, 0) = src(0, 27) = src(0, 81) = src(0, 108) = cv::Vec3f(0, 0, kNaN); + src(0, 1) = src(0, 28) = src(0, 82) = src(0, 109) = cv::Vec3f(0, kNaN, 0); + src(0, 2) = src(0, 29) = src(0, 83) = src(0, 110) = cv::Vec3f(kNaN, 0, 0); + EXPECT_NO_THROW(cvtColor(src, dst, COLOR_RGB2Lab)); + +#if 0 // no NaN propagation guarantee + for (int i = 0; i < 20; ++i) + { + for (int j = 0; j < 3; ++j) + { + EXPECT_TRUE(cvIsNaN(dst(0, i)[j])); + } + } +#endif +} }} // namespace From 4827fe86bbfd657a3a536ab2905b63a5e5172718 Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 14 Dec 2021 19:58:06 +0300 Subject: [PATCH 6/8] Merge pull request #21088 from rogday:onnx_tests Onnx conformance tests * Add ONNX conformance tests * dnn(test): add filters for ONNX conformance tests * add filter lists for OCV backend * address review comments * move test_clip_inbounds to all_denylist * address clip issue * avoid empty lists Co-authored-by: Alexander Alekhin --- modules/dnn/src/onnx/onnx_importer.cpp | 1 + modules/dnn/test/test_common.hpp | 6 +- modules/dnn/test/test_common.impl.hpp | 12 +- modules/dnn/test/test_onnx_conformance.cpp | 1250 +++++++++++++++++ ...e_layer_filter_opencv_all_denylist.inl.hpp | 23 + ...e_layer_filter_opencv_cpu_denylist.inl.hpp | 0 ...er_filter_opencv_ocl_fp16_denylist.inl.hpp | 21 + ...er_filter_opencv_ocl_fp32_denylist.inl.hpp | 2 + ..._conformance_layer_parser_denylist.inl.hpp | 717 ++++++++++ modules/ts/misc/testlog_parser.py | 5 +- 10 files changed, 2034 insertions(+), 3 deletions(-) create mode 100644 modules/dnn/test/test_onnx_conformance.cpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index f776bdc5da..8ff91f5e44 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1205,6 +1205,7 @@ void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx: void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { + CV_CheckEQ(node_proto.input_size(), 1, ""); layerParams.type = "ReLU6"; layerParams.set("min_value", layerParams.get("min", -FLT_MAX)); layerParams.set("max_value", layerParams.get("max", FLT_MAX)); diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index 02a676da36..f2573f7562 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -18,8 +18,9 @@ #define INF_ENGINE_VER_MAJOR_LE(ver) (((INF_ENGINE_RELEASE) / 10000) <= ((ver) / 10000)) #define INF_ENGINE_VER_MAJOR_EQ(ver) (((INF_ENGINE_RELEASE) / 10000) == ((ver) / 10000)) - +#define CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND "dnn_skip_opencv_backend" #define CV_TEST_TAG_DNN_SKIP_HALIDE "dnn_skip_halide" +#define CV_TEST_TAG_DNN_SKIP_CPU "dnn_skip_cpu" #define CV_TEST_TAG_DNN_SKIP_OPENCL "dnn_skip_ocl" #define CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 "dnn_skip_ocl_fp16" #define CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER "dnn_skip_ie_nn_builder" @@ -38,6 +39,9 @@ #define CV_TEST_TAG_DNN_SKIP_IE_MYRIAD CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2, CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X #define CV_TEST_TAG_DNN_SKIP_IE_ARM_CPU "dnn_skip_ie_arm_cpu" +#define CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE "dnn_skip_onnx_conformance" +#define CV_TEST_TAG_DNN_SKIP_PARSER "dnn_skip_parser" + #ifdef HAVE_INF_ENGINE #if INF_ENGINE_VER_MAJOR_EQ(2018050000) diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index a11c6641b2..a55a8f788c 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -361,9 +361,15 @@ void initDNNTests() cvtest::addDataSearchPath(extraTestDataPath); registerGlobalSkipTag( - CV_TEST_TAG_DNN_SKIP_HALIDE, + CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, + CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16 ); +#if defined(HAVE_HALIDE) + registerGlobalSkipTag( + CV_TEST_TAG_DNN_SKIP_HALIDE + ) +#endif #if defined(INF_ENGINE_RELEASE) registerGlobalSkipTag( CV_TEST_TAG_DNN_SKIP_IE, @@ -392,6 +398,10 @@ void initDNNTests() CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16 ); #endif + registerGlobalSkipTag( + CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE, + CV_TEST_TAG_DNN_SKIP_PARSER + ); } } // namespace diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp new file mode 100644 index 0000000000..a1d83bd614 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -0,0 +1,1250 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + + +#include "test_precomp.hpp" +#include +#include +#include "npy_blob.hpp" +#include + +#if defined(_MSC_VER) // workaround for 32-bit MSVC compiler +#pragma optimize("", off) +#endif + + +#define CV_TEST_TAG_DNN_ERROR_PARSER "dnn_error_parser" +#define CV_TEST_TAG_DNN_ERROR_NET_SETUP "dnn_error_net_setup" +#define CV_TEST_TAG_DNN_ERROR_FORWARD "dnn_error_forward" +#define CV_TEST_TAG_DNN_LAYER_FALLBACK "dnn_layer_fallback" +#define CV_TEST_TAG_DNN_NO_ACCURACY_CHECK "dnn_no_accuracy_check" + + +namespace opencv_test { + +struct TestCase +{ + const char* name; + uint32_t inputs; + uint32_t outputs; +}; + +static const TestCase testConformanceConfig[] = { + {"test_abs", 1, 1}, + {"test_acos", 1, 1}, + {"test_acos_example", 1, 1}, + {"test_acosh", 1, 1}, + {"test_acosh_example", 1, 1}, + {"test_adagrad", 5, 2}, + {"test_adagrad_multiple", 8, 4}, + {"test_adam", 6, 3}, + {"test_adam_multiple", 10, 6}, + {"test_add", 2, 1}, + {"test_add_bcast", 2, 1}, + {"test_add_uint8", 2, 1}, + {"test_and2d", 2, 1}, + {"test_and3d", 2, 1}, + {"test_and4d", 2, 1}, + {"test_and_bcast3v1d", 2, 1}, + {"test_and_bcast3v2d", 2, 1}, + {"test_and_bcast4v2d", 2, 1}, + {"test_and_bcast4v3d", 2, 1}, + {"test_and_bcast4v4d", 2, 1}, + {"test_argmax_default_axis_example", 1, 1}, + {"test_argmax_default_axis_example_select_last_index", 1, 1}, + {"test_argmax_default_axis_random", 1, 1}, + {"test_argmax_default_axis_random_select_last_index", 1, 1}, + {"test_argmax_keepdims_example", 1, 1}, + {"test_argmax_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_keepdims_random", 1, 1}, + {"test_argmax_keepdims_random_select_last_index", 1, 1}, + {"test_argmax_negative_axis_keepdims_example", 1, 1}, + {"test_argmax_negative_axis_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_negative_axis_keepdims_random", 1, 1}, + {"test_argmax_negative_axis_keepdims_random_select_last_index", 1, 1}, + {"test_argmax_no_keepdims_example", 1, 1}, + {"test_argmax_no_keepdims_example_select_last_index", 1, 1}, + {"test_argmax_no_keepdims_random", 1, 1}, + {"test_argmax_no_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_default_axis_example", 1, 1}, + {"test_argmin_default_axis_example_select_last_index", 1, 1}, + {"test_argmin_default_axis_random", 1, 1}, + {"test_argmin_default_axis_random_select_last_index", 1, 1}, + {"test_argmin_keepdims_example", 1, 1}, + {"test_argmin_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_keepdims_random", 1, 1}, + {"test_argmin_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_negative_axis_keepdims_example", 1, 1}, + {"test_argmin_negative_axis_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_negative_axis_keepdims_random", 1, 1}, + {"test_argmin_negative_axis_keepdims_random_select_last_index", 1, 1}, + {"test_argmin_no_keepdims_example", 1, 1}, + {"test_argmin_no_keepdims_example_select_last_index", 1, 1}, + {"test_argmin_no_keepdims_random", 1, 1}, + {"test_argmin_no_keepdims_random_select_last_index", 1, 1}, + {"test_asin", 1, 1}, + {"test_asin_example", 1, 1}, + {"test_asinh", 1, 1}, + {"test_asinh_example", 1, 1}, + {"test_atan", 1, 1}, + {"test_atan_example", 1, 1}, + {"test_atanh", 1, 1}, + {"test_atanh_example", 1, 1}, + {"test_averagepool_1d_default", 1, 1}, + {"test_averagepool_2d_ceil", 1, 1}, + {"test_averagepool_2d_default", 1, 1}, + {"test_averagepool_2d_pads", 1, 1}, + {"test_averagepool_2d_pads_count_include_pad", 1, 1}, + {"test_averagepool_2d_precomputed_pads", 1, 1}, + {"test_averagepool_2d_precomputed_pads_count_include_pad", 1, 1}, + {"test_averagepool_2d_precomputed_same_upper", 1, 1}, + {"test_averagepool_2d_precomputed_strides", 1, 1}, + {"test_averagepool_2d_same_lower", 1, 1}, + {"test_averagepool_2d_same_upper", 1, 1}, + {"test_averagepool_2d_strides", 1, 1}, + {"test_averagepool_3d_default", 1, 1}, + {"test_basic_conv_with_padding", 2, 1}, + {"test_basic_conv_without_padding", 2, 1}, + {"test_basic_convinteger", 3, 1}, + {"test_batchnorm_epsilon", 5, 1}, + {"test_batchnorm_epsilon_training_mode", 5, 3}, + {"test_batchnorm_example", 5, 1}, + {"test_batchnorm_example_training_mode", 5, 3}, + {"test_bernoulli", 1, 1}, + {"test_bernoulli_double", 1, 1}, + {"test_bernoulli_double_expanded", 1, 1}, + {"test_bernoulli_expanded", 1, 1}, + {"test_bernoulli_seed", 1, 1}, + {"test_bernoulli_seed_expanded", 1, 1}, + {"test_bitshift_left_uint16", 2, 1}, + {"test_bitshift_left_uint32", 2, 1}, + {"test_bitshift_left_uint64", 2, 1}, + {"test_bitshift_left_uint8", 2, 1}, + {"test_bitshift_right_uint16", 2, 1}, + {"test_bitshift_right_uint32", 2, 1}, + {"test_bitshift_right_uint64", 2, 1}, + {"test_bitshift_right_uint8", 2, 1}, + {"test_cast_BFLOAT16_to_FLOAT", 1, 1}, + {"test_cast_DOUBLE_to_FLOAT", 1, 1}, + {"test_cast_DOUBLE_to_FLOAT16", 1, 1}, + {"test_cast_FLOAT16_to_DOUBLE", 1, 1}, + {"test_cast_FLOAT16_to_FLOAT", 1, 1}, + {"test_cast_FLOAT_to_BFLOAT16", 1, 1}, + {"test_cast_FLOAT_to_DOUBLE", 1, 1}, + {"test_cast_FLOAT_to_FLOAT16", 1, 1}, + {"test_cast_FLOAT_to_STRING", 1, 1}, + {"test_cast_STRING_to_FLOAT", 1, 1}, + {"test_castlike_BFLOAT16_to_FLOAT", 2, 1}, + {"test_castlike_BFLOAT16_to_FLOAT_expanded", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT16", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT16_expanded", 2, 1}, + {"test_castlike_DOUBLE_to_FLOAT_expanded", 2, 1}, + {"test_castlike_FLOAT16_to_DOUBLE", 2, 1}, + {"test_castlike_FLOAT16_to_DOUBLE_expanded", 2, 1}, + {"test_castlike_FLOAT16_to_FLOAT", 2, 1}, + {"test_castlike_FLOAT16_to_FLOAT_expanded", 2, 1}, + {"test_castlike_FLOAT_to_BFLOAT16", 2, 1}, + {"test_castlike_FLOAT_to_BFLOAT16_expanded", 2, 1}, + {"test_castlike_FLOAT_to_DOUBLE", 2, 1}, + {"test_castlike_FLOAT_to_DOUBLE_expanded", 2, 1}, + {"test_castlike_FLOAT_to_FLOAT16", 2, 1}, + {"test_castlike_FLOAT_to_FLOAT16_expanded", 2, 1}, + {"test_castlike_FLOAT_to_STRING", 2, 1}, + {"test_castlike_FLOAT_to_STRING_expanded", 2, 1}, + {"test_castlike_STRING_to_FLOAT", 2, 1}, + {"test_castlike_STRING_to_FLOAT_expanded", 2, 1}, + {"test_ceil", 1, 1}, + {"test_ceil_example", 1, 1}, + {"test_celu", 1, 1}, + {"test_celu_expanded", 1, 1}, + {"test_clip", 3, 1}, + {"test_clip_default_inbounds", 1, 1}, + {"test_clip_default_int8_inbounds", 1, 1}, + {"test_clip_default_int8_max", 2, 1}, + {"test_clip_default_int8_min", 2, 1}, + {"test_clip_default_max", 2, 1}, + {"test_clip_default_min", 2, 1}, + {"test_clip_example", 3, 1}, + {"test_clip_inbounds", 3, 1}, + {"test_clip_outbounds", 3, 1}, + {"test_clip_splitbounds", 3, 1}, + {"test_compress_0", 2, 1}, + {"test_compress_1", 2, 1}, + {"test_compress_default_axis", 2, 1}, + {"test_compress_negative_axis", 2, 1}, + {"test_concat_1d_axis_0", 2, 1}, + {"test_concat_1d_axis_negative_1", 2, 1}, + {"test_concat_2d_axis_0", 2, 1}, + {"test_concat_2d_axis_1", 2, 1}, + {"test_concat_2d_axis_negative_1", 2, 1}, + {"test_concat_2d_axis_negative_2", 2, 1}, + {"test_concat_3d_axis_0", 2, 1}, + {"test_concat_3d_axis_1", 2, 1}, + {"test_concat_3d_axis_2", 2, 1}, + {"test_concat_3d_axis_negative_1", 2, 1}, + {"test_concat_3d_axis_negative_2", 2, 1}, + {"test_concat_3d_axis_negative_3", 2, 1}, + {"test_constant", 0, 1}, + {"test_constant_pad", 3, 1}, + {"test_constantofshape_float_ones", 1, 1}, + {"test_constantofshape_int_shape_zero", 1, 1}, + {"test_constantofshape_int_zeros", 1, 1}, + {"test_conv_with_autopad_same", 2, 1}, + {"test_conv_with_strides_and_asymmetric_padding", 2, 1}, + {"test_conv_with_strides_no_padding", 2, 1}, + {"test_conv_with_strides_padding", 2, 1}, + {"test_convinteger_with_padding", 3, 1}, + {"test_convinteger_without_padding", 3, 1}, + {"test_convtranspose", 2, 1}, + {"test_convtranspose_1d", 2, 1}, + {"test_convtranspose_3d", 2, 1}, + {"test_convtranspose_autopad_same", 2, 1}, + {"test_convtranspose_dilations", 2, 1}, + {"test_convtranspose_kernel_shape", 2, 1}, + {"test_convtranspose_output_shape", 2, 1}, + {"test_convtranspose_pad", 2, 1}, + {"test_convtranspose_pads", 2, 1}, + {"test_convtranspose_with_kernel", 2, 1}, + {"test_cos", 1, 1}, + {"test_cos_example", 1, 1}, + {"test_cosh", 1, 1}, + {"test_cosh_example", 1, 1}, + {"test_cumsum_1d", 2, 1}, + {"test_cumsum_1d_exclusive", 2, 1}, + {"test_cumsum_1d_reverse", 2, 1}, + {"test_cumsum_1d_reverse_exclusive", 2, 1}, + {"test_cumsum_2d_axis_0", 2, 1}, + {"test_cumsum_2d_axis_1", 2, 1}, + {"test_cumsum_2d_negative_axis", 2, 1}, + {"test_depthtospace_crd_mode", 1, 1}, + {"test_depthtospace_crd_mode_example", 1, 1}, + {"test_depthtospace_dcr_mode", 1, 1}, + {"test_depthtospace_example", 1, 1}, + {"test_dequantizelinear", 3, 1}, + {"test_dequantizelinear_axis", 3, 1}, + {"test_det_2d", 1, 1}, + {"test_det_nd", 1, 1}, + {"test_div", 2, 1}, + {"test_div_bcast", 2, 1}, + {"test_div_example", 2, 1}, + {"test_div_uint8", 2, 1}, + {"test_dropout_default", 1, 1}, + {"test_dropout_default_mask", 1, 2}, + {"test_dropout_default_mask_ratio", 2, 2}, + {"test_dropout_default_old", 1, 1}, + {"test_dropout_default_ratio", 2, 1}, + {"test_dropout_random_old", 1, 1}, + {"test_dynamicquantizelinear", 1, 3}, + {"test_dynamicquantizelinear_expanded", 1, 3}, + {"test_dynamicquantizelinear_max_adjusted", 1, 3}, + {"test_dynamicquantizelinear_max_adjusted_expanded", 1, 3}, + {"test_dynamicquantizelinear_min_adjusted", 1, 3}, + {"test_dynamicquantizelinear_min_adjusted_expanded", 1, 3}, + {"test_edge_pad", 2, 1}, + {"test_einsum_batch_diagonal", 1, 1}, + {"test_einsum_batch_matmul", 2, 1}, + {"test_einsum_inner_prod", 2, 1}, + {"test_einsum_sum", 1, 1}, + {"test_einsum_transpose", 1, 1}, + {"test_elu", 1, 1}, + {"test_elu_default", 1, 1}, + {"test_elu_example", 1, 1}, + {"test_equal", 2, 1}, + {"test_equal_bcast", 2, 1}, + {"test_erf", 1, 1}, + {"test_exp", 1, 1}, + {"test_exp_example", 1, 1}, + {"test_expand_dim_changed", 2, 1}, + {"test_expand_dim_unchanged", 2, 1}, + {"test_eyelike_populate_off_main_diagonal", 1, 1}, + {"test_eyelike_with_dtype", 1, 1}, + {"test_eyelike_without_dtype", 1, 1}, + {"test_flatten_axis0", 1, 1}, + {"test_flatten_axis1", 1, 1}, + {"test_flatten_axis2", 1, 1}, + {"test_flatten_axis3", 1, 1}, + {"test_flatten_default_axis", 1, 1}, + {"test_flatten_negative_axis1", 1, 1}, + {"test_flatten_negative_axis2", 1, 1}, + {"test_flatten_negative_axis3", 1, 1}, + {"test_flatten_negative_axis4", 1, 1}, + {"test_floor", 1, 1}, + {"test_floor_example", 1, 1}, + {"test_gather_0", 2, 1}, + {"test_gather_1", 2, 1}, + {"test_gather_2d_indices", 2, 1}, + {"test_gather_elements_0", 2, 1}, + {"test_gather_elements_1", 2, 1}, + {"test_gather_elements_negative_indices", 2, 1}, + {"test_gather_negative_indices", 2, 1}, + {"test_gathernd_example_float32", 2, 1}, + {"test_gathernd_example_int32", 2, 1}, + {"test_gathernd_example_int32_batch_dim1", 2, 1}, + {"test_gemm_all_attributes", 3, 1}, + {"test_gemm_alpha", 3, 1}, + {"test_gemm_beta", 3, 1}, + {"test_gemm_default_matrix_bias", 3, 1}, + {"test_gemm_default_no_bias", 2, 1}, + {"test_gemm_default_scalar_bias", 3, 1}, + {"test_gemm_default_single_elem_vector_bias", 3, 1}, + {"test_gemm_default_vector_bias", 3, 1}, + {"test_gemm_default_zero_bias", 3, 1}, + {"test_gemm_transposeA", 3, 1}, + {"test_gemm_transposeB", 3, 1}, + {"test_globalaveragepool", 1, 1}, + {"test_globalaveragepool_precomputed", 1, 1}, + {"test_globalmaxpool", 1, 1}, + {"test_globalmaxpool_precomputed", 1, 1}, + {"test_greater", 2, 1}, + {"test_greater_bcast", 2, 1}, + {"test_greater_equal", 2, 1}, + {"test_greater_equal_bcast", 2, 1}, + {"test_greater_equal_bcast_expanded", 2, 1}, + {"test_greater_equal_expanded", 2, 1}, + {"test_gridsample", 2, 1}, + {"test_gridsample_aligncorners_true", 2, 1}, + {"test_gridsample_bicubic", 2, 1}, + {"test_gridsample_bilinear", 2, 1}, + {"test_gridsample_border_padding", 2, 1}, + {"test_gridsample_nearest", 2, 1}, + {"test_gridsample_reflection_padding", 2, 1}, + {"test_gridsample_zeros_padding", 2, 1}, + {"test_gru_batchwise", 3, 2}, + {"test_gru_defaults", 3, 1}, + {"test_gru_seq_length", 4, 1}, + {"test_gru_with_initial_bias", 4, 1}, + {"test_hardmax_axis_0", 1, 1}, + {"test_hardmax_axis_1", 1, 1}, + {"test_hardmax_axis_2", 1, 1}, + {"test_hardmax_default_axis", 1, 1}, + {"test_hardmax_example", 1, 1}, + {"test_hardmax_negative_axis", 1, 1}, + {"test_hardmax_one_hot", 1, 1}, + {"test_hardsigmoid", 1, 1}, + {"test_hardsigmoid_default", 1, 1}, + {"test_hardsigmoid_example", 1, 1}, + {"test_hardswish", 1, 1}, + {"test_hardswish_expanded", 1, 1}, + {"test_identity", 1, 1}, + {"test_identity_opt", 1, 1}, + {"test_identity_sequence", 1, 1}, + {"test_if", 1, 1}, + {"test_if_opt", 1, 1}, + {"test_if_seq", 1, 1}, + {"test_instancenorm_epsilon", 3, 1}, + {"test_instancenorm_example", 3, 1}, + {"test_isinf", 1, 1}, + {"test_isinf_negative", 1, 1}, + {"test_isinf_positive", 1, 1}, + {"test_isnan", 1, 1}, + {"test_leakyrelu", 1, 1}, + {"test_leakyrelu_default", 1, 1}, + {"test_leakyrelu_example", 1, 1}, + {"test_less", 2, 1}, + {"test_less_bcast", 2, 1}, + {"test_less_equal", 2, 1}, + {"test_less_equal_bcast", 2, 1}, + {"test_less_equal_bcast_expanded", 2, 1}, + {"test_less_equal_expanded", 2, 1}, + {"test_log", 1, 1}, + {"test_log_example", 1, 1}, + {"test_logsoftmax_axis_0", 1, 1}, + {"test_logsoftmax_axis_0_expanded", 1, 1}, + {"test_logsoftmax_axis_1", 1, 1}, + {"test_logsoftmax_axis_1_expanded", 1, 1}, + {"test_logsoftmax_axis_2", 1, 1}, + {"test_logsoftmax_axis_2_expanded", 1, 1}, + {"test_logsoftmax_default_axis", 1, 1}, + {"test_logsoftmax_default_axis_expanded", 1, 1}, + {"test_logsoftmax_example_1", 1, 1}, + {"test_logsoftmax_example_1_expanded", 1, 1}, + {"test_logsoftmax_large_number", 1, 1}, + {"test_logsoftmax_large_number_expanded", 1, 1}, + {"test_logsoftmax_negative_axis", 1, 1}, + {"test_logsoftmax_negative_axis_expanded", 1, 1}, + {"test_loop11", 3, 2}, + {"test_loop13_seq", 3, 1}, + {"test_loop16_seq_none", 3, 1}, + {"test_lrn", 1, 1}, + {"test_lrn_default", 1, 1}, + {"test_lstm_batchwise", 3, 2}, + {"test_lstm_defaults", 3, 1}, + {"test_lstm_with_initial_bias", 4, 1}, + {"test_lstm_with_peepholes", 8, 1}, + {"test_matmul_2d", 2, 1}, + {"test_matmul_3d", 2, 1}, + {"test_matmul_4d", 2, 1}, + {"test_matmulinteger", 4, 1}, + {"test_max_example", 3, 1}, + {"test_max_float16", 2, 1}, + {"test_max_float32", 2, 1}, + {"test_max_float64", 2, 1}, + {"test_max_int16", 2, 1}, + {"test_max_int32", 2, 1}, + {"test_max_int64", 2, 1}, + {"test_max_int8", 2, 1}, + {"test_max_one_input", 1, 1}, + {"test_max_two_inputs", 2, 1}, + {"test_max_uint16", 2, 1}, + {"test_max_uint32", 2, 1}, + {"test_max_uint64", 2, 1}, + {"test_max_uint8", 2, 1}, + {"test_maxpool_1d_default", 1, 1}, + {"test_maxpool_2d_ceil", 1, 1}, + {"test_maxpool_2d_default", 1, 1}, + {"test_maxpool_2d_dilations", 1, 1}, + {"test_maxpool_2d_pads", 1, 1}, + {"test_maxpool_2d_precomputed_pads", 1, 1}, + {"test_maxpool_2d_precomputed_same_upper", 1, 1}, + {"test_maxpool_2d_precomputed_strides", 1, 1}, + {"test_maxpool_2d_same_lower", 1, 1}, + {"test_maxpool_2d_same_upper", 1, 1}, + {"test_maxpool_2d_strides", 1, 1}, + {"test_maxpool_2d_uint8", 1, 1}, + {"test_maxpool_3d_default", 1, 1}, + {"test_maxpool_with_argmax_2d_precomputed_pads", 1, 2}, + {"test_maxpool_with_argmax_2d_precomputed_strides", 1, 2}, + {"test_maxunpool_export_with_output_shape", 3, 1}, + {"test_maxunpool_export_without_output_shape", 2, 1}, + {"test_mean_example", 3, 1}, + {"test_mean_one_input", 1, 1}, + {"test_mean_two_inputs", 2, 1}, + {"test_min_example", 3, 1}, + {"test_min_float16", 2, 1}, + {"test_min_float32", 2, 1}, + {"test_min_float64", 2, 1}, + {"test_min_int16", 2, 1}, + {"test_min_int32", 2, 1}, + {"test_min_int64", 2, 1}, + {"test_min_int8", 2, 1}, + {"test_min_one_input", 1, 1}, + {"test_min_two_inputs", 2, 1}, + {"test_min_uint16", 2, 1}, + {"test_min_uint32", 2, 1}, + {"test_min_uint64", 2, 1}, + {"test_min_uint8", 2, 1}, + {"test_mod_broadcast", 2, 1}, + {"test_mod_int64_fmod", 2, 1}, + {"test_mod_mixed_sign_float16", 2, 1}, + {"test_mod_mixed_sign_float32", 2, 1}, + {"test_mod_mixed_sign_float64", 2, 1}, + {"test_mod_mixed_sign_int16", 2, 1}, + {"test_mod_mixed_sign_int32", 2, 1}, + {"test_mod_mixed_sign_int64", 2, 1}, + {"test_mod_mixed_sign_int8", 2, 1}, + {"test_mod_uint16", 2, 1}, + {"test_mod_uint32", 2, 1}, + {"test_mod_uint64", 2, 1}, + {"test_mod_uint8", 2, 1}, + {"test_momentum", 5, 2}, + {"test_momentum_multiple", 8, 4}, + {"test_mul", 2, 1}, + {"test_mul_bcast", 2, 1}, + {"test_mul_example", 2, 1}, + {"test_mul_uint8", 2, 1}, + {"test_mvn", 1, 1}, + {"test_mvn_expanded", 1, 1}, + {"test_neg", 1, 1}, + {"test_neg_example", 1, 1}, + {"test_nesterov_momentum", 5, 2}, + {"test_nllloss_NC", 2, 1}, + {"test_nllloss_NC_expanded", 2, 1}, + {"test_nllloss_NCd1", 2, 1}, + {"test_nllloss_NCd1_expanded", 2, 1}, + {"test_nllloss_NCd1_ii", 2, 1}, + {"test_nllloss_NCd1_ii_expanded", 2, 1}, + {"test_nllloss_NCd1_mean_weight_negative_ii", 3, 1}, + {"test_nllloss_NCd1_mean_weight_negative_ii_expanded", 3, 1}, + {"test_nllloss_NCd1_weight", 3, 1}, + {"test_nllloss_NCd1_weight_expanded", 3, 1}, + {"test_nllloss_NCd1_weight_ii", 3, 1}, + {"test_nllloss_NCd1_weight_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2", 2, 1}, + {"test_nllloss_NCd1d2_expanded", 2, 1}, + {"test_nllloss_NCd1d2_no_weight_reduction_mean_ii", 2, 1}, + {"test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", 2, 1}, + {"test_nllloss_NCd1d2_reduction_mean", 2, 1}, + {"test_nllloss_NCd1d2_reduction_mean_expanded", 2, 1}, + {"test_nllloss_NCd1d2_reduction_sum", 2, 1}, + {"test_nllloss_NCd1d2_reduction_sum_expanded", 2, 1}, + {"test_nllloss_NCd1d2_with_weight", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_mean", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_ii", 3, 1}, + {"test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3_none_no_weight_negative_ii", 2, 1}, + {"test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", 2, 1}, + {"test_nllloss_NCd1d2d3_sum_weight_high_ii", 3, 1}, + {"test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_mean_weight", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", 3, 1}, + {"test_nllloss_NCd1d2d3d4d5_none_no_weight", 2, 1}, + {"test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", 2, 1}, + {"test_nonmaxsuppression_center_point_box_format", 5, 1}, + {"test_nonmaxsuppression_flipped_coordinates", 5, 1}, + {"test_nonmaxsuppression_identical_boxes", 5, 1}, + {"test_nonmaxsuppression_limit_output_size", 5, 1}, + {"test_nonmaxsuppression_single_box", 5, 1}, + {"test_nonmaxsuppression_suppress_by_IOU", 5, 1}, + {"test_nonmaxsuppression_suppress_by_IOU_and_scores", 5, 1}, + {"test_nonmaxsuppression_two_batches", 5, 1}, + {"test_nonmaxsuppression_two_classes", 5, 1}, + {"test_nonzero_example", 1, 1}, + {"test_not_2d", 1, 1}, + {"test_not_3d", 1, 1}, + {"test_not_4d", 1, 1}, + {"test_onehot_negative_indices", 3, 1}, + {"test_onehot_with_axis", 3, 1}, + {"test_onehot_with_negative_axis", 3, 1}, + {"test_onehot_without_axis", 3, 1}, + {"test_optional_get_element", 1, 1}, + {"test_optional_get_element_sequence", 1, 1}, + {"test_optional_has_element", 1, 1}, + {"test_optional_has_element_empty", 1, 1}, + {"test_or2d", 2, 1}, + {"test_or3d", 2, 1}, + {"test_or4d", 2, 1}, + {"test_or_bcast3v1d", 2, 1}, + {"test_or_bcast3v2d", 2, 1}, + {"test_or_bcast4v2d", 2, 1}, + {"test_or_bcast4v3d", 2, 1}, + {"test_or_bcast4v4d", 2, 1}, + {"test_pow", 2, 1}, + {"test_pow_bcast_array", 2, 1}, + {"test_pow_bcast_scalar", 2, 1}, + {"test_pow_example", 2, 1}, + {"test_pow_types_float", 2, 1}, + {"test_pow_types_float32_int32", 2, 1}, + {"test_pow_types_float32_int64", 2, 1}, + {"test_pow_types_float32_uint32", 2, 1}, + {"test_pow_types_float32_uint64", 2, 1}, + {"test_pow_types_int", 2, 1}, + {"test_pow_types_int32_float32", 2, 1}, + {"test_pow_types_int32_int32", 2, 1}, + {"test_pow_types_int64_float32", 2, 1}, + {"test_pow_types_int64_int64", 2, 1}, + {"test_prelu_broadcast", 2, 1}, + {"test_prelu_example", 2, 1}, + {"test_qlinearconv", 8, 1}, + {"test_qlinearmatmul_2D", 8, 1}, + {"test_qlinearmatmul_3D", 8, 1}, + {"test_quantizelinear", 3, 1}, + {"test_quantizelinear_axis", 3, 1}, + {"test_range_float_type_positive_delta", 3, 1}, + {"test_range_float_type_positive_delta_expanded", 3, 1}, + {"test_range_int32_type_negative_delta", 3, 1}, + {"test_range_int32_type_negative_delta_expanded", 3, 1}, + {"test_reciprocal", 1, 1}, + {"test_reciprocal_example", 1, 1}, + {"test_reduce_l1_default_axes_keepdims_example", 1, 1}, + {"test_reduce_l1_default_axes_keepdims_random", 1, 1}, + {"test_reduce_l1_do_not_keepdims_example", 1, 1}, + {"test_reduce_l1_do_not_keepdims_random", 1, 1}, + {"test_reduce_l1_keep_dims_example", 1, 1}, + {"test_reduce_l1_keep_dims_random", 1, 1}, + {"test_reduce_l1_negative_axes_keep_dims_example", 1, 1}, + {"test_reduce_l1_negative_axes_keep_dims_random", 1, 1}, + {"test_reduce_l2_default_axes_keepdims_example", 1, 1}, + {"test_reduce_l2_default_axes_keepdims_random", 1, 1}, + {"test_reduce_l2_do_not_keepdims_example", 1, 1}, + {"test_reduce_l2_do_not_keepdims_random", 1, 1}, + {"test_reduce_l2_keep_dims_example", 1, 1}, + {"test_reduce_l2_keep_dims_random", 1, 1}, + {"test_reduce_l2_negative_axes_keep_dims_example", 1, 1}, + {"test_reduce_l2_negative_axes_keep_dims_random", 1, 1}, + {"test_reduce_log_sum", 1, 1}, + {"test_reduce_log_sum_asc_axes", 1, 1}, + {"test_reduce_log_sum_default", 1, 1}, + {"test_reduce_log_sum_desc_axes", 1, 1}, + {"test_reduce_log_sum_exp_default_axes_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_default_axes_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_do_not_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_do_not_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_keepdims_random", 1, 1}, + {"test_reduce_log_sum_exp_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_log_sum_exp_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_log_sum_negative_axes", 1, 1}, + {"test_reduce_max_default_axes_keepdim_example", 1, 1}, + {"test_reduce_max_default_axes_keepdims_random", 1, 1}, + {"test_reduce_max_do_not_keepdims_example", 1, 1}, + {"test_reduce_max_do_not_keepdims_random", 1, 1}, + {"test_reduce_max_keepdims_example", 1, 1}, + {"test_reduce_max_keepdims_random", 1, 1}, + {"test_reduce_max_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_max_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_mean_default_axes_keepdims_example", 1, 1}, + {"test_reduce_mean_default_axes_keepdims_random", 1, 1}, + {"test_reduce_mean_do_not_keepdims_example", 1, 1}, + {"test_reduce_mean_do_not_keepdims_random", 1, 1}, + {"test_reduce_mean_keepdims_example", 1, 1}, + {"test_reduce_mean_keepdims_random", 1, 1}, + {"test_reduce_mean_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_mean_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_min_default_axes_keepdims_example", 1, 1}, + {"test_reduce_min_default_axes_keepdims_random", 1, 1}, + {"test_reduce_min_do_not_keepdims_example", 1, 1}, + {"test_reduce_min_do_not_keepdims_random", 1, 1}, + {"test_reduce_min_keepdims_example", 1, 1}, + {"test_reduce_min_keepdims_random", 1, 1}, + {"test_reduce_min_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_min_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_prod_default_axes_keepdims_example", 1, 1}, + {"test_reduce_prod_default_axes_keepdims_random", 1, 1}, + {"test_reduce_prod_do_not_keepdims_example", 1, 1}, + {"test_reduce_prod_do_not_keepdims_random", 1, 1}, + {"test_reduce_prod_keepdims_example", 1, 1}, + {"test_reduce_prod_keepdims_random", 1, 1}, + {"test_reduce_prod_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_prod_negative_axes_keepdims_random", 1, 1}, + {"test_reduce_sum_default_axes_keepdims_example", 2, 1}, + {"test_reduce_sum_default_axes_keepdims_random", 2, 1}, + {"test_reduce_sum_do_not_keepdims_example", 2, 1}, + {"test_reduce_sum_do_not_keepdims_random", 2, 1}, + {"test_reduce_sum_empty_axes_input_noop_example", 2, 1}, + {"test_reduce_sum_empty_axes_input_noop_random", 2, 1}, + {"test_reduce_sum_keepdims_example", 2, 1}, + {"test_reduce_sum_keepdims_random", 2, 1}, + {"test_reduce_sum_negative_axes_keepdims_example", 2, 1}, + {"test_reduce_sum_negative_axes_keepdims_random", 2, 1}, + {"test_reduce_sum_square_default_axes_keepdims_example", 1, 1}, + {"test_reduce_sum_square_default_axes_keepdims_random", 1, 1}, + {"test_reduce_sum_square_do_not_keepdims_example", 1, 1}, + {"test_reduce_sum_square_do_not_keepdims_random", 1, 1}, + {"test_reduce_sum_square_keepdims_example", 1, 1}, + {"test_reduce_sum_square_keepdims_random", 1, 1}, + {"test_reduce_sum_square_negative_axes_keepdims_example", 1, 1}, + {"test_reduce_sum_square_negative_axes_keepdims_random", 1, 1}, + {"test_reflect_pad", 2, 1}, + {"test_relu", 1, 1}, + {"test_reshape_allowzero_reordered", 2, 1}, + {"test_reshape_extended_dims", 2, 1}, + {"test_reshape_negative_dim", 2, 1}, + {"test_reshape_negative_extended_dims", 2, 1}, + {"test_reshape_one_dim", 2, 1}, + {"test_reshape_reduced_dims", 2, 1}, + {"test_reshape_reordered_all_dims", 2, 1}, + {"test_reshape_reordered_last_dims", 2, 1}, + {"test_reshape_zero_and_negative_dim", 2, 1}, + {"test_reshape_zero_dim", 2, 1}, + {"test_resize_downsample_scales_cubic", 2, 1}, + {"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", 2, 1}, + {"test_resize_downsample_scales_cubic_align_corners", 2, 1}, + {"test_resize_downsample_scales_linear", 2, 1}, + {"test_resize_downsample_scales_linear_align_corners", 2, 1}, + {"test_resize_downsample_scales_nearest", 2, 1}, + {"test_resize_downsample_sizes_cubic", 2, 1}, + {"test_resize_downsample_sizes_linear_pytorch_half_pixel", 2, 1}, + {"test_resize_downsample_sizes_nearest", 2, 1}, + {"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", 2, 1}, + {"test_resize_tf_crop_and_resize", 3, 1}, + {"test_resize_upsample_scales_cubic", 2, 1}, + {"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", 2, 1}, + {"test_resize_upsample_scales_cubic_align_corners", 2, 1}, + {"test_resize_upsample_scales_cubic_asymmetric", 2, 1}, + {"test_resize_upsample_scales_linear", 2, 1}, + {"test_resize_upsample_scales_linear_align_corners", 2, 1}, + {"test_resize_upsample_scales_nearest", 2, 1}, + {"test_resize_upsample_sizes_cubic", 2, 1}, + {"test_resize_upsample_sizes_nearest", 2, 1}, + {"test_resize_upsample_sizes_nearest_ceil_half_pixel", 2, 1}, + {"test_resize_upsample_sizes_nearest_floor_align_corners", 2, 1}, + {"test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", 2, 1}, + {"test_reversesequence_batch", 2, 1}, + {"test_reversesequence_time", 2, 1}, + {"test_rnn_seq_length", 4, 1}, + {"test_roialign_aligned_false", 3, 1}, + {"test_roialign_aligned_true", 3, 1}, + {"test_round", 1, 1}, + {"test_scan9_sum", 2, 2}, + {"test_scan_sum", 2, 2}, + {"test_scatter_elements_with_axis", 3, 1}, + {"test_scatter_elements_with_duplicate_indices", 3, 1}, + {"test_scatter_elements_with_negative_indices", 3, 1}, + {"test_scatter_elements_without_axis", 3, 1}, + {"test_scatter_with_axis", 3, 1}, + {"test_scatter_without_axis", 3, 1}, + {"test_scatternd", 3, 1}, + {"test_scatternd_add", 3, 1}, + {"test_scatternd_multiply", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii_expanded", 3, 1}, + {"test_sce_NCd1_mean_weight_negative_ii_log_prob", 3, 2}, + {"test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii", 2, 1}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", 2, 1}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", 2, 2}, + {"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", 2, 2}, + {"test_sce_NCd1d2d3_sum_weight_high_ii", 3, 1}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_expanded", 3, 1}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", 3, 2}, + {"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3d4d5_mean_weight", 3, 1}, + {"test_sce_NCd1d2d3d4d5_mean_weight_expanded", 3, 1}, + {"test_sce_NCd1d2d3d4d5_mean_weight_log_prob", 3, 2}, + {"test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", 3, 2}, + {"test_sce_NCd1d2d3d4d5_none_no_weight", 2, 1}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_expanded", 2, 1}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", 2, 2}, + {"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", 2, 2}, + {"test_sce_mean", 2, 1}, + {"test_sce_mean_3d", 2, 1}, + {"test_sce_mean_3d_expanded", 2, 1}, + {"test_sce_mean_3d_log_prob", 2, 2}, + {"test_sce_mean_3d_log_prob_expanded", 2, 2}, + {"test_sce_mean_expanded", 2, 1}, + {"test_sce_mean_log_prob", 2, 2}, + {"test_sce_mean_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii", 2, 1}, + {"test_sce_mean_no_weight_ii_3d", 2, 1}, + {"test_sce_mean_no_weight_ii_3d_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_3d_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_3d_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii_4d", 2, 1}, + {"test_sce_mean_no_weight_ii_4d_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_4d_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_4d_log_prob_expanded", 2, 2}, + {"test_sce_mean_no_weight_ii_expanded", 2, 1}, + {"test_sce_mean_no_weight_ii_log_prob", 2, 2}, + {"test_sce_mean_no_weight_ii_log_prob_expanded", 2, 2}, + {"test_sce_mean_weight", 3, 1}, + {"test_sce_mean_weight_expanded", 3, 1}, + {"test_sce_mean_weight_ii", 3, 1}, + {"test_sce_mean_weight_ii_3d", 3, 1}, + {"test_sce_mean_weight_ii_3d_expanded", 3, 1}, + {"test_sce_mean_weight_ii_3d_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_3d_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_ii_4d", 3, 1}, + {"test_sce_mean_weight_ii_4d_expanded", 3, 1}, + {"test_sce_mean_weight_ii_4d_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_4d_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_ii_expanded", 3, 1}, + {"test_sce_mean_weight_ii_log_prob", 3, 2}, + {"test_sce_mean_weight_ii_log_prob_expanded", 3, 2}, + {"test_sce_mean_weight_log_prob", 3, 2}, + {"test_sce_mean_weight_log_prob_expanded", 3, 2}, + {"test_sce_none", 2, 1}, + {"test_sce_none_expanded", 2, 1}, + {"test_sce_none_log_prob", 2, 2}, + {"test_sce_none_log_prob_expanded", 2, 2}, + {"test_sce_none_weights", 3, 1}, + {"test_sce_none_weights_expanded", 3, 1}, + {"test_sce_none_weights_log_prob", 3, 2}, + {"test_sce_none_weights_log_prob_expanded", 3, 2}, + {"test_sce_sum", 2, 1}, + {"test_sce_sum_expanded", 2, 1}, + {"test_sce_sum_log_prob", 2, 2}, + {"test_sce_sum_log_prob_expanded", 2, 2}, + {"test_selu", 1, 1}, + {"test_selu_default", 1, 1}, + {"test_selu_example", 1, 1}, + {"test_sequence_insert_at_back", 2, 1}, + {"test_sequence_insert_at_front", 3, 1}, + {"test_shape", 1, 1}, + {"test_shape_clip_end", 1, 1}, + {"test_shape_clip_start", 1, 1}, + {"test_shape_end_1", 1, 1}, + {"test_shape_end_negative_1", 1, 1}, + {"test_shape_example", 1, 1}, + {"test_shape_start_1", 1, 1}, + {"test_shape_start_1_end_2", 1, 1}, + {"test_shape_start_1_end_negative_1", 1, 1}, + {"test_shape_start_negative_1", 1, 1}, + {"test_shrink_hard", 1, 1}, + {"test_shrink_soft", 1, 1}, + {"test_sigmoid", 1, 1}, + {"test_sigmoid_example", 1, 1}, + {"test_sign", 1, 1}, + {"test_simple_rnn_batchwise", 3, 2}, + {"test_simple_rnn_defaults", 3, 1}, + {"test_simple_rnn_with_initial_bias", 4, 1}, + {"test_sin", 1, 1}, + {"test_sin_example", 1, 1}, + {"test_sinh", 1, 1}, + {"test_sinh_example", 1, 1}, + {"test_size", 1, 1}, + {"test_size_example", 1, 1}, + {"test_slice", 5, 1}, + {"test_slice_default_axes", 3, 1}, + {"test_slice_default_steps", 4, 1}, + {"test_slice_end_out_of_bounds", 5, 1}, + {"test_slice_neg", 5, 1}, + {"test_slice_neg_steps", 5, 1}, + {"test_slice_negative_axes", 4, 1}, + {"test_slice_start_out_of_bounds", 5, 1}, + {"test_softmax_axis_0", 1, 1}, + {"test_softmax_axis_0_expanded", 1, 1}, + {"test_softmax_axis_1", 1, 1}, + {"test_softmax_axis_1_expanded", 1, 1}, + {"test_softmax_axis_2", 1, 1}, + {"test_softmax_axis_2_expanded", 1, 1}, + {"test_softmax_default_axis", 1, 1}, + {"test_softmax_default_axis_expanded", 1, 1}, + {"test_softmax_example", 1, 1}, + {"test_softmax_example_expanded", 1, 1}, + {"test_softmax_large_number", 1, 1}, + {"test_softmax_large_number_expanded", 1, 1}, + {"test_softmax_negative_axis", 1, 1}, + {"test_softmax_negative_axis_expanded", 1, 1}, + {"test_softplus", 1, 1}, + {"test_softplus_example", 1, 1}, + {"test_softsign", 1, 1}, + {"test_softsign_example", 1, 1}, + {"test_spacetodepth", 1, 1}, + {"test_spacetodepth_example", 1, 1}, + {"test_split_equal_parts_1d", 1, 3}, + {"test_split_equal_parts_2d", 1, 2}, + {"test_split_equal_parts_default_axis", 1, 3}, + {"test_split_variable_parts_1d", 2, 2}, + {"test_split_variable_parts_2d", 2, 2}, + {"test_split_variable_parts_default_axis", 2, 2}, + {"test_split_zero_size_splits", 2, 3}, + {"test_sqrt", 1, 1}, + {"test_sqrt_example", 1, 1}, + {"test_squeeze", 2, 1}, + {"test_squeeze_negative_axes", 2, 1}, + {"test_strnormalizer_export_monday_casesensintive_lower", 1, 1}, + {"test_strnormalizer_export_monday_casesensintive_nochangecase", 1, 1}, + {"test_strnormalizer_export_monday_casesensintive_upper", 1, 1}, + {"test_strnormalizer_export_monday_empty_output", 1, 1}, + {"test_strnormalizer_export_monday_insensintive_upper_twodim", 1, 1}, + {"test_strnormalizer_nostopwords_nochangecase", 1, 1}, + {"test_sub", 2, 1}, + {"test_sub_bcast", 2, 1}, + {"test_sub_example", 2, 1}, + {"test_sub_uint8", 2, 1}, + {"test_sum_example", 3, 1}, + {"test_sum_one_input", 1, 1}, + {"test_sum_two_inputs", 2, 1}, + {"test_tan", 1, 1}, + {"test_tan_example", 1, 1}, + {"test_tanh", 1, 1}, + {"test_tanh_example", 1, 1}, + {"test_tfidfvectorizer_tf_batch_onlybigrams_skip0", 1, 1}, + {"test_tfidfvectorizer_tf_batch_onlybigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_only_bigrams_skip0", 1, 1}, + {"test_tfidfvectorizer_tf_onlybigrams_levelempty", 1, 1}, + {"test_tfidfvectorizer_tf_onlybigrams_skip5", 1, 1}, + {"test_tfidfvectorizer_tf_uniandbigrams_skip5", 1, 1}, + {"test_thresholdedrelu", 1, 1}, + {"test_thresholdedrelu_default", 1, 1}, + {"test_thresholdedrelu_example", 1, 1}, + {"test_tile", 2, 1}, + {"test_tile_precomputed", 2, 1}, + {"test_top_k", 2, 2}, + {"test_top_k_negative_axis", 2, 2}, + {"test_top_k_smallest", 2, 2}, + {"test_training_dropout", 3, 1}, + {"test_training_dropout_default", 3, 1}, + {"test_training_dropout_default_mask", 3, 2}, + {"test_training_dropout_mask", 3, 2}, + {"test_training_dropout_zero_ratio", 3, 1}, + {"test_training_dropout_zero_ratio_mask", 3, 2}, + {"test_transpose_all_permutations_0", 1, 1}, + {"test_transpose_all_permutations_1", 1, 1}, + {"test_transpose_all_permutations_2", 1, 1}, + {"test_transpose_all_permutations_3", 1, 1}, + {"test_transpose_all_permutations_4", 1, 1}, + {"test_transpose_all_permutations_5", 1, 1}, + {"test_transpose_default", 1, 1}, + {"test_tril", 1, 1}, + {"test_tril_neg", 2, 1}, + {"test_tril_one_row_neg", 1, 1}, + {"test_tril_out_neg", 2, 1}, + {"test_tril_out_pos", 2, 1}, + {"test_tril_pos", 2, 1}, + {"test_tril_square", 1, 1}, + {"test_tril_square_neg", 2, 1}, + {"test_tril_zero", 2, 1}, + {"test_triu", 1, 1}, + {"test_triu_neg", 2, 1}, + {"test_triu_one_row", 2, 1}, + {"test_triu_out_neg_out", 2, 1}, + {"test_triu_out_pos", 2, 1}, + {"test_triu_pos", 2, 1}, + {"test_triu_square", 1, 1}, + {"test_triu_square_neg", 2, 1}, + {"test_triu_zero", 2, 1}, + {"test_unique_not_sorted_without_axis", 1, 4}, + {"test_unique_sorted_with_axis", 1, 4}, + {"test_unique_sorted_with_axis_3d", 1, 4}, + {"test_unique_sorted_with_negative_axis", 1, 4}, + {"test_unique_sorted_without_axis", 1, 4}, + {"test_unsqueeze_axis_0", 2, 1}, + {"test_unsqueeze_axis_1", 2, 1}, + {"test_unsqueeze_axis_2", 2, 1}, + {"test_unsqueeze_axis_3", 1, 1}, + {"test_unsqueeze_negative_axes", 2, 1}, + {"test_unsqueeze_three_axes", 2, 1}, + {"test_unsqueeze_two_axes", 2, 1}, + {"test_unsqueeze_unsorted_axes", 2, 1}, + {"test_upsample_nearest", 2, 1}, + {"test_where_example", 3, 1}, + {"test_where_long_example", 3, 1}, + {"test_xor2d", 2, 1}, + {"test_xor3d", 2, 1}, + {"test_xor4d", 2, 1}, + {"test_xor_bcast3v1d", 2, 1}, + {"test_xor_bcast3v2d", 2, 1}, + {"test_xor_bcast4v2d", 2, 1}, + {"test_xor_bcast4v3d", 2, 1}, + {"test_xor_bcast4v4d", 2, 1}, +}; + + +struct TestCaseInput +{ + std::vector input_paths; + std::vector output_paths; + std::string model_path; + std::string name; +}; + +std::ostream& operator<<(std::ostream& os, const TestCaseInput& test_case) +{ + return os << test_case.name; +} + +typedef tuple > ONNXConfParams; + +std::string printOnnxConfParams(const testing::TestParamInfo& params) +{ + TestCaseInput test_case = get<0>(params.param); + Backend backend = get<0>(get<1>(params.param)); + Target target = get<1>(get<1>(params.param)); + + std::stringstream ss; + ss << test_case.name << "_"; + PrintTo(backend, &ss); + ss << "_"; + PrintTo(target, &ss); + + return ss.str(); +} + +template +static std::string _tf(TString filename, bool required = true) +{ + return findDataFile(std::string("dnn/onnx/") + filename, required); +} + +std::vector readTestCases() +{ + std::vector ret; + for (size_t i = 0; i < sizeof(testConformanceConfig) / sizeof(testConformanceConfig[0]); ++i) + { + const TestCase& test_case = testConformanceConfig[i]; + + TestCaseInput input; + + std::string prefix = cv::format("conformance/node/%s", test_case.name); + input.name = test_case.name; + input.model_path = _tf(cv::format("%s/model.onnx", prefix.c_str())); + + for (int i = 0; i < test_case.inputs; ++i) + { + input.input_paths.push_back(_tf(cv::format("%s/test_data_set_0/input_%d.pb", prefix.c_str(), i))); + } + + for (int i = 0; i < test_case.outputs; ++i) + { + input.output_paths.push_back(_tf(cv::format("%s/test_data_set_0/output_%d.pb", prefix.c_str(), i))); + } + + ret.push_back(input); + } + + return ret; +} + +class Test_ONNX_conformance : public TestWithParam +{ +public: + TestCaseInput test_case; + Backend backend; + Target target; + + double default_l1; + double default_lInf; + + static std::set parser_deny_list; + static std::set global_deny_list; + static std::set opencl_fp16_deny_list; + static std::set opencl_deny_list; + static std::set cpu_deny_list; + + Test_ONNX_conformance() + { + test_case = get<0>(GetParam()); + backend = get<0>(get<1>(GetParam())); + target = get<1>(get<1>(GetParam())); + + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) + { + default_l1 = 4e-3; + default_lInf = 2e-2; + } + else + { + default_l1 = 1e-5; + default_lInf = 1e-4; + } + } + + bool checkFallbacks(Net& net) const + { + // Check if all the layers are supported with current backend and target. + // Some layers might be fused so their timings equal to zero. + std::vector timings; + net.getPerfProfile(timings); + std::vector names = net.getLayerNames(); + CV_CheckEQ(names.size(), timings.size(), "DNN critical error"); + + bool hasFallbacks = false; + for (int i = 0; i < names.size(); ++i) + { + Ptr l = net.getLayer(net.getLayerId(names[i])); + bool fused = timings[i] == 0.; + if ((!l->supportBackend(backend) || l->preferableTarget != target) && !fused) + { + hasFallbacks = true; + std::cout << "FALLBACK: Layer [" << l->type << "]:[" << l->name << "] is expected to has backend implementation" << endl; + } + } + return hasFallbacks; + } + + static void initDenyList(std::set& deny_set, const char* const deny_list[], const size_t n) + { + for (size_t i = 0; i < n; ++i) + { + deny_set.insert(deny_list[i]); + } + } + + static void SetUpTestCase() + { + const char* const parser[] = { + #include "test_onnx_conformance_layer_parser_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(parser_deny_list, parser, sizeof(parser)/sizeof(parser[0])); + + const char* const global[] = { + #include "test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(global_deny_list, global, sizeof(global)/sizeof(global[0])); + + const char* const opencl_fp16[] = { + #include "test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(opencl_fp16_deny_list, opencl_fp16, sizeof(opencl_fp16)/sizeof(opencl_fp16[0])); + + const char* const opencl[] = { + #include "test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(opencl_deny_list, opencl, sizeof(opencl)/sizeof(opencl[0])); + + const char* const cpu[] = { + #include "test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(cpu_deny_list, cpu, sizeof(cpu)/sizeof(cpu[0])); + } + + void checkFilterLists() const + { + const std::string& name = test_case.name; + if(parser_deny_list.find(name) != parser_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + + if (backend == DNN_BACKEND_OPENCV) + { + if(global_deny_list.find(name) != global_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#if 0 //def HAVE_HALIDE + else if (backend == DNN_BACKEND_HALIDE) + { + #include "test_onnx_conformance_layer_filter__halide.inl.hpp" + } +#endif +#if 0 //def HAVE_INF_ENGINE + else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + #include "test_onnx_conformance_layer_filter__ngraph.inl.hpp" + } +#endif +#if 0 //def HAVE_VULKAN + else if (backend == DNN_BACKEND_VKCOM) + { + #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" + } +#endif +#if 0 //def HAVE_CUDA + else if (backend == DNN_BACKEND_CUDA) + { + #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" + } +#endif + else + { + std::ostringstream ss; + ss << "No test filter available for backend "; + PrintTo(backend, &ss); + ss << ". Run test by default"; + std::cout << ss.str() << std::endl; + } + } +}; + +std::set Test_ONNX_conformance::parser_deny_list; +std::set Test_ONNX_conformance::global_deny_list; +std::set Test_ONNX_conformance::opencl_fp16_deny_list; +std::set Test_ONNX_conformance::opencl_deny_list; +std::set Test_ONNX_conformance::cpu_deny_list; + +TEST_P(Test_ONNX_conformance, Layer_Test) +{ + std::string name = test_case.name; + ASSERT_FALSE(name.empty()); + + bool checkLayersFallbacks = true; + bool checkAccuracy = true; + + checkFilterLists(); + + std::vector inputs; + std::vector ref_outputs; + + Net net; + try + { + //cout << "Read ONNX inputs..." << endl; + std::transform(test_case.input_paths.begin(), test_case.input_paths.end(), + std::back_inserter(inputs), readTensorFromONNX); + + //cout << "Read ONNX reference outputs..." << endl; + std::transform(test_case.output_paths.begin(), test_case.output_paths.end(), + std::back_inserter(ref_outputs), readTensorFromONNX); + + //cout << "Parse model..." << endl; + net = readNetFromONNX(test_case.model_path); + if (net.empty()) + { + applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); + } + } + catch (...) + { + cout << "Exception during ONNX model parse / loading input / loading reference data!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); + throw; + } + ASSERT_FALSE(net.empty()); + + std::vector inputNames; + for (int i = 0; i < inputs.size(); ++i) + inputNames.push_back(cv::format("%d", i)); + net.setInputsNames(inputNames); + + try + { + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + for (int i = 0; i < inputs.size(); ++i) + { + net.setInput(inputs[i], inputNames[i]); + } + } + catch (...) + { + cout << "Exception during network configuration!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_NET_SETUP); + throw; + } + + std::vector layerNames = net.getUnconnectedOutLayersNames(); + std::vector< std::vector > outputs_; + try + { + net.forward(outputs_, layerNames); + } + catch (...) + { + cout << "Exception during net.forward() call!" << endl; + applyTestTag(CV_TEST_TAG_DNN_ERROR_FORWARD); + throw; + } + ASSERT_GE(outputs_.size(), 1); + const std::vector& outputs = outputs_[0]; + + if (checkLayersFallbacks && checkFallbacks(net)) + { + applyTestTag(CV_TEST_TAG_DNN_LAYER_FALLBACK); + } + + if (checkAccuracy) + { + try + { + if (ref_outputs.size() == 1) + { + // probably we found random unconnected layers. + normAssert(ref_outputs[0], outputs[0], "", default_l1, default_lInf); + } + else + { + ASSERT_EQ(outputs.size(), ref_outputs.size()); + for (size_t i = 0; i < ref_outputs.size(); ++i) + { + normAssert(ref_outputs[i], outputs[i], "", default_l1, default_lInf); + } + } + } + catch (...) + { + cout << "Exception during accuracy check!" << endl; + throw; + } + } + else + { + applyTestTag(CV_TEST_TAG_DNN_NO_ACCURACY_CHECK); + } + + if (!HasFailure()) + cout << "Test passed!" << endl; +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance, + testing::Combine(testing::ValuesIn(readTestCases()), dnnBackendsAndTargets()), + printOnnxConfParams); + +}; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp new file mode 100644 index 0000000000..cff1e93aa0 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp @@ -0,0 +1,23 @@ +"test_add_bcast", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_same_lower", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_logsoftmax_default_axis", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_softmax_default_axis", +"test_sub_bcast", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_cpu_denylist.inl.hpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp new file mode 100644 index 0000000000..573d847985 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp16_denylist.inl.hpp @@ -0,0 +1,21 @@ +"test_averagepool_3d_default", +"test_dropout_default_ratio", +"test_globalmaxpool", +"test_globalmaxpool_precomputed", +"test_logsoftmax_large_number", +"test_logsoftmax_large_number_expanded", +"test_maxpool_1d_default", +"test_maxpool_2d_ceil", +"test_maxpool_2d_default", +"test_maxpool_2d_pads", +"test_maxpool_2d_precomputed_pads", +"test_maxpool_2d_precomputed_same_upper", +"test_maxpool_2d_precomputed_strides", +"test_maxpool_2d_same_upper", +"test_maxpool_2d_strides", +"test_maxpool_3d_default", +"test_softmax_large_number", +"test_softmax_large_number_expanded", +"test_split_equal_parts_1d", +"test_split_equal_parts_2d", +"test_split_equal_parts_default_axis", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp new file mode 100644 index 0000000000..9a7a21f393 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_ocl_fp32_denylist.inl.hpp @@ -0,0 +1,2 @@ +"test_averagepool_3d_default", +"test_maxpool_3d_default", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp new file mode 100644 index 0000000000..5f1d99d4bb --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -0,0 +1,717 @@ +// The file is autogenerated +// Update note: execute /testdata/dnn/onnx/generate_conformance_list.py +"test_abs", +"test_acos", +"test_acos_example", +"test_acosh", +"test_acosh_example", +"test_adagrad", +"test_adagrad_multiple", +"test_adam", +"test_adam_multiple", +"test_add_uint8", +"test_and2d", +"test_and3d", +"test_and4d", +"test_and_bcast3v1d", +"test_and_bcast3v2d", +"test_and_bcast4v2d", +"test_and_bcast4v3d", +"test_and_bcast4v4d", +"test_argmax_default_axis_example", +"test_argmax_default_axis_example_select_last_index", +"test_argmax_default_axis_random", +"test_argmax_default_axis_random_select_last_index", +"test_argmax_keepdims_example", +"test_argmax_keepdims_example_select_last_index", +"test_argmax_keepdims_random", +"test_argmax_keepdims_random_select_last_index", +"test_argmax_negative_axis_keepdims_example", +"test_argmax_negative_axis_keepdims_example_select_last_index", +"test_argmax_negative_axis_keepdims_random", +"test_argmax_negative_axis_keepdims_random_select_last_index", +"test_argmax_no_keepdims_example", +"test_argmax_no_keepdims_example_select_last_index", +"test_argmax_no_keepdims_random", +"test_argmax_no_keepdims_random_select_last_index", +"test_argmin_default_axis_example", +"test_argmin_default_axis_example_select_last_index", +"test_argmin_default_axis_random", +"test_argmin_default_axis_random_select_last_index", +"test_argmin_keepdims_example", +"test_argmin_keepdims_example_select_last_index", +"test_argmin_keepdims_random", +"test_argmin_keepdims_random_select_last_index", +"test_argmin_negative_axis_keepdims_example", +"test_argmin_negative_axis_keepdims_example_select_last_index", +"test_argmin_negative_axis_keepdims_random", +"test_argmin_negative_axis_keepdims_random_select_last_index", +"test_argmin_no_keepdims_example", +"test_argmin_no_keepdims_example_select_last_index", +"test_argmin_no_keepdims_random", +"test_argmin_no_keepdims_random_select_last_index", +"test_asin", +"test_asin_example", +"test_asinh", +"test_asinh_example", +"test_atan", +"test_atan_example", +"test_atanh", +"test_atanh_example", +"test_basic_convinteger", +"test_batchnorm_epsilon", +"test_batchnorm_epsilon_training_mode", +"test_batchnorm_example", +"test_batchnorm_example_training_mode", +"test_bernoulli", +"test_bernoulli_double", +"test_bernoulli_double_expanded", +"test_bernoulli_expanded", +"test_bernoulli_seed", +"test_bernoulli_seed_expanded", +"test_bitshift_left_uint16", +"test_bitshift_left_uint32", +"test_bitshift_left_uint64", +"test_bitshift_left_uint8", +"test_bitshift_right_uint16", +"test_bitshift_right_uint32", +"test_bitshift_right_uint64", +"test_bitshift_right_uint8", +"test_cast_BFLOAT16_to_FLOAT", +"test_cast_DOUBLE_to_FLOAT", +"test_cast_DOUBLE_to_FLOAT16", +"test_cast_FLOAT16_to_DOUBLE", +"test_cast_FLOAT16_to_FLOAT", +"test_cast_FLOAT_to_BFLOAT16", +"test_cast_FLOAT_to_DOUBLE", +"test_cast_FLOAT_to_FLOAT16", +"test_castlike_BFLOAT16_to_FLOAT", +"test_castlike_BFLOAT16_to_FLOAT_expanded", +"test_castlike_DOUBLE_to_FLOAT", +"test_castlike_DOUBLE_to_FLOAT16", +"test_castlike_DOUBLE_to_FLOAT16_expanded", +"test_castlike_DOUBLE_to_FLOAT_expanded", +"test_castlike_FLOAT16_to_DOUBLE", +"test_castlike_FLOAT16_to_DOUBLE_expanded", +"test_castlike_FLOAT16_to_FLOAT", +"test_castlike_FLOAT16_to_FLOAT_expanded", +"test_castlike_FLOAT_to_BFLOAT16", +"test_castlike_FLOAT_to_BFLOAT16_expanded", +"test_castlike_FLOAT_to_DOUBLE", +"test_castlike_FLOAT_to_DOUBLE_expanded", +"test_castlike_FLOAT_to_FLOAT16", +"test_castlike_FLOAT_to_FLOAT16_expanded", +"test_castlike_FLOAT_to_STRING", +"test_castlike_STRING_to_FLOAT", +"test_ceil", +"test_ceil_example", +"test_celu", +"test_clip", +"test_clip_default_inbounds", +"test_clip_default_int8_inbounds", +"test_clip_default_int8_max", +"test_clip_default_int8_min", +"test_clip_default_max", +"test_clip_default_min", +"test_clip_example", +"test_clip_inbounds", +"test_clip_outbounds", +"test_clip_splitbounds", +"test_compress_0", +"test_compress_1", +"test_compress_default_axis", +"test_compress_negative_axis", +"test_constant", +"test_constant_pad", +"test_constantofshape_float_ones", +"test_constantofshape_int_shape_zero", +"test_constantofshape_int_zeros", +"test_convinteger_with_padding", +"test_convinteger_without_padding", +"test_convtranspose", +"test_convtranspose_1d", +"test_convtranspose_3d", +"test_convtranspose_autopad_same", +"test_convtranspose_dilations", +"test_convtranspose_kernel_shape", +"test_convtranspose_output_shape", +"test_convtranspose_pad", +"test_convtranspose_pads", +"test_convtranspose_with_kernel", +"test_cos", +"test_cos_example", +"test_cosh", +"test_cosh_example", +"test_cumsum_1d", +"test_cumsum_1d_exclusive", +"test_cumsum_1d_reverse", +"test_cumsum_1d_reverse_exclusive", +"test_cumsum_2d_axis_0", +"test_cumsum_2d_axis_1", +"test_cumsum_2d_negative_axis", +"test_depthtospace_crd_mode", +"test_depthtospace_crd_mode_example", +"test_depthtospace_dcr_mode", +"test_depthtospace_example", +"test_dequantizelinear", +"test_dequantizelinear_axis", +"test_det_2d", +"test_det_nd", +"test_div_example", +"test_div_uint8", +"test_dropout_default_mask", +"test_dropout_default_mask_ratio", +"test_dynamicquantizelinear", +"test_dynamicquantizelinear_expanded", +"test_dynamicquantizelinear_max_adjusted", +"test_dynamicquantizelinear_max_adjusted_expanded", +"test_dynamicquantizelinear_min_adjusted", +"test_dynamicquantizelinear_min_adjusted_expanded", +"test_edge_pad", +"test_einsum_batch_diagonal", +"test_einsum_batch_matmul", +"test_einsum_inner_prod", +"test_einsum_sum", +"test_einsum_transpose", +"test_equal", +"test_equal_bcast", +"test_erf", +"test_expand_dim_changed", +"test_expand_dim_unchanged", +"test_eyelike_populate_off_main_diagonal", +"test_eyelike_with_dtype", +"test_eyelike_without_dtype", +"test_floor", +"test_floor_example", +"test_gather_0", +"test_gather_1", +"test_gather_2d_indices", +"test_gather_elements_0", +"test_gather_elements_1", +"test_gather_elements_negative_indices", +"test_gather_negative_indices", +"test_gathernd_example_float32", +"test_gathernd_example_int32", +"test_gathernd_example_int32_batch_dim1", +"test_gemm_all_attributes", +"test_gemm_alpha", +"test_gemm_beta", +"test_gemm_default_matrix_bias", +"test_gemm_default_no_bias", +"test_gemm_default_scalar_bias", +"test_gemm_default_single_elem_vector_bias", +"test_gemm_default_vector_bias", +"test_gemm_default_zero_bias", +"test_gemm_transposeA", +"test_gemm_transposeB", +"test_greater", +"test_greater_bcast", +"test_greater_equal", +"test_greater_equal_bcast", +"test_greater_equal_bcast_expanded", +"test_greater_equal_expanded", +"test_gridsample", +"test_gridsample_aligncorners_true", +"test_gridsample_bicubic", +"test_gridsample_bilinear", +"test_gridsample_border_padding", +"test_gridsample_nearest", +"test_gridsample_reflection_padding", +"test_gridsample_zeros_padding", +"test_gru_batchwise", +"test_gru_defaults", +"test_gru_seq_length", +"test_gru_with_initial_bias", +"test_hardmax_axis_0", +"test_hardmax_axis_1", +"test_hardmax_axis_2", +"test_hardmax_default_axis", +"test_hardmax_example", +"test_hardmax_negative_axis", +"test_hardmax_one_hot", +"test_hardsigmoid", +"test_hardsigmoid_default", +"test_hardsigmoid_example", +"test_hardswish", +"test_hardswish_expanded", +"test_identity_opt", +"test_identity_sequence", +"test_if", +"test_if_opt", +"test_if_seq", +"test_instancenorm_epsilon", +"test_instancenorm_example", +"test_isinf", +"test_isinf_negative", +"test_isinf_positive", +"test_isnan", +"test_less", +"test_less_bcast", +"test_less_equal", +"test_less_equal_bcast", +"test_less_equal_bcast_expanded", +"test_less_equal_expanded", +"test_log", +"test_log_example", +"test_loop11", +"test_loop13_seq", +"test_loop16_seq_none", +"test_lstm_batchwise", +"test_lstm_defaults", +"test_lstm_with_initial_bias", +"test_lstm_with_peepholes", +"test_matmulinteger", +"test_max_example", +"test_max_float16", +"test_max_float32", +"test_max_float64", +"test_max_int16", +"test_max_int32", +"test_max_int64", +"test_max_int8", +"test_max_one_input", +"test_max_two_inputs", +"test_max_uint16", +"test_max_uint32", +"test_max_uint64", +"test_max_uint8", +"test_maxpool_2d_uint8", +"test_maxunpool_export_with_output_shape", +"test_maxunpool_export_without_output_shape", +"test_mean_example", +"test_mean_one_input", +"test_mean_two_inputs", +"test_min_example", +"test_min_float16", +"test_min_float32", +"test_min_float64", +"test_min_int16", +"test_min_int32", +"test_min_int64", +"test_min_int8", +"test_min_one_input", +"test_min_two_inputs", +"test_min_uint16", +"test_min_uint32", +"test_min_uint64", +"test_min_uint8", +"test_mod_broadcast", +"test_mod_int64_fmod", +"test_mod_mixed_sign_float16", +"test_mod_mixed_sign_float32", +"test_mod_mixed_sign_float64", +"test_mod_mixed_sign_int16", +"test_mod_mixed_sign_int32", +"test_mod_mixed_sign_int64", +"test_mod_mixed_sign_int8", +"test_mod_uint16", +"test_mod_uint32", +"test_mod_uint64", +"test_mod_uint8", +"test_momentum", +"test_momentum_multiple", +"test_mul_example", +"test_mul_uint8", +"test_mvn", +"test_mvn_expanded", +"test_nesterov_momentum", +"test_nllloss_NC", +"test_nllloss_NC_expanded", +"test_nllloss_NCd1", +"test_nllloss_NCd1_expanded", +"test_nllloss_NCd1_ii", +"test_nllloss_NCd1_ii_expanded", +"test_nllloss_NCd1_mean_weight_negative_ii", +"test_nllloss_NCd1_mean_weight_negative_ii_expanded", +"test_nllloss_NCd1_weight", +"test_nllloss_NCd1_weight_expanded", +"test_nllloss_NCd1_weight_ii", +"test_nllloss_NCd1_weight_ii_expanded", +"test_nllloss_NCd1d2", +"test_nllloss_NCd1d2_expanded", +"test_nllloss_NCd1d2_no_weight_reduction_mean_ii", +"test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", +"test_nllloss_NCd1d2_reduction_mean", +"test_nllloss_NCd1d2_reduction_mean_expanded", +"test_nllloss_NCd1d2_reduction_sum", +"test_nllloss_NCd1d2_reduction_sum_expanded", +"test_nllloss_NCd1d2_with_weight", +"test_nllloss_NCd1d2_with_weight_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_mean", +"test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_sum", +"test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", +"test_nllloss_NCd1d2_with_weight_reduction_sum_ii", +"test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", +"test_nllloss_NCd1d2d3_none_no_weight_negative_ii", +"test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", +"test_nllloss_NCd1d2d3_sum_weight_high_ii", +"test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", +"test_nllloss_NCd1d2d3d4d5_mean_weight", +"test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", +"test_nllloss_NCd1d2d3d4d5_none_no_weight", +"test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", +"test_nonmaxsuppression_center_point_box_format", +"test_nonmaxsuppression_flipped_coordinates", +"test_nonmaxsuppression_identical_boxes", +"test_nonmaxsuppression_limit_output_size", +"test_nonmaxsuppression_single_box", +"test_nonmaxsuppression_suppress_by_IOU", +"test_nonmaxsuppression_suppress_by_IOU_and_scores", +"test_nonmaxsuppression_two_batches", +"test_nonmaxsuppression_two_classes", +"test_nonzero_example", +"test_not_2d", +"test_not_3d", +"test_not_4d", +"test_onehot_negative_indices", +"test_onehot_with_axis", +"test_onehot_with_negative_axis", +"test_onehot_without_axis", +"test_optional_get_element", +"test_optional_get_element_sequence", +"test_optional_has_element", +"test_optional_has_element_empty", +"test_or2d", +"test_or3d", +"test_or4d", +"test_or_bcast3v1d", +"test_or_bcast3v2d", +"test_or_bcast4v2d", +"test_or_bcast4v3d", +"test_or_bcast4v4d", +"test_pow", +"test_pow_bcast_array", +"test_pow_bcast_scalar", +"test_pow_example", +"test_pow_types_float", +"test_pow_types_float32_int32", +"test_pow_types_float32_int64", +"test_pow_types_float32_uint32", +"test_pow_types_float32_uint64", +"test_pow_types_int", +"test_pow_types_int32_float32", +"test_pow_types_int32_int32", +"test_pow_types_int64_float32", +"test_pow_types_int64_int64", +"test_prelu_broadcast", +"test_prelu_example", +"test_qlinearconv", +"test_qlinearmatmul_2D", +"test_qlinearmatmul_3D", +"test_quantizelinear", +"test_quantizelinear_axis", +"test_range_float_type_positive_delta", +"test_range_float_type_positive_delta_expanded", +"test_range_int32_type_negative_delta", +"test_range_int32_type_negative_delta_expanded", +"test_reciprocal", +"test_reciprocal_example", +"test_reduce_l1_default_axes_keepdims_example", +"test_reduce_l1_default_axes_keepdims_random", +"test_reduce_l1_do_not_keepdims_example", +"test_reduce_l1_do_not_keepdims_random", +"test_reduce_l1_keep_dims_example", +"test_reduce_l1_keep_dims_random", +"test_reduce_l1_negative_axes_keep_dims_example", +"test_reduce_l1_negative_axes_keep_dims_random", +"test_reduce_l2_default_axes_keepdims_example", +"test_reduce_l2_default_axes_keepdims_random", +"test_reduce_l2_do_not_keepdims_example", +"test_reduce_l2_do_not_keepdims_random", +"test_reduce_l2_keep_dims_example", +"test_reduce_l2_keep_dims_random", +"test_reduce_l2_negative_axes_keep_dims_example", +"test_reduce_l2_negative_axes_keep_dims_random", +"test_reduce_log_sum", +"test_reduce_log_sum_asc_axes", +"test_reduce_log_sum_default", +"test_reduce_log_sum_desc_axes", +"test_reduce_log_sum_exp_default_axes_keepdims_example", +"test_reduce_log_sum_exp_default_axes_keepdims_random", +"test_reduce_log_sum_exp_do_not_keepdims_example", +"test_reduce_log_sum_exp_do_not_keepdims_random", +"test_reduce_log_sum_exp_keepdims_example", +"test_reduce_log_sum_exp_keepdims_random", +"test_reduce_log_sum_exp_negative_axes_keepdims_example", +"test_reduce_log_sum_exp_negative_axes_keepdims_random", +"test_reduce_log_sum_negative_axes", +"test_reduce_max_default_axes_keepdim_example", +"test_reduce_max_default_axes_keepdims_random", +"test_reduce_mean_default_axes_keepdims_example", +"test_reduce_mean_default_axes_keepdims_random", +"test_reduce_min_default_axes_keepdims_example", +"test_reduce_min_default_axes_keepdims_random", +"test_reduce_min_do_not_keepdims_example", +"test_reduce_min_do_not_keepdims_random", +"test_reduce_min_keepdims_example", +"test_reduce_min_keepdims_random", +"test_reduce_min_negative_axes_keepdims_example", +"test_reduce_min_negative_axes_keepdims_random", +"test_reduce_prod_default_axes_keepdims_example", +"test_reduce_prod_default_axes_keepdims_random", +"test_reduce_prod_do_not_keepdims_example", +"test_reduce_prod_do_not_keepdims_random", +"test_reduce_prod_keepdims_example", +"test_reduce_prod_keepdims_random", +"test_reduce_prod_negative_axes_keepdims_example", +"test_reduce_prod_negative_axes_keepdims_random", +"test_reduce_sum_default_axes_keepdims_example", +"test_reduce_sum_default_axes_keepdims_random", +"test_reduce_sum_do_not_keepdims_example", +"test_reduce_sum_do_not_keepdims_random", +"test_reduce_sum_empty_axes_input_noop_example", +"test_reduce_sum_empty_axes_input_noop_random", +"test_reduce_sum_keepdims_example", +"test_reduce_sum_keepdims_random", +"test_reduce_sum_negative_axes_keepdims_example", +"test_reduce_sum_negative_axes_keepdims_random", +"test_reduce_sum_square_default_axes_keepdims_example", +"test_reduce_sum_square_default_axes_keepdims_random", +"test_reduce_sum_square_do_not_keepdims_example", +"test_reduce_sum_square_do_not_keepdims_random", +"test_reduce_sum_square_keepdims_example", +"test_reduce_sum_square_keepdims_random", +"test_reduce_sum_square_negative_axes_keepdims_example", +"test_reduce_sum_square_negative_axes_keepdims_random", +"test_reflect_pad", +"test_reshape_allowzero_reordered", +"test_reshape_extended_dims", +"test_reshape_negative_dim", +"test_reshape_negative_extended_dims", +"test_reshape_one_dim", +"test_reshape_reduced_dims", +"test_reshape_reordered_all_dims", +"test_reshape_reordered_last_dims", +"test_reshape_zero_and_negative_dim", +"test_reshape_zero_dim", +"test_resize_downsample_scales_cubic", +"test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", +"test_resize_downsample_scales_cubic_align_corners", +"test_resize_downsample_scales_linear", +"test_resize_downsample_scales_linear_align_corners", +"test_resize_downsample_scales_nearest", +"test_resize_downsample_sizes_cubic", +"test_resize_downsample_sizes_linear_pytorch_half_pixel", +"test_resize_downsample_sizes_nearest", +"test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn", +"test_resize_tf_crop_and_resize", +"test_resize_upsample_scales_cubic", +"test_resize_upsample_scales_cubic_A_n0p5_exclude_outside", +"test_resize_upsample_scales_cubic_align_corners", +"test_resize_upsample_scales_cubic_asymmetric", +"test_resize_upsample_scales_linear", +"test_resize_upsample_scales_linear_align_corners", +"test_resize_upsample_scales_nearest", +"test_resize_upsample_sizes_cubic", +"test_resize_upsample_sizes_nearest", +"test_resize_upsample_sizes_nearest_ceil_half_pixel", +"test_resize_upsample_sizes_nearest_floor_align_corners", +"test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", +"test_reversesequence_batch", +"test_reversesequence_time", +"test_rnn_seq_length", +"test_roialign_aligned_false", +"test_roialign_aligned_true", +"test_round", +"test_scan9_sum", +"test_scan_sum", +"test_scatter_elements_with_axis", +"test_scatter_elements_with_duplicate_indices", +"test_scatter_elements_with_negative_indices", +"test_scatter_elements_without_axis", +"test_scatter_with_axis", +"test_scatter_without_axis", +"test_scatternd", +"test_scatternd_add", +"test_scatternd_multiply", +"test_sce_NCd1_mean_weight_negative_ii", +"test_sce_NCd1_mean_weight_negative_ii_expanded", +"test_sce_NCd1_mean_weight_negative_ii_log_prob", +"test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", +"test_sce_NCd1d2d3_none_no_weight_negative_ii", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", +"test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", +"test_sce_NCd1d2d3_sum_weight_high_ii", +"test_sce_NCd1d2d3_sum_weight_high_ii_expanded", +"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", +"test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", +"test_sce_NCd1d2d3d4d5_mean_weight", +"test_sce_NCd1d2d3d4d5_mean_weight_expanded", +"test_sce_NCd1d2d3d4d5_mean_weight_log_prob", +"test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", +"test_sce_NCd1d2d3d4d5_none_no_weight", +"test_sce_NCd1d2d3d4d5_none_no_weight_expanded", +"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", +"test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", +"test_sce_mean", +"test_sce_mean_3d", +"test_sce_mean_3d_expanded", +"test_sce_mean_3d_log_prob", +"test_sce_mean_3d_log_prob_expanded", +"test_sce_mean_expanded", +"test_sce_mean_log_prob", +"test_sce_mean_log_prob_expanded", +"test_sce_mean_no_weight_ii", +"test_sce_mean_no_weight_ii_3d", +"test_sce_mean_no_weight_ii_3d_expanded", +"test_sce_mean_no_weight_ii_3d_log_prob", +"test_sce_mean_no_weight_ii_3d_log_prob_expanded", +"test_sce_mean_no_weight_ii_4d", +"test_sce_mean_no_weight_ii_4d_expanded", +"test_sce_mean_no_weight_ii_4d_log_prob", +"test_sce_mean_no_weight_ii_4d_log_prob_expanded", +"test_sce_mean_no_weight_ii_expanded", +"test_sce_mean_no_weight_ii_log_prob", +"test_sce_mean_no_weight_ii_log_prob_expanded", +"test_sce_mean_weight", +"test_sce_mean_weight_expanded", +"test_sce_mean_weight_ii", +"test_sce_mean_weight_ii_3d", +"test_sce_mean_weight_ii_3d_expanded", +"test_sce_mean_weight_ii_3d_log_prob", +"test_sce_mean_weight_ii_3d_log_prob_expanded", +"test_sce_mean_weight_ii_4d", +"test_sce_mean_weight_ii_4d_expanded", +"test_sce_mean_weight_ii_4d_log_prob", +"test_sce_mean_weight_ii_4d_log_prob_expanded", +"test_sce_mean_weight_ii_expanded", +"test_sce_mean_weight_ii_log_prob", +"test_sce_mean_weight_ii_log_prob_expanded", +"test_sce_mean_weight_log_prob", +"test_sce_mean_weight_log_prob_expanded", +"test_sce_none", +"test_sce_none_expanded", +"test_sce_none_log_prob", +"test_sce_none_log_prob_expanded", +"test_sce_none_weights", +"test_sce_none_weights_expanded", +"test_sce_none_weights_log_prob", +"test_sce_none_weights_log_prob_expanded", +"test_sce_sum", +"test_sce_sum_expanded", +"test_sce_sum_log_prob", +"test_sce_sum_log_prob_expanded", +"test_selu", +"test_selu_default", +"test_selu_example", +"test_sequence_insert_at_back", +"test_sequence_insert_at_front", +"test_shape", +"test_shape_clip_end", +"test_shape_clip_start", +"test_shape_end_1", +"test_shape_end_negative_1", +"test_shape_example", +"test_shape_start_1", +"test_shape_start_1_end_2", +"test_shape_start_1_end_negative_1", +"test_shape_start_negative_1", +"test_shrink_hard", +"test_shrink_soft", +"test_sign", +"test_simple_rnn_batchwise", +"test_simple_rnn_defaults", +"test_simple_rnn_with_initial_bias", +"test_sin", +"test_sin_example", +"test_sinh", +"test_sinh_example", +"test_size", +"test_size_example", +"test_slice", +"test_slice_default_axes", +"test_slice_default_steps", +"test_slice_end_out_of_bounds", +"test_slice_neg", +"test_slice_neg_steps", +"test_slice_negative_axes", +"test_slice_start_out_of_bounds", +"test_softplus", +"test_softplus_example", +"test_softsign", +"test_softsign_example", +"test_spacetodepth", +"test_spacetodepth_example", +"test_split_variable_parts_1d", +"test_split_variable_parts_2d", +"test_split_variable_parts_default_axis", +"test_split_zero_size_splits", +"test_sqrt", +"test_sqrt_example", +"test_squeeze", +"test_squeeze_negative_axes", +"test_strnormalizer_export_monday_casesensintive_lower", +"test_strnormalizer_export_monday_casesensintive_nochangecase", +"test_strnormalizer_export_monday_casesensintive_upper", +"test_strnormalizer_export_monday_empty_output", +"test_strnormalizer_export_monday_insensintive_upper_twodim", +"test_strnormalizer_nostopwords_nochangecase", +"test_sub_example", +"test_sub_uint8", +"test_sum_example", +"test_sum_two_inputs", +"test_tan", +"test_tan_example", +"test_tfidfvectorizer_tf_batch_onlybigrams_skip0", +"test_tfidfvectorizer_tf_batch_onlybigrams_skip5", +"test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", +"test_tfidfvectorizer_tf_only_bigrams_skip0", +"test_tfidfvectorizer_tf_onlybigrams_levelempty", +"test_tfidfvectorizer_tf_onlybigrams_skip5", +"test_tfidfvectorizer_tf_uniandbigrams_skip5", +"test_thresholdedrelu", +"test_thresholdedrelu_default", +"test_thresholdedrelu_example", +"test_tile", +"test_tile_precomputed", +"test_top_k", +"test_top_k_negative_axis", +"test_top_k_smallest", +"test_training_dropout", +"test_training_dropout_default", +"test_training_dropout_default_mask", +"test_training_dropout_mask", +"test_training_dropout_zero_ratio", +"test_training_dropout_zero_ratio_mask", +"test_tril", +"test_tril_neg", +"test_tril_one_row_neg", +"test_tril_out_neg", +"test_tril_out_pos", +"test_tril_pos", +"test_tril_square", +"test_tril_square_neg", +"test_tril_zero", +"test_triu", +"test_triu_neg", +"test_triu_one_row", +"test_triu_out_neg_out", +"test_triu_out_pos", +"test_triu_pos", +"test_triu_square", +"test_triu_square_neg", +"test_triu_zero", +"test_unique_not_sorted_without_axis", +"test_unique_sorted_with_axis", +"test_unique_sorted_with_axis_3d", +"test_unique_sorted_with_negative_axis", +"test_unique_sorted_without_axis", +"test_unsqueeze_axis_0", +"test_unsqueeze_axis_1", +"test_unsqueeze_axis_2", +"test_unsqueeze_negative_axes", +"test_unsqueeze_three_axes", +"test_unsqueeze_two_axes", +"test_unsqueeze_unsorted_axes", +"test_where_example", +"test_where_long_example", +"test_xor2d", +"test_xor3d", +"test_xor4d", +"test_xor_bcast3v1d", +"test_xor_bcast3v2d", +"test_xor_bcast4v2d", +"test_xor_bcast4v3d", +"test_xor_bcast4v4d", diff --git a/modules/ts/misc/testlog_parser.py b/modules/ts/misc/testlog_parser.py index 2e9718be3e..f52a051c8f 100755 --- a/modules/ts/misc/testlog_parser.py +++ b/modules/ts/misc/testlog_parser.py @@ -182,9 +182,12 @@ class TestInfo(object): return 1 return 0 + def __lt__(self, other): + return self.__cmp__(other) == -1 + # This is a Sequence for compatibility with old scripts, # which treat parseLogFile's return value as a list. -class TestRunInfo(collections.Sequence): +class TestRunInfo(object): def __init__(self, properties, tests): self.properties = properties self.tests = tests From e97c7e042b2e4a15b1b5a284873efbff8e0dce02 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 14 Dec 2021 20:54:43 +0300 Subject: [PATCH 7/8] fix max_unpool missing attributes, add default value of keepdims in reducemean/max/sum, add support for keepdims=true in full reduction branch, add new padding type to Pad --- modules/dnn/src/layers/padding_layer.cpp | 9 ++-- modules/dnn/src/onnx/onnx_importer.cpp | 48 +++++++++++++++++-- ..._conformance_layer_parser_denylist.inl.hpp | 5 -- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/modules/dnn/src/layers/padding_layer.cpp b/modules/dnn/src/layers/padding_layer.cpp index af3dacdd7a..9d7ca23714 100644 --- a/modules/dnn/src/layers/padding_layer.cpp +++ b/modules/dnn/src/layers/padding_layer.cpp @@ -130,12 +130,13 @@ public: outputs[0].setTo(paddingValue); inputs[0].copyTo(outputs[0](dstRanges)); } - else if (paddingType == "reflect") + else if (paddingType == "reflect" || paddingType == "edge") { CV_Assert(inputs.size() == 1); CV_Assert(outputs.size() == 1); CV_Assert(inputs[0].dims == 4); CV_Assert(outputs[0].dims == 4); + int borderType = paddingType == "reflect" ? BORDER_REFLECT_101 : BORDER_REPLICATE; if (inputs[0].size[0] != outputs[0].size[0] || inputs[0].size[1] != outputs[0].size[1]) CV_Error(Error::StsNotImplemented, "Only spatial reflection padding is supported."); @@ -148,8 +149,8 @@ public: const int padBottom = outHeight - dstRanges[2].end; const int padLeft = dstRanges[3].start; const int padRight = outWidth - dstRanges[3].end; - CV_CheckLT(padTop, inpHeight, ""); CV_CheckLT(padBottom, inpHeight, ""); - CV_CheckLT(padLeft, inpWidth, ""); CV_CheckLT(padRight, inpWidth, ""); + CV_CheckLE(padTop, inpHeight, ""); CV_CheckLE(padBottom, inpHeight, ""); + CV_CheckLE(padLeft, inpWidth, ""); CV_CheckLE(padRight, inpWidth, ""); for (size_t n = 0; n < inputs[0].size[0]; ++n) { @@ -158,7 +159,7 @@ public: copyMakeBorder(getPlane(inputs[0], n, ch), getPlane(outputs[0], n, ch), padTop, padBottom, padLeft, padRight, - BORDER_REFLECT_101); + borderType); } } } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 8ff91f5e44..052b1b8529 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -131,6 +131,7 @@ private: typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); typedef std::map DispatchMap; + void parseMaxUnpool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -625,6 +626,41 @@ void setCeilMode(LayerParams& layerParams) } } +void ONNXImporter::parseMaxUnpool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "MaxUnpool"; + + DictValue kernel_shape = layerParams.get("kernel_size"); + CV_Assert(kernel_shape.size() == 2); + layerParams.set("pool_k_w", kernel_shape.get(0)); + layerParams.set("pool_k_h", kernel_shape.get(1)); + + int pool_pad_w = 0, pool_pad_h = 0; + if (layerParams.has("pad")) + { + DictValue pads = layerParams.get("pad"); + CV_CheckEQ(pads.size(), 2, ""); + pool_pad_w = pads.get(0); + pool_pad_h = pads.get(1); + } + layerParams.set("pool_pad_w", pool_pad_w); + layerParams.set("pool_pad_h", pool_pad_h); + + + int pool_stride_w = 1, pool_stride_h = 1; + if (layerParams.has("stride")) + { + DictValue strides = layerParams.get("stride"); + CV_CheckEQ(strides.size(), 2, ""); + pool_stride_w = strides.get(0); + pool_stride_h = strides.get(1); + } + layerParams.set("pool_stride_w", pool_stride_w); + layerParams.set("pool_stride_h", pool_stride_h); + + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "Pooling"; @@ -659,11 +695,11 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node pool = "AVE"; layerParams.set("pool", pool); layerParams.set("global_pooling", !layerParams.has("axes")); + bool keepdims = layerParams.get("keepdims", 1) == 1; if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) { MatShape inpShape = outShapes[node_proto.input(0)]; DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); MatShape targetShape; std::vector shouldDelete(inpShape.size(), false); for (int i = 0; i < axes.size(); i++) { @@ -771,7 +807,10 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node } else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const size_t dims = keepdims ? shapeIt->second.size() : 1; + LayerParams reshapeLp; reshapeLp.name = layerParams.name + "/reshape"; reshapeLp.type = "Reshape"; @@ -793,8 +832,8 @@ void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::Node addLayer(poolLp, node_proto); layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + std::vector targetShape(dims, 1); + layerParams.set("dim", DictValue::arrayInt(targetShape.data(), targetShape.size())); node_proto.set_input(0, node_proto.output(0)); node_proto.set_output(0, layerParams.name); @@ -2341,6 +2380,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() { DispatchMap dispatch; + dispatch["MaxUnpool"] = &ONNXImporter::parseMaxUnpool; dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 5f1d99d4bb..2489105e91 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -277,7 +277,6 @@ "test_max_uint8", "test_maxpool_2d_uint8", "test_maxunpool_export_with_output_shape", -"test_maxunpool_export_without_output_shape", "test_mean_example", "test_mean_one_input", "test_mean_two_inputs", @@ -436,10 +435,6 @@ "test_reduce_log_sum_exp_negative_axes_keepdims_example", "test_reduce_log_sum_exp_negative_axes_keepdims_random", "test_reduce_log_sum_negative_axes", -"test_reduce_max_default_axes_keepdim_example", -"test_reduce_max_default_axes_keepdims_random", -"test_reduce_mean_default_axes_keepdims_example", -"test_reduce_mean_default_axes_keepdims_random", "test_reduce_min_default_axes_keepdims_example", "test_reduce_min_default_axes_keepdims_random", "test_reduce_min_do_not_keepdims_example", From f3ba88c87c79115531b9d74dd78682b1d0151713 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 15 Dec 2021 08:57:50 +0000 Subject: [PATCH 8/8] dnn(test): update ONNX conformance filters --- modules/dnn/test/test_common.impl.hpp | 2 +- modules/dnn/test/test_onnx_conformance.cpp | 214 +- ...ance_layer_filter__halide_denylist.inl.hpp | 60 + ...conformance_layer_filter__openvino.inl.hpp | 1945 +++++++++++++++++ 4 files changed, 2105 insertions(+), 116 deletions(-) create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp create mode 100644 modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index a55a8f788c..5546f68916 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -368,7 +368,7 @@ void initDNNTests() #if defined(HAVE_HALIDE) registerGlobalSkipTag( CV_TEST_TAG_DNN_SKIP_HALIDE - ) + ); #endif #if defined(INF_ENGINE_RELEASE) registerGlobalSkipTag( diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index a1d83bd614..71ac4bef19 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -898,24 +898,16 @@ static const TestCase testConformanceConfig[] = { }; -struct TestCaseInput -{ - std::vector input_paths; - std::vector output_paths; - std::string model_path; - std::string name; -}; - -std::ostream& operator<<(std::ostream& os, const TestCaseInput& test_case) +std::ostream& operator<<(std::ostream& os, const TestCase& test_case) { return os << test_case.name; } -typedef tuple > ONNXConfParams; +typedef tuple > ONNXConfParams; std::string printOnnxConfParams(const testing::TestParamInfo& params) { - TestCaseInput test_case = get<0>(params.param); + TestCase test_case = get<0>(params.param); Backend backend = get<0>(get<1>(params.param)); Target target = get<1>(get<1>(params.param)); @@ -928,45 +920,11 @@ std::string printOnnxConfParams(const testing::TestParamInfo& pa return ss.str(); } -template -static std::string _tf(TString filename, bool required = true) -{ - return findDataFile(std::string("dnn/onnx/") + filename, required); -} - -std::vector readTestCases() -{ - std::vector ret; - for (size_t i = 0; i < sizeof(testConformanceConfig) / sizeof(testConformanceConfig[0]); ++i) - { - const TestCase& test_case = testConformanceConfig[i]; - - TestCaseInput input; - - std::string prefix = cv::format("conformance/node/%s", test_case.name); - input.name = test_case.name; - input.model_path = _tf(cv::format("%s/model.onnx", prefix.c_str())); - - for (int i = 0; i < test_case.inputs; ++i) - { - input.input_paths.push_back(_tf(cv::format("%s/test_data_set_0/input_%d.pb", prefix.c_str(), i))); - } - - for (int i = 0; i < test_case.outputs; ++i) - { - input.output_paths.push_back(_tf(cv::format("%s/test_data_set_0/output_%d.pb", prefix.c_str(), i))); - } - - ret.push_back(input); - } - - return ret; -} - class Test_ONNX_conformance : public TestWithParam { public: - TestCaseInput test_case; + + TestCase test_case; Backend backend; Target target; @@ -978,6 +936,9 @@ public: static std::set opencl_fp16_deny_list; static std::set opencl_deny_list; static std::set cpu_deny_list; +#ifdef HAVE_HALIDE + static std::set halide_deny_list; +#endif Test_ONNX_conformance() { @@ -1059,68 +1020,16 @@ public: "" // dummy element of non empty list }; initDenyList(cpu_deny_list, cpu, sizeof(cpu)/sizeof(cpu[0])); - } - void checkFilterLists() const - { - const std::string& name = test_case.name; - if(parser_deny_list.find(name) != parser_deny_list.end()) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - - if (backend == DNN_BACKEND_OPENCV) - { - if(global_deny_list.find(name) != global_deny_list.end()) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - if((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - if((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - if((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); - } - } -#if 0 //def HAVE_HALIDE - else if (backend == DNN_BACKEND_HALIDE) - { - #include "test_onnx_conformance_layer_filter__halide.inl.hpp" - } -#endif -#if 0 //def HAVE_INF_ENGINE - else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - { - #include "test_onnx_conformance_layer_filter__ngraph.inl.hpp" - } -#endif -#if 0 //def HAVE_VULKAN - else if (backend == DNN_BACKEND_VKCOM) - { - #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" - } -#endif -#if 0 //def HAVE_CUDA - else if (backend == DNN_BACKEND_CUDA) - { - #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" - } +#ifdef HAVE_HALIDE + const char* const halide_deny_list_[] = { + #include "test_onnx_conformance_layer_filter__halide_denylist.inl.hpp" + "" // dummy element of non empty list + }; + initDenyList(halide_deny_list, halide_deny_list_, sizeof(halide_deny_list_)/sizeof(halide_deny_list_[0])); #endif - else - { - std::ostringstream ss; - ss << "No test filter available for backend "; - PrintTo(backend, &ss); - ss << ". Run test by default"; - std::cout << ss.str() << std::endl; - } } + }; std::set Test_ONNX_conformance::parser_deny_list; @@ -1128,33 +1037,104 @@ std::set Test_ONNX_conformance::global_deny_list; std::set Test_ONNX_conformance::opencl_fp16_deny_list; std::set Test_ONNX_conformance::opencl_deny_list; std::set Test_ONNX_conformance::cpu_deny_list; +#ifdef HAVE_HALIDE +std::set Test_ONNX_conformance::halide_deny_list; +#endif TEST_P(Test_ONNX_conformance, Layer_Test) { - std::string name = test_case.name; + const std::string& name = test_case.name; ASSERT_FALSE(name.empty()); bool checkLayersFallbacks = true; bool checkAccuracy = true; - checkFilterLists(); + if (parser_deny_list.find(name) != parser_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + + if (backend == DNN_BACKEND_OPENCV) + { + if (global_deny_list.find(name) != global_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if ((target == DNN_TARGET_OPENCL_FP16) && (opencl_fp16_deny_list.find(name) != opencl_fp16_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if ((target == DNN_TARGET_OPENCL) && (opencl_deny_list.find(name) != opencl_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + if ((target == DNN_TARGET_CPU) && (cpu_deny_list.find(name) != cpu_deny_list.end())) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU, CV_TEST_TAG_DNN_SKIP_OPENCV_BACKEND, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#ifdef HAVE_HALIDE + else if (backend == DNN_BACKEND_HALIDE) + { + if (halide_deny_list.find(name) != halide_deny_list.end()) + { + applyTestTag(CV_TEST_TAG_DNN_SKIP_HALIDE, CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE); + } + } +#endif +#ifdef HAVE_INF_ENGINE + else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + #include "test_onnx_conformance_layer_filter__openvino.inl.hpp" + } +#endif +#if 0 //def HAVE_VULKAN + else if (backend == DNN_BACKEND_VKCOM) + { + #include "test_onnx_conformance_layer_filter__vulkan.inl.hpp" + } +#endif +#if 0 //def HAVE_CUDA + else if (backend == DNN_BACKEND_CUDA) + { + #include "test_onnx_conformance_layer_filter__cuda.inl.hpp" + } +#endif + else + { + std::ostringstream ss; + ss << "No test filter available for backend "; + PrintTo(backend, &ss); + ss << ". Run test by default"; + std::cout << ss.str() << std::endl; + } std::vector inputs; std::vector ref_outputs; + std::string prefix = cv::format("dnn/onnx/conformance/node/%s", test_case.name); + Net net; try { + std::string model_path = findDataFile(prefix + "/model.onnx"); + //cout << "Read ONNX inputs..." << endl; - std::transform(test_case.input_paths.begin(), test_case.input_paths.end(), - std::back_inserter(inputs), readTensorFromONNX); + for (int i = 0; i < test_case.inputs; ++i) + { + Mat input = readTensorFromONNX(findDataFile(prefix + cv::format("/test_data_set_0/input_%d.pb", i))); + inputs.push_back(input); + } //cout << "Read ONNX reference outputs..." << endl; - std::transform(test_case.output_paths.begin(), test_case.output_paths.end(), - std::back_inserter(ref_outputs), readTensorFromONNX); + for (int i = 0; i < test_case.outputs; ++i) + { + Mat output = readTensorFromONNX(findDataFile(prefix + cv::format("/test_data_set_0/output_%d.pb", i))); + ref_outputs.push_back(output); + } //cout << "Parse model..." << endl; - net = readNetFromONNX(test_case.model_path); + net = readNetFromONNX(model_path); if (net.empty()) { applyTestTag(CV_TEST_TAG_DNN_ERROR_PARSER); @@ -1244,7 +1224,11 @@ TEST_P(Test_ONNX_conformance, Layer_Test) } INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_conformance, - testing::Combine(testing::ValuesIn(readTestCases()), dnnBackendsAndTargets()), - printOnnxConfParams); + testing::Combine( + testing::ValuesIn(testConformanceConfig), + dnnBackendsAndTargets(/*withInferenceEngine=*/true, /*withHalide=*/true) + ), + printOnnxConfParams +); }; diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp new file mode 100644 index 0000000000..dd0a249081 --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__halide_denylist.inl.hpp @@ -0,0 +1,60 @@ +"test_add", +"test_add_bcast", +"test_averagepool_2d_ceil", +"test_averagepool_2d_pads_count_include_pad", +"test_averagepool_2d_precomputed_pads_count_include_pad", +"test_averagepool_2d_precomputed_strides", +"test_averagepool_2d_same_lower", +"test_averagepool_2d_same_upper", +"test_cast_FLOAT_to_STRING", +"test_cast_STRING_to_FLOAT", +"test_castlike_FLOAT_to_STRING_expanded", +"test_castlike_STRING_to_FLOAT_expanded", +"test_concat_1d_axis_negative_1", +"test_concat_3d_axis_1", +"test_div", +"test_div_bcast", +"test_elu", +"test_elu_default", +"test_exp", +"test_flatten_axis0", +"test_flatten_axis2", +"test_flatten_axis3", +"test_flatten_negative_axis1", +"test_flatten_negative_axis2", +"test_flatten_negative_axis4", +"test_leakyrelu", +"test_leakyrelu_default", +"test_logsoftmax_axis_1", +"test_logsoftmax_axis_1_expanded", +"test_logsoftmax_default_axis", +"test_logsoftmax_example_1", +"test_logsoftmax_large_number", +"test_matmul_2d", +"test_matmul_3d", +"test_matmul_4d", +"test_maxpool_2d_dilations", +"test_maxpool_2d_same_lower", +"test_maxpool_with_argmax_2d_precomputed_pads", +"test_maxpool_with_argmax_2d_precomputed_strides", +"test_mul", +"test_mul_bcast", +"test_neg", +"test_reduce_max_default_axes_keepdim_example", +"test_reduce_max_default_axes_keepdims_random", +"test_reduce_max_do_not_keepdims_example", +"test_reduce_max_do_not_keepdims_random", +"test_reduce_max_keepdims_example", +"test_reduce_max_keepdims_random", +"test_reduce_max_negative_axes_keepdims_example", +"test_reduce_max_negative_axes_keepdims_random", +"test_relu", +"test_sigmoid", +"test_softmax_axis_1", +"test_softmax_axis_1_expanded", +"test_softmax_default_axis", +"test_softmax_large_number", +"test_sub", +"test_sub_bcast", +"test_tanh", +"test_upsample_nearest", diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp new file mode 100644 index 0000000000..25bb8dff9a --- /dev/null +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -0,0 +1,1945 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// not a standalone file, see test_onnx_conformance.cpp +#if 0 +cout << "Filtering is disabled: OpenVINO" << endl; +#else + + +#if 0 +// Stats for --gtest_filter=*ONNX_conformance*NGRAPH* +[ SKIPSTAT ] TAG='dnn_skip_ie_myriadx' skip 48 tests +[ SKIPSTAT ] TAG='dnn_skip_ie' skip 0 tests (149 times in extra skip list) +[ SKIPSTAT ] TAG='dnn_skip_ie_ngraph' skip 0 tests (149 times in extra skip list) +[ SKIPSTAT ] TAG='dnn_skip_ie_cpu' skip 29 tests +[ SKIPSTAT ] TAG='dnn_skip_ie_ocl' skip 34 tests +[ SKIPSTAT ] TAG='dnn_skip_ie_ocl_fp16' skip 38 tests +#endif + + +#define SKIP_TAGS \ + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, \ + CV_TEST_TAG_DNN_SKIP_IE, \ + CV_TEST_TAG_DNN_SKIP_ONNX_CONFORMANCE +#define SKIP_(...) applyTestTag(__VA_ARGS__, SKIP_TAGS) +#define SKIP applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_CPU if (target == DNN_TARGET_CPU) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_NON_CPU if (target != DNN_TARGET_CPU) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_OPENCL if (target == DNN_TARGET_OPENCL) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_OPENCL_FP16 if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(tag_target_skip, SKIP_TAGS) +#define SKIP_MYRIAD if (target == DNN_TARGET_MYRIAD) applyTestTag(tag_target_skip, SKIP_TAGS) + +std::string tag_target_skip = + (target == DNN_TARGET_CPU) ? CV_TEST_TAG_DNN_SKIP_IE_CPU : + (target == DNN_TARGET_OPENCL) ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : + (target == DNN_TARGET_OPENCL_FP16) ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16 : + (target == DNN_TARGET_MYRIAD) ? CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X : + ""; + +ASSERT_FALSE(name.empty()); + +#define EOF_LABEL exit_filter_opencv +#define BEGIN_SWITCH() \ +if (name.empty() /*false*/) \ +{ + +#define CASE(t) \ + goto EOF_LABEL; \ +} \ +if (name == #t) \ +{ \ + filterApplied = true; + +#define END_SWITCH() \ + goto EOF_LABEL; \ +} \ +EOF_LABEL: + +bool filterApplied = false; + +// Update note: execute /testdata/dnn/onnx/generate_conformance_list.py +BEGIN_SWITCH() +CASE(test_abs) + // no filter +CASE(test_acos) + // no filter +CASE(test_acos_example) + // no filter +CASE(test_acosh) + // no filter +CASE(test_acosh_example) + // no filter +CASE(test_adagrad) + // no filter +CASE(test_adagrad_multiple) + // no filter +CASE(test_adam) + // no filter +CASE(test_adam_multiple) + // no filter +CASE(test_add) + // no filter +CASE(test_add_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_add_uint8) + // no filter +CASE(test_and2d) + // no filter +CASE(test_and3d) + // no filter +CASE(test_and4d) + // no filter +CASE(test_and_bcast3v1d) + // no filter +CASE(test_and_bcast3v2d) + // no filter +CASE(test_and_bcast4v2d) + // no filter +CASE(test_and_bcast4v3d) + // no filter +CASE(test_and_bcast4v4d) + // no filter +CASE(test_argmax_default_axis_example) + // no filter +CASE(test_argmax_default_axis_example_select_last_index) + // no filter +CASE(test_argmax_default_axis_random) + // no filter +CASE(test_argmax_default_axis_random_select_last_index) + // no filter +CASE(test_argmax_keepdims_example) + // no filter +CASE(test_argmax_keepdims_example_select_last_index) + // no filter +CASE(test_argmax_keepdims_random) + // no filter +CASE(test_argmax_keepdims_random_select_last_index) + // no filter +CASE(test_argmax_negative_axis_keepdims_example) + // no filter +CASE(test_argmax_negative_axis_keepdims_example_select_last_index) + // no filter +CASE(test_argmax_negative_axis_keepdims_random) + // no filter +CASE(test_argmax_negative_axis_keepdims_random_select_last_index) + // no filter +CASE(test_argmax_no_keepdims_example) + // no filter +CASE(test_argmax_no_keepdims_example_select_last_index) + // no filter +CASE(test_argmax_no_keepdims_random) + // no filter +CASE(test_argmax_no_keepdims_random_select_last_index) + // no filter +CASE(test_argmin_default_axis_example) + // no filter +CASE(test_argmin_default_axis_example_select_last_index) + // no filter +CASE(test_argmin_default_axis_random) + // no filter +CASE(test_argmin_default_axis_random_select_last_index) + // no filter +CASE(test_argmin_keepdims_example) + // no filter +CASE(test_argmin_keepdims_example_select_last_index) + // no filter +CASE(test_argmin_keepdims_random) + // no filter +CASE(test_argmin_keepdims_random_select_last_index) + // no filter +CASE(test_argmin_negative_axis_keepdims_example) + // no filter +CASE(test_argmin_negative_axis_keepdims_example_select_last_index) + // no filter +CASE(test_argmin_negative_axis_keepdims_random) + // no filter +CASE(test_argmin_negative_axis_keepdims_random_select_last_index) + // no filter +CASE(test_argmin_no_keepdims_example) + // no filter +CASE(test_argmin_no_keepdims_example_select_last_index) + // no filter +CASE(test_argmin_no_keepdims_random) + // no filter +CASE(test_argmin_no_keepdims_random_select_last_index) + // no filter +CASE(test_asin) + // no filter +CASE(test_asin_example) + // no filter +CASE(test_asinh) + // no filter +CASE(test_asinh_example) + // no filter +CASE(test_atan) + // no filter +CASE(test_atan_example) + // no filter +CASE(test_atanh) + // no filter +CASE(test_atanh_example) + // no filter +CASE(test_averagepool_1d_default) + // no filter +CASE(test_averagepool_2d_ceil) + // no filter +CASE(test_averagepool_2d_default) + // no filter +CASE(test_averagepool_2d_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_averagepool_2d_pads_count_include_pad) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_averagepool_2d_precomputed_pads) + // no filter +CASE(test_averagepool_2d_precomputed_pads_count_include_pad) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_averagepool_2d_precomputed_same_upper) + // no filter +CASE(test_averagepool_2d_precomputed_strides) + // no filter +CASE(test_averagepool_2d_same_lower) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_averagepool_2d_same_upper) + // no filter +CASE(test_averagepool_2d_strides) + // no filter +CASE(test_averagepool_3d_default) + // no filter +CASE(test_basic_conv_with_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_basic_conv_without_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_basic_convinteger) + // no filter +CASE(test_batchnorm_epsilon) + // no filter +CASE(test_batchnorm_epsilon_training_mode) + // no filter +CASE(test_batchnorm_example) + // no filter +CASE(test_batchnorm_example_training_mode) + // no filter +CASE(test_bernoulli) + // no filter +CASE(test_bernoulli_double) + // no filter +CASE(test_bernoulli_double_expanded) + // no filter +CASE(test_bernoulli_expanded) + // no filter +CASE(test_bernoulli_seed) + // no filter +CASE(test_bernoulli_seed_expanded) + // no filter +CASE(test_bitshift_left_uint16) + // no filter +CASE(test_bitshift_left_uint32) + // no filter +CASE(test_bitshift_left_uint64) + // no filter +CASE(test_bitshift_left_uint8) + // no filter +CASE(test_bitshift_right_uint16) + // no filter +CASE(test_bitshift_right_uint32) + // no filter +CASE(test_bitshift_right_uint64) + // no filter +CASE(test_bitshift_right_uint8) + // no filter +CASE(test_cast_BFLOAT16_to_FLOAT) + // no filter +CASE(test_cast_DOUBLE_to_FLOAT) + // no filter +CASE(test_cast_DOUBLE_to_FLOAT16) + // no filter +CASE(test_cast_FLOAT16_to_DOUBLE) + // no filter +CASE(test_cast_FLOAT16_to_FLOAT) + // no filter +CASE(test_cast_FLOAT_to_BFLOAT16) + // no filter +CASE(test_cast_FLOAT_to_DOUBLE) + // no filter +CASE(test_cast_FLOAT_to_FLOAT16) + // no filter +CASE(test_cast_FLOAT_to_STRING) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_cast_STRING_to_FLOAT) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_castlike_BFLOAT16_to_FLOAT) + // no filter +CASE(test_castlike_BFLOAT16_to_FLOAT_expanded) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT16) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT16_expanded) + // no filter +CASE(test_castlike_DOUBLE_to_FLOAT_expanded) + // no filter +CASE(test_castlike_FLOAT16_to_DOUBLE) + // no filter +CASE(test_castlike_FLOAT16_to_DOUBLE_expanded) + // no filter +CASE(test_castlike_FLOAT16_to_FLOAT) + // no filter +CASE(test_castlike_FLOAT16_to_FLOAT_expanded) + // no filter +CASE(test_castlike_FLOAT_to_BFLOAT16) + // no filter +CASE(test_castlike_FLOAT_to_BFLOAT16_expanded) + // no filter +CASE(test_castlike_FLOAT_to_DOUBLE) + // no filter +CASE(test_castlike_FLOAT_to_DOUBLE_expanded) + // no filter +CASE(test_castlike_FLOAT_to_FLOAT16) + // no filter +CASE(test_castlike_FLOAT_to_FLOAT16_expanded) + // no filter +CASE(test_castlike_FLOAT_to_STRING) + // no filter +CASE(test_castlike_FLOAT_to_STRING_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_castlike_STRING_to_FLOAT) + // no filter +CASE(test_castlike_STRING_to_FLOAT_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_ceil) + // no filter +CASE(test_ceil_example) + // no filter +CASE(test_celu) + // no filter +CASE(test_celu_expanded) + // no filter +CASE(test_clip) + // no filter +CASE(test_clip_default_inbounds) + // no filter +CASE(test_clip_default_int8_inbounds) + // no filter +CASE(test_clip_default_int8_max) + // no filter +CASE(test_clip_default_int8_min) + // no filter +CASE(test_clip_default_max) + // no filter +CASE(test_clip_default_min) + // no filter +CASE(test_clip_example) + // no filter +CASE(test_clip_inbounds) + // no filter +CASE(test_clip_outbounds) + // no filter +CASE(test_clip_splitbounds) + // no filter +CASE(test_compress_0) + // no filter +CASE(test_compress_1) + // no filter +CASE(test_compress_default_axis) + // no filter +CASE(test_compress_negative_axis) + // no filter +CASE(test_concat_1d_axis_0) + // no filter +CASE(test_concat_1d_axis_negative_1) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_concat_2d_axis_0) + // no filter +CASE(test_concat_2d_axis_1) + // no filter +CASE(test_concat_2d_axis_negative_1) + // no filter +CASE(test_concat_2d_axis_negative_2) + // no filter +CASE(test_concat_3d_axis_0) + // no filter +CASE(test_concat_3d_axis_1) + // no filter +CASE(test_concat_3d_axis_2) + // no filter +CASE(test_concat_3d_axis_negative_1) + // no filter +CASE(test_concat_3d_axis_negative_2) + // no filter +CASE(test_concat_3d_axis_negative_3) + // no filter +CASE(test_constant) + // no filter +CASE(test_constant_pad) + // no filter +CASE(test_constantofshape_float_ones) + // no filter +CASE(test_constantofshape_int_shape_zero) + // no filter +CASE(test_constantofshape_int_zeros) + // no filter +CASE(test_conv_with_autopad_same) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_conv_with_strides_and_asymmetric_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_conv_with_strides_no_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_conv_with_strides_padding) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_convinteger_with_padding) + // no filter +CASE(test_convinteger_without_padding) + // no filter +CASE(test_convtranspose) + // no filter +CASE(test_convtranspose_1d) + // no filter +CASE(test_convtranspose_3d) + // no filter +CASE(test_convtranspose_autopad_same) + // no filter +CASE(test_convtranspose_dilations) + // no filter +CASE(test_convtranspose_kernel_shape) + // no filter +CASE(test_convtranspose_output_shape) + // no filter +CASE(test_convtranspose_pad) + // no filter +CASE(test_convtranspose_pads) + // no filter +CASE(test_convtranspose_with_kernel) + // no filter +CASE(test_cos) + // no filter +CASE(test_cos_example) + // no filter +CASE(test_cosh) + // no filter +CASE(test_cosh_example) + // no filter +CASE(test_cumsum_1d) + // no filter +CASE(test_cumsum_1d_exclusive) + // no filter +CASE(test_cumsum_1d_reverse) + // no filter +CASE(test_cumsum_1d_reverse_exclusive) + // no filter +CASE(test_cumsum_2d_axis_0) + // no filter +CASE(test_cumsum_2d_axis_1) + // no filter +CASE(test_cumsum_2d_negative_axis) + // no filter +CASE(test_depthtospace_crd_mode) + // no filter +CASE(test_depthtospace_crd_mode_example) + // no filter +CASE(test_depthtospace_dcr_mode) + // no filter +CASE(test_depthtospace_example) + // no filter +CASE(test_dequantizelinear) + // no filter +CASE(test_dequantizelinear_axis) + // no filter +CASE(test_det_2d) + // no filter +CASE(test_det_nd) + // no filter +CASE(test_div) + // no filter +CASE(test_div_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_div_example) + // no filter +CASE(test_div_uint8) + // no filter +CASE(test_dropout_default) + // no filter +CASE(test_dropout_default_mask) + // no filter +CASE(test_dropout_default_mask_ratio) + // no filter +CASE(test_dropout_default_old) + // no filter +CASE(test_dropout_default_ratio) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_dropout_random_old) + // no filter +CASE(test_dynamicquantizelinear) + // no filter +CASE(test_dynamicquantizelinear_expanded) + // no filter +CASE(test_dynamicquantizelinear_max_adjusted) + // no filter +CASE(test_dynamicquantizelinear_max_adjusted_expanded) + // no filter +CASE(test_dynamicquantizelinear_min_adjusted) + // no filter +CASE(test_dynamicquantizelinear_min_adjusted_expanded) + // no filter +CASE(test_edge_pad) + // no filter +CASE(test_einsum_batch_diagonal) + // no filter +CASE(test_einsum_batch_matmul) + // no filter +CASE(test_einsum_inner_prod) + // no filter +CASE(test_einsum_sum) + // no filter +CASE(test_einsum_transpose) + // no filter +CASE(test_elu) + // no filter +CASE(test_elu_default) + // no filter +CASE(test_elu_example) + // no filter +CASE(test_equal) + // no filter +CASE(test_equal_bcast) + // no filter +CASE(test_erf) + // no filter +CASE(test_exp) + // no filter +CASE(test_exp_example) + // no filter +CASE(test_expand_dim_changed) + // no filter +CASE(test_expand_dim_unchanged) + // no filter +CASE(test_eyelike_populate_off_main_diagonal) + // no filter +CASE(test_eyelike_with_dtype) + // no filter +CASE(test_eyelike_without_dtype) + // no filter +CASE(test_flatten_axis0) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_axis1) + // no filter +CASE(test_flatten_axis2) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_axis3) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_default_axis) + // no filter +CASE(test_flatten_negative_axis1) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_negative_axis2) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_flatten_negative_axis3) + // no filter +CASE(test_flatten_negative_axis4) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_floor) + // no filter +CASE(test_floor_example) + // no filter +CASE(test_gather_0) + // no filter +CASE(test_gather_1) + // no filter +CASE(test_gather_2d_indices) + // no filter +CASE(test_gather_elements_0) + // no filter +CASE(test_gather_elements_1) + // no filter +CASE(test_gather_elements_negative_indices) + // no filter +CASE(test_gather_negative_indices) + // no filter +CASE(test_gathernd_example_float32) + // no filter +CASE(test_gathernd_example_int32) + // no filter +CASE(test_gathernd_example_int32_batch_dim1) + // no filter +CASE(test_gemm_all_attributes) + // no filter +CASE(test_gemm_alpha) + // no filter +CASE(test_gemm_beta) + // no filter +CASE(test_gemm_default_matrix_bias) + // no filter +CASE(test_gemm_default_no_bias) + // no filter +CASE(test_gemm_default_scalar_bias) + // no filter +CASE(test_gemm_default_single_elem_vector_bias) + // no filter +CASE(test_gemm_default_vector_bias) + // no filter +CASE(test_gemm_default_zero_bias) + // no filter +CASE(test_gemm_transposeA) + // no filter +CASE(test_gemm_transposeB) + // no filter +CASE(test_globalaveragepool) + // no filter +CASE(test_globalaveragepool_precomputed) + // no filter +CASE(test_globalmaxpool) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_globalmaxpool_precomputed) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_greater) + // no filter +CASE(test_greater_bcast) + // no filter +CASE(test_greater_equal) + // no filter +CASE(test_greater_equal_bcast) + // no filter +CASE(test_greater_equal_bcast_expanded) + // no filter +CASE(test_greater_equal_expanded) + // no filter +CASE(test_gridsample) + // no filter +CASE(test_gridsample_aligncorners_true) + // no filter +CASE(test_gridsample_bicubic) + // no filter +CASE(test_gridsample_bilinear) + // no filter +CASE(test_gridsample_border_padding) + // no filter +CASE(test_gridsample_nearest) + // no filter +CASE(test_gridsample_reflection_padding) + // no filter +CASE(test_gridsample_zeros_padding) + // no filter +CASE(test_gru_batchwise) + // no filter +CASE(test_gru_defaults) + // no filter +CASE(test_gru_seq_length) + // no filter +CASE(test_gru_with_initial_bias) + // no filter +CASE(test_hardmax_axis_0) + // no filter +CASE(test_hardmax_axis_1) + // no filter +CASE(test_hardmax_axis_2) + // no filter +CASE(test_hardmax_default_axis) + // no filter +CASE(test_hardmax_example) + // no filter +CASE(test_hardmax_negative_axis) + // no filter +CASE(test_hardmax_one_hot) + // no filter +CASE(test_hardsigmoid) + // no filter +CASE(test_hardsigmoid_default) + // no filter +CASE(test_hardsigmoid_example) + // no filter +CASE(test_hardswish) + // no filter +CASE(test_hardswish_expanded) + // no filter +CASE(test_identity) + // no filter +CASE(test_identity_opt) + // no filter +CASE(test_identity_sequence) + // no filter +CASE(test_if) + // no filter +CASE(test_if_opt) + // no filter +CASE(test_if_seq) + // no filter +CASE(test_instancenorm_epsilon) + // no filter +CASE(test_instancenorm_example) + // no filter +CASE(test_isinf) + // no filter +CASE(test_isinf_negative) + // no filter +CASE(test_isinf_positive) + // no filter +CASE(test_isnan) + // no filter +CASE(test_leakyrelu) + // no filter +CASE(test_leakyrelu_default) + // no filter +CASE(test_leakyrelu_example) + // no filter +CASE(test_less) + // no filter +CASE(test_less_bcast) + // no filter +CASE(test_less_equal) + // no filter +CASE(test_less_equal_bcast) + // no filter +CASE(test_less_equal_bcast_expanded) + // no filter +CASE(test_less_equal_expanded) + // no filter +CASE(test_log) + // no filter +CASE(test_log_example) + // no filter +CASE(test_logsoftmax_axis_0) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_logsoftmax_axis_0_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_logsoftmax_axis_1) + // no filter +CASE(test_logsoftmax_axis_1_expanded) + // no filter +CASE(test_logsoftmax_axis_2) + // no filter +CASE(test_logsoftmax_axis_2_expanded) + // no filter +CASE(test_logsoftmax_default_axis) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_logsoftmax_default_axis_expanded) + // no filter +CASE(test_logsoftmax_example_1) + // no filter +CASE(test_logsoftmax_example_1_expanded) + // no filter +CASE(test_logsoftmax_large_number) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_logsoftmax_large_number_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_logsoftmax_negative_axis) + // no filter +CASE(test_logsoftmax_negative_axis_expanded) + // no filter +CASE(test_loop11) + // no filter +CASE(test_loop13_seq) + // no filter +CASE(test_loop16_seq_none) + // no filter +CASE(test_lrn) + // no filter +CASE(test_lrn_default) + // no filter +CASE(test_lstm_batchwise) + // no filter +CASE(test_lstm_defaults) + // no filter +CASE(test_lstm_with_initial_bias) + // no filter +CASE(test_lstm_with_peepholes) + // no filter +CASE(test_matmul_2d) + // no filter +CASE(test_matmul_3d) + // no filter +CASE(test_matmul_4d) + // no filter +CASE(test_matmulinteger) + // no filter +CASE(test_max_example) + // no filter +CASE(test_max_float16) + // no filter +CASE(test_max_float32) + // no filter +CASE(test_max_float64) + // no filter +CASE(test_max_int16) + // no filter +CASE(test_max_int32) + // no filter +CASE(test_max_int64) + // no filter +CASE(test_max_int8) + // no filter +CASE(test_max_one_input) + // no filter +CASE(test_max_two_inputs) + // no filter +CASE(test_max_uint16) + // no filter +CASE(test_max_uint32) + // no filter +CASE(test_max_uint64) + // no filter +CASE(test_max_uint8) + // no filter +CASE(test_maxpool_1d_default) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_ceil) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_default) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_dilations) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxpool_2d_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_precomputed_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_precomputed_same_upper) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_precomputed_strides) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_same_lower) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxpool_2d_same_upper) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_strides) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_MYRIAD; +#endif +CASE(test_maxpool_2d_uint8) + // no filter +CASE(test_maxpool_3d_default) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_NON_CPU; +#endif +CASE(test_maxpool_with_argmax_2d_precomputed_pads) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxpool_with_argmax_2d_precomputed_strides) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_maxunpool_export_with_output_shape) + // no filter +CASE(test_maxunpool_export_without_output_shape) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_mean_example) + // no filter +CASE(test_mean_one_input) + // no filter +CASE(test_mean_two_inputs) + // no filter +CASE(test_min_example) + // no filter +CASE(test_min_float16) + // no filter +CASE(test_min_float32) + // no filter +CASE(test_min_float64) + // no filter +CASE(test_min_int16) + // no filter +CASE(test_min_int32) + // no filter +CASE(test_min_int64) + // no filter +CASE(test_min_int8) + // no filter +CASE(test_min_one_input) + // no filter +CASE(test_min_two_inputs) + // no filter +CASE(test_min_uint16) + // no filter +CASE(test_min_uint32) + // no filter +CASE(test_min_uint64) + // no filter +CASE(test_min_uint8) + // no filter +CASE(test_mod_broadcast) + // no filter +CASE(test_mod_int64_fmod) + // no filter +CASE(test_mod_mixed_sign_float16) + // no filter +CASE(test_mod_mixed_sign_float32) + // no filter +CASE(test_mod_mixed_sign_float64) + // no filter +CASE(test_mod_mixed_sign_int16) + // no filter +CASE(test_mod_mixed_sign_int32) + // no filter +CASE(test_mod_mixed_sign_int64) + // no filter +CASE(test_mod_mixed_sign_int8) + // no filter +CASE(test_mod_uint16) + // no filter +CASE(test_mod_uint32) + // no filter +CASE(test_mod_uint64) + // no filter +CASE(test_mod_uint8) + // no filter +CASE(test_momentum) + // no filter +CASE(test_momentum_multiple) + // no filter +CASE(test_mul) + // no filter +CASE(test_mul_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_mul_example) + // no filter +CASE(test_mul_uint8) + // no filter +CASE(test_mvn) + // no filter +CASE(test_mvn_expanded) + // no filter +CASE(test_neg) + // no filter +CASE(test_neg_example) + // no filter +CASE(test_nesterov_momentum) + // no filter +CASE(test_nllloss_NC) + // no filter +CASE(test_nllloss_NC_expanded) + // no filter +CASE(test_nllloss_NCd1) + // no filter +CASE(test_nllloss_NCd1_expanded) + // no filter +CASE(test_nllloss_NCd1_ii) + // no filter +CASE(test_nllloss_NCd1_ii_expanded) + // no filter +CASE(test_nllloss_NCd1_mean_weight_negative_ii) + // no filter +CASE(test_nllloss_NCd1_mean_weight_negative_ii_expanded) + // no filter +CASE(test_nllloss_NCd1_weight) + // no filter +CASE(test_nllloss_NCd1_weight_expanded) + // no filter +CASE(test_nllloss_NCd1_weight_ii) + // no filter +CASE(test_nllloss_NCd1_weight_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2) + // no filter +CASE(test_nllloss_NCd1d2_expanded) + // no filter +CASE(test_nllloss_NCd1d2_no_weight_reduction_mean_ii) + // no filter +CASE(test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2_reduction_mean) + // no filter +CASE(test_nllloss_NCd1d2_reduction_mean_expanded) + // no filter +CASE(test_nllloss_NCd1d2_reduction_sum) + // no filter +CASE(test_nllloss_NCd1d2_reduction_sum_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_mean) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_mean_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum_expanded) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum_ii) + // no filter +CASE(test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3_none_no_weight_negative_ii) + // no filter +CASE(test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3_sum_weight_high_ii) + // no filter +CASE(test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_mean_weight) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_mean_weight_expanded) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight) + // no filter +CASE(test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded) + // no filter +CASE(test_nonmaxsuppression_center_point_box_format) + // no filter +CASE(test_nonmaxsuppression_flipped_coordinates) + // no filter +CASE(test_nonmaxsuppression_identical_boxes) + // no filter +CASE(test_nonmaxsuppression_limit_output_size) + // no filter +CASE(test_nonmaxsuppression_single_box) + // no filter +CASE(test_nonmaxsuppression_suppress_by_IOU) + // no filter +CASE(test_nonmaxsuppression_suppress_by_IOU_and_scores) + // no filter +CASE(test_nonmaxsuppression_two_batches) + // no filter +CASE(test_nonmaxsuppression_two_classes) + // no filter +CASE(test_nonzero_example) + // no filter +CASE(test_not_2d) + // no filter +CASE(test_not_3d) + // no filter +CASE(test_not_4d) + // no filter +CASE(test_onehot_negative_indices) + // no filter +CASE(test_onehot_with_axis) + // no filter +CASE(test_onehot_with_negative_axis) + // no filter +CASE(test_onehot_without_axis) + // no filter +CASE(test_optional_get_element) + // no filter +CASE(test_optional_get_element_sequence) + // no filter +CASE(test_optional_has_element) + // no filter +CASE(test_optional_has_element_empty) + // no filter +CASE(test_or2d) + // no filter +CASE(test_or3d) + // no filter +CASE(test_or4d) + // no filter +CASE(test_or_bcast3v1d) + // no filter +CASE(test_or_bcast3v2d) + // no filter +CASE(test_or_bcast4v2d) + // no filter +CASE(test_or_bcast4v3d) + // no filter +CASE(test_or_bcast4v4d) + // no filter +CASE(test_pow) + // no filter +CASE(test_pow_bcast_array) + // no filter +CASE(test_pow_bcast_scalar) + // no filter +CASE(test_pow_example) + // no filter +CASE(test_pow_types_float) + // no filter +CASE(test_pow_types_float32_int32) + // no filter +CASE(test_pow_types_float32_int64) + // no filter +CASE(test_pow_types_float32_uint32) + // no filter +CASE(test_pow_types_float32_uint64) + // no filter +CASE(test_pow_types_int) + // no filter +CASE(test_pow_types_int32_float32) + // no filter +CASE(test_pow_types_int32_int32) + // no filter +CASE(test_pow_types_int64_float32) + // no filter +CASE(test_pow_types_int64_int64) + // no filter +CASE(test_prelu_broadcast) + // no filter +CASE(test_prelu_example) + // no filter +CASE(test_qlinearconv) + // no filter +CASE(test_qlinearmatmul_2D) + // no filter +CASE(test_qlinearmatmul_3D) + // no filter +CASE(test_quantizelinear) + // no filter +CASE(test_quantizelinear_axis) + // no filter +CASE(test_range_float_type_positive_delta) + // no filter +CASE(test_range_float_type_positive_delta_expanded) + // no filter +CASE(test_range_int32_type_negative_delta) + // no filter +CASE(test_range_int32_type_negative_delta_expanded) + // no filter +CASE(test_reciprocal) + // no filter +CASE(test_reciprocal_example) + // no filter +CASE(test_reduce_l1_default_axes_keepdims_example) + // no filter +CASE(test_reduce_l1_default_axes_keepdims_random) + // no filter +CASE(test_reduce_l1_do_not_keepdims_example) + // no filter +CASE(test_reduce_l1_do_not_keepdims_random) + // no filter +CASE(test_reduce_l1_keep_dims_example) + // no filter +CASE(test_reduce_l1_keep_dims_random) + // no filter +CASE(test_reduce_l1_negative_axes_keep_dims_example) + // no filter +CASE(test_reduce_l1_negative_axes_keep_dims_random) + // no filter +CASE(test_reduce_l2_default_axes_keepdims_example) + // no filter +CASE(test_reduce_l2_default_axes_keepdims_random) + // no filter +CASE(test_reduce_l2_do_not_keepdims_example) + // no filter +CASE(test_reduce_l2_do_not_keepdims_random) + // no filter +CASE(test_reduce_l2_keep_dims_example) + // no filter +CASE(test_reduce_l2_keep_dims_random) + // no filter +CASE(test_reduce_l2_negative_axes_keep_dims_example) + // no filter +CASE(test_reduce_l2_negative_axes_keep_dims_random) + // no filter +CASE(test_reduce_log_sum) + // no filter +CASE(test_reduce_log_sum_asc_axes) + // no filter +CASE(test_reduce_log_sum_default) + // no filter +CASE(test_reduce_log_sum_desc_axes) + // no filter +CASE(test_reduce_log_sum_exp_default_axes_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_default_axes_keepdims_random) + // no filter +CASE(test_reduce_log_sum_exp_do_not_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_do_not_keepdims_random) + // no filter +CASE(test_reduce_log_sum_exp_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_keepdims_random) + // no filter +CASE(test_reduce_log_sum_exp_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_log_sum_exp_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_log_sum_negative_axes) + // no filter +CASE(test_reduce_max_default_axes_keepdim_example) + // no filter +CASE(test_reduce_max_default_axes_keepdims_random) + // no filter +CASE(test_reduce_max_do_not_keepdims_example) + // no filter +CASE(test_reduce_max_do_not_keepdims_random) + // no filter +CASE(test_reduce_max_keepdims_example) + // no filter +CASE(test_reduce_max_keepdims_random) + // no filter +CASE(test_reduce_max_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_max_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_mean_default_axes_keepdims_example) + // no filter +CASE(test_reduce_mean_default_axes_keepdims_random) + // no filter +CASE(test_reduce_mean_do_not_keepdims_example) + // no filter +CASE(test_reduce_mean_do_not_keepdims_random) + // no filter +CASE(test_reduce_mean_keepdims_example) + // no filter +CASE(test_reduce_mean_keepdims_random) + // no filter +CASE(test_reduce_mean_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_mean_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_min_default_axes_keepdims_example) + // no filter +CASE(test_reduce_min_default_axes_keepdims_random) + // no filter +CASE(test_reduce_min_do_not_keepdims_example) + // no filter +CASE(test_reduce_min_do_not_keepdims_random) + // no filter +CASE(test_reduce_min_keepdims_example) + // no filter +CASE(test_reduce_min_keepdims_random) + // no filter +CASE(test_reduce_min_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_min_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_prod_default_axes_keepdims_example) + // no filter +CASE(test_reduce_prod_default_axes_keepdims_random) + // no filter +CASE(test_reduce_prod_do_not_keepdims_example) + // no filter +CASE(test_reduce_prod_do_not_keepdims_random) + // no filter +CASE(test_reduce_prod_keepdims_example) + // no filter +CASE(test_reduce_prod_keepdims_random) + // no filter +CASE(test_reduce_prod_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_prod_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_default_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_default_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_do_not_keepdims_example) + // no filter +CASE(test_reduce_sum_do_not_keepdims_random) + // no filter +CASE(test_reduce_sum_empty_axes_input_noop_example) + // no filter +CASE(test_reduce_sum_empty_axes_input_noop_random) + // no filter +CASE(test_reduce_sum_keepdims_example) + // no filter +CASE(test_reduce_sum_keepdims_random) + // no filter +CASE(test_reduce_sum_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_negative_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_square_default_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_square_default_axes_keepdims_random) + // no filter +CASE(test_reduce_sum_square_do_not_keepdims_example) + // no filter +CASE(test_reduce_sum_square_do_not_keepdims_random) + // no filter +CASE(test_reduce_sum_square_keepdims_example) + // no filter +CASE(test_reduce_sum_square_keepdims_random) + // no filter +CASE(test_reduce_sum_square_negative_axes_keepdims_example) + // no filter +CASE(test_reduce_sum_square_negative_axes_keepdims_random) + // no filter +CASE(test_reflect_pad) + // no filter +CASE(test_relu) + // no filter +CASE(test_reshape_allowzero_reordered) + // no filter +CASE(test_reshape_extended_dims) + // no filter +CASE(test_reshape_negative_dim) + // no filter +CASE(test_reshape_negative_extended_dims) + // no filter +CASE(test_reshape_one_dim) + // no filter +CASE(test_reshape_reduced_dims) + // no filter +CASE(test_reshape_reordered_all_dims) + // no filter +CASE(test_reshape_reordered_last_dims) + // no filter +CASE(test_reshape_zero_and_negative_dim) + // no filter +CASE(test_reshape_zero_dim) + // no filter +CASE(test_resize_downsample_scales_cubic) + // no filter +CASE(test_resize_downsample_scales_cubic_A_n0p5_exclude_outside) + // no filter +CASE(test_resize_downsample_scales_cubic_align_corners) + // no filter +CASE(test_resize_downsample_scales_linear) + // no filter +CASE(test_resize_downsample_scales_linear_align_corners) + // no filter +CASE(test_resize_downsample_scales_nearest) + // no filter +CASE(test_resize_downsample_sizes_cubic) + // no filter +CASE(test_resize_downsample_sizes_linear_pytorch_half_pixel) + // no filter +CASE(test_resize_downsample_sizes_nearest) + // no filter +CASE(test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn) + // no filter +CASE(test_resize_tf_crop_and_resize) + // no filter +CASE(test_resize_upsample_scales_cubic) + // no filter +CASE(test_resize_upsample_scales_cubic_A_n0p5_exclude_outside) + // no filter +CASE(test_resize_upsample_scales_cubic_align_corners) + // no filter +CASE(test_resize_upsample_scales_cubic_asymmetric) + // no filter +CASE(test_resize_upsample_scales_linear) + // no filter +CASE(test_resize_upsample_scales_linear_align_corners) + // no filter +CASE(test_resize_upsample_scales_nearest) + // no filter +CASE(test_resize_upsample_sizes_cubic) + // no filter +CASE(test_resize_upsample_sizes_nearest) + // no filter +CASE(test_resize_upsample_sizes_nearest_ceil_half_pixel) + // no filter +CASE(test_resize_upsample_sizes_nearest_floor_align_corners) + // no filter +CASE(test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric) + // no filter +CASE(test_reversesequence_batch) + // no filter +CASE(test_reversesequence_time) + // no filter +CASE(test_rnn_seq_length) + // no filter +CASE(test_roialign_aligned_false) + // no filter +CASE(test_roialign_aligned_true) + // no filter +CASE(test_round) + // no filter +CASE(test_scan9_sum) + // no filter +CASE(test_scan_sum) + // no filter +CASE(test_scatter_elements_with_axis) + // no filter +CASE(test_scatter_elements_with_duplicate_indices) + // no filter +CASE(test_scatter_elements_with_negative_indices) + // no filter +CASE(test_scatter_elements_without_axis) + // no filter +CASE(test_scatter_with_axis) + // no filter +CASE(test_scatter_without_axis) + // no filter +CASE(test_scatternd) + // no filter +CASE(test_scatternd_add) + // no filter +CASE(test_scatternd_multiply) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii_expanded) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii_log_prob) + // no filter +CASE(test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob) + // no filter +CASE(test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii_expanded) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii_log_prob) + // no filter +CASE(test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight_log_prob) + // no filter +CASE(test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight_expanded) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight_log_prob) + // no filter +CASE(test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded) + // no filter +CASE(test_sce_mean) + // no filter +CASE(test_sce_mean_3d) + // no filter +CASE(test_sce_mean_3d_expanded) + // no filter +CASE(test_sce_mean_3d_log_prob) + // no filter +CASE(test_sce_mean_3d_log_prob_expanded) + // no filter +CASE(test_sce_mean_expanded) + // no filter +CASE(test_sce_mean_log_prob) + // no filter +CASE(test_sce_mean_log_prob_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii) + // no filter +CASE(test_sce_mean_no_weight_ii_3d) + // no filter +CASE(test_sce_mean_no_weight_ii_3d_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_3d_log_prob) + // no filter +CASE(test_sce_mean_no_weight_ii_3d_log_prob_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_4d) + // no filter +CASE(test_sce_mean_no_weight_ii_4d_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_4d_log_prob) + // no filter +CASE(test_sce_mean_no_weight_ii_4d_log_prob_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_expanded) + // no filter +CASE(test_sce_mean_no_weight_ii_log_prob) + // no filter +CASE(test_sce_mean_no_weight_ii_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight) + // no filter +CASE(test_sce_mean_weight_expanded) + // no filter +CASE(test_sce_mean_weight_ii) + // no filter +CASE(test_sce_mean_weight_ii_3d) + // no filter +CASE(test_sce_mean_weight_ii_3d_expanded) + // no filter +CASE(test_sce_mean_weight_ii_3d_log_prob) + // no filter +CASE(test_sce_mean_weight_ii_3d_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight_ii_4d) + // no filter +CASE(test_sce_mean_weight_ii_4d_expanded) + // no filter +CASE(test_sce_mean_weight_ii_4d_log_prob) + // no filter +CASE(test_sce_mean_weight_ii_4d_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight_ii_expanded) + // no filter +CASE(test_sce_mean_weight_ii_log_prob) + // no filter +CASE(test_sce_mean_weight_ii_log_prob_expanded) + // no filter +CASE(test_sce_mean_weight_log_prob) + // no filter +CASE(test_sce_mean_weight_log_prob_expanded) + // no filter +CASE(test_sce_none) + // no filter +CASE(test_sce_none_expanded) + // no filter +CASE(test_sce_none_log_prob) + // no filter +CASE(test_sce_none_log_prob_expanded) + // no filter +CASE(test_sce_none_weights) + // no filter +CASE(test_sce_none_weights_expanded) + // no filter +CASE(test_sce_none_weights_log_prob) + // no filter +CASE(test_sce_none_weights_log_prob_expanded) + // no filter +CASE(test_sce_sum) + // no filter +CASE(test_sce_sum_expanded) + // no filter +CASE(test_sce_sum_log_prob) + // no filter +CASE(test_sce_sum_log_prob_expanded) + // no filter +CASE(test_selu) + // no filter +CASE(test_selu_default) + // no filter +CASE(test_selu_example) + // no filter +CASE(test_sequence_insert_at_back) + // no filter +CASE(test_sequence_insert_at_front) + // no filter +CASE(test_shape) + // no filter +CASE(test_shape_clip_end) + // no filter +CASE(test_shape_clip_start) + // no filter +CASE(test_shape_end_1) + // no filter +CASE(test_shape_end_negative_1) + // no filter +CASE(test_shape_example) + // no filter +CASE(test_shape_start_1) + // no filter +CASE(test_shape_start_1_end_2) + // no filter +CASE(test_shape_start_1_end_negative_1) + // no filter +CASE(test_shape_start_negative_1) + // no filter +CASE(test_shrink_hard) + // no filter +CASE(test_shrink_soft) + // no filter +CASE(test_sigmoid) + // no filter +CASE(test_sigmoid_example) + // no filter +CASE(test_sign) + // no filter +CASE(test_simple_rnn_batchwise) + // no filter +CASE(test_simple_rnn_defaults) + // no filter +CASE(test_simple_rnn_with_initial_bias) + // no filter +CASE(test_sin) + // no filter +CASE(test_sin_example) + // no filter +CASE(test_sinh) + // no filter +CASE(test_sinh_example) + // no filter +CASE(test_size) + // no filter +CASE(test_size_example) + // no filter +CASE(test_slice) + // no filter +CASE(test_slice_default_axes) + // no filter +CASE(test_slice_default_steps) + // no filter +CASE(test_slice_end_out_of_bounds) + // no filter +CASE(test_slice_neg) + // no filter +CASE(test_slice_neg_steps) + // no filter +CASE(test_slice_negative_axes) + // no filter +CASE(test_slice_start_out_of_bounds) + // no filter +CASE(test_softmax_axis_0) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_softmax_axis_0_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_softmax_axis_1) + // no filter +CASE(test_softmax_axis_1_expanded) + // no filter +CASE(test_softmax_axis_2) + // no filter +CASE(test_softmax_axis_2_expanded) + // no filter +CASE(test_softmax_default_axis) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_softmax_default_axis_expanded) + // no filter +CASE(test_softmax_example) + // no filter +CASE(test_softmax_example_expanded) + // no filter +CASE(test_softmax_large_number) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_softmax_large_number_expanded) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_OPENCL_FP16; + SKIP_MYRIAD; +#endif +CASE(test_softmax_negative_axis) + // no filter +CASE(test_softmax_negative_axis_expanded) + // no filter +CASE(test_softplus) + // no filter +CASE(test_softplus_example) + // no filter +CASE(test_softsign) + // no filter +CASE(test_softsign_example) + // no filter +CASE(test_spacetodepth) + // no filter +CASE(test_spacetodepth_example) + // no filter +CASE(test_split_equal_parts_1d) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_split_equal_parts_2d) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_split_equal_parts_default_axis) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP_CPU; + // MYRIAD is ok + SKIP_OPENCL; + SKIP_OPENCL_FP16; +#endif +CASE(test_split_variable_parts_1d) + // no filter +CASE(test_split_variable_parts_2d) + // no filter +CASE(test_split_variable_parts_default_axis) + // no filter +CASE(test_split_zero_size_splits) + // no filter +CASE(test_sqrt) + // no filter +CASE(test_sqrt_example) + // no filter +CASE(test_squeeze) + // no filter +CASE(test_squeeze_negative_axes) + // no filter +CASE(test_strnormalizer_export_monday_casesensintive_lower) + // no filter +CASE(test_strnormalizer_export_monday_casesensintive_nochangecase) + // no filter +CASE(test_strnormalizer_export_monday_casesensintive_upper) + // no filter +CASE(test_strnormalizer_export_monday_empty_output) + // no filter +CASE(test_strnormalizer_export_monday_insensintive_upper_twodim) + // no filter +CASE(test_strnormalizer_nostopwords_nochangecase) + // no filter +CASE(test_sub) + // no filter +CASE(test_sub_bcast) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_sub_example) + // no filter +CASE(test_sub_uint8) + // no filter +CASE(test_sum_example) + // no filter +CASE(test_sum_one_input) + // no filter +CASE(test_sum_two_inputs) + // no filter +CASE(test_tan) + // no filter +CASE(test_tan_example) + // no filter +CASE(test_tanh) + // no filter +CASE(test_tanh_example) + // no filter +CASE(test_tfidfvectorizer_tf_batch_onlybigrams_skip0) + // no filter +CASE(test_tfidfvectorizer_tf_batch_onlybigrams_skip5) + // no filter +CASE(test_tfidfvectorizer_tf_batch_uniandbigrams_skip5) + // no filter +CASE(test_tfidfvectorizer_tf_only_bigrams_skip0) + // no filter +CASE(test_tfidfvectorizer_tf_onlybigrams_levelempty) + // no filter +CASE(test_tfidfvectorizer_tf_onlybigrams_skip5) + // no filter +CASE(test_tfidfvectorizer_tf_uniandbigrams_skip5) + // no filter +CASE(test_thresholdedrelu) + // no filter +CASE(test_thresholdedrelu_default) + // no filter +CASE(test_thresholdedrelu_example) + // no filter +CASE(test_tile) + // no filter +CASE(test_tile_precomputed) + // no filter +CASE(test_top_k) + // no filter +CASE(test_top_k_negative_axis) + // no filter +CASE(test_top_k_smallest) + // no filter +CASE(test_training_dropout) + // no filter +CASE(test_training_dropout_default) + // no filter +CASE(test_training_dropout_default_mask) + // no filter +CASE(test_training_dropout_mask) + // no filter +CASE(test_training_dropout_zero_ratio) + // no filter +CASE(test_training_dropout_zero_ratio_mask) + // no filter +CASE(test_transpose_all_permutations_0) + // no filter +CASE(test_transpose_all_permutations_1) + // no filter +CASE(test_transpose_all_permutations_2) + // no filter +CASE(test_transpose_all_permutations_3) + // no filter +CASE(test_transpose_all_permutations_4) + // no filter +CASE(test_transpose_all_permutations_5) + // no filter +CASE(test_transpose_default) + // no filter +CASE(test_tril) + // no filter +CASE(test_tril_neg) + // no filter +CASE(test_tril_one_row_neg) + // no filter +CASE(test_tril_out_neg) + // no filter +CASE(test_tril_out_pos) + // no filter +CASE(test_tril_pos) + // no filter +CASE(test_tril_square) + // no filter +CASE(test_tril_square_neg) + // no filter +CASE(test_tril_zero) + // no filter +CASE(test_triu) + // no filter +CASE(test_triu_neg) + // no filter +CASE(test_triu_one_row) + // no filter +CASE(test_triu_out_neg_out) + // no filter +CASE(test_triu_out_pos) + // no filter +CASE(test_triu_pos) + // no filter +CASE(test_triu_square) + // no filter +CASE(test_triu_square_neg) + // no filter +CASE(test_triu_zero) + // no filter +CASE(test_unique_not_sorted_without_axis) + // no filter +CASE(test_unique_sorted_with_axis) + // no filter +CASE(test_unique_sorted_with_axis_3d) + // no filter +CASE(test_unique_sorted_with_negative_axis) + // no filter +CASE(test_unique_sorted_without_axis) + // no filter +CASE(test_unsqueeze_axis_0) + // no filter +CASE(test_unsqueeze_axis_1) + // no filter +CASE(test_unsqueeze_axis_2) + // no filter +CASE(test_unsqueeze_axis_3) + // no filter +CASE(test_unsqueeze_negative_axes) + // no filter +CASE(test_unsqueeze_three_axes) + // no filter +CASE(test_unsqueeze_two_axes) + // no filter +CASE(test_unsqueeze_unsorted_axes) + // no filter +CASE(test_upsample_nearest) +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + SKIP; +#endif +CASE(test_where_example) + // no filter +CASE(test_where_long_example) + // no filter +CASE(test_xor2d) + // no filter +CASE(test_xor3d) + // no filter +CASE(test_xor4d) + // no filter +CASE(test_xor_bcast3v1d) + // no filter +CASE(test_xor_bcast3v2d) + // no filter +CASE(test_xor_bcast4v2d) + // no filter +CASE(test_xor_bcast4v3d) + // no filter +CASE(test_xor_bcast4v4d) + // no filter +END_SWITCH() +#undef EOF_LABEL +#undef BEGIN_SWITCH +#undef CASE +#undef END_SWITCH +if (!filterApplied) +{ + ADD_FAILURE() << "OpenVINO backend: unknown test='" << name << "'. Update filter configuration"; +} + +#undef SKIP_TAGS +#undef SKIP_ +#undef SKIP +#undef SKIP_CPU +#undef SKIP_NON_CPU +#undef SKIP_OPENCL +#undef SKIP_OPENCL_FP16 +#undef SKIP_MYRIAD + +#endif