From b8af7c5f86068b1b8d945e84759283e8ab471dff Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 21 Sep 2017 18:50:54 +0300 Subject: [PATCH 1/2] ts: update perf test - use GTest tuple definitions instead of std::tr1 - use "const static" for cv::Size contants to reduce generated binary code - PERF_TEST_P() violates TEST_P() original semantic. Added PERF_TEST_P_() macro --- modules/ts/include/opencv2/ts.hpp | 11 ++- modules/ts/include/opencv2/ts/cuda_perf.hpp | 4 +- modules/ts/include/opencv2/ts/ocl_perf.hpp | 3 - modules/ts/include/opencv2/ts/ocl_test.hpp | 2 +- modules/ts/include/opencv2/ts/ts_ext.hpp | 27 ++++--- modules/ts/include/opencv2/ts/ts_perf.hpp | 84 +++++++++++++-------- 6 files changed, 81 insertions(+), 50 deletions(-) diff --git a/modules/ts/include/opencv2/ts.hpp b/modules/ts/include/opencv2/ts.hpp index 46a555e9ef..d630a5a384 100644 --- a/modules/ts/include/opencv2/ts.hpp +++ b/modules/ts/include/opencv2/ts.hpp @@ -58,8 +58,8 @@ # define GTEST_USES_POSIX_RE 0 #endif -#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< std::tr1::tuple< __VA_ARGS__ > > -#define GET_PARAM(k) std::tr1::get< k >(GetParam()) +#define PARAM_TEST_CASE(name, ...) struct name : testing::TestWithParam< testing::tuple< __VA_ARGS__ > > +#define GET_PARAM(k) testing::get< k >(GetParam()) namespace cvtest { @@ -70,6 +70,13 @@ using namespace cv; using testing::Values; using testing::Combine; +// Tuple stuff from Google Tests +using testing::get; +using testing::make_tuple; +using testing::tuple; +using testing::tuple_size; +using testing::tuple_element; + class SkipTestException: public cv::Exception { diff --git a/modules/ts/include/opencv2/ts/cuda_perf.hpp b/modules/ts/include/opencv2/ts/cuda_perf.hpp index 3c162adc8d..c0c2b7dad5 100644 --- a/modules/ts/include/opencv2/ts/cuda_perf.hpp +++ b/modules/ts/include/opencv2/ts/cuda_perf.hpp @@ -62,9 +62,9 @@ namespace perf #define CUDA_CHANNELS_1_3_4 testing::Values(MatCn(Gray), MatCn(BGR), MatCn(BGRA)) #define CUDA_CHANNELS_1_3 testing::Values(MatCn(Gray), MatCn(BGR)) - #define GET_PARAM(k) std::tr1::get< k >(GetParam()) + #define GET_PARAM(k) testing::get< k >(GetParam()) - #define DEF_PARAM_TEST(name, ...) typedef ::perf::TestBaseWithParam< std::tr1::tuple< __VA_ARGS__ > > name + #define DEF_PARAM_TEST(name, ...) typedef ::perf::TestBaseWithParam< testing::tuple< __VA_ARGS__ > > name #define DEF_PARAM_TEST_1(name, param_type) typedef ::perf::TestBaseWithParam< param_type > name DEF_PARAM_TEST_1(Sz, cv::Size); diff --git a/modules/ts/include/opencv2/ts/ocl_perf.hpp b/modules/ts/include/opencv2/ts/ocl_perf.hpp index 4d57aaad21..ce25b7193b 100644 --- a/modules/ts/include/opencv2/ts/ocl_perf.hpp +++ b/modules/ts/include/opencv2/ts/ocl_perf.hpp @@ -52,9 +52,6 @@ namespace ocl { using namespace perf; -using std::tr1::get; -using std::tr1::tuple; - #define OCL_PERF_STRATEGY PERF_STRATEGY_SIMPLE #define OCL_PERF_TEST(fixture, name) SIMPLE_PERF_TEST(fixture, name) diff --git a/modules/ts/include/opencv2/ts/ocl_test.hpp b/modules/ts/include/opencv2/ts/ocl_test.hpp index 54b33ece85..294fab472c 100644 --- a/modules/ts/include/opencv2/ts/ocl_test.hpp +++ b/modules/ts/include/opencv2/ts/ocl_test.hpp @@ -325,7 +325,7 @@ struct CV_EXPORTS TSTestWithParam : public TestUtils, public ::testing::TestWith }; #undef PARAM_TEST_CASE -#define PARAM_TEST_CASE(name, ...) struct name : public ::cvtest::ocl::TSTestWithParam< std::tr1::tuple< __VA_ARGS__ > > +#define PARAM_TEST_CASE(name, ...) struct name : public ::cvtest::ocl::TSTestWithParam< testing::tuple< __VA_ARGS__ > > #ifndef IMPLEMENT_PARAM_CLASS #define IMPLEMENT_PARAM_CLASS(name, type) \ diff --git a/modules/ts/include/opencv2/ts/ts_ext.hpp b/modules/ts/include/opencv2/ts/ts_ext.hpp index 0bdd346ddf..259f869f45 100644 --- a/modules/ts/include/opencv2/ts/ts_ext.hpp +++ b/modules/ts/include/opencv2/ts/ts_ext.hpp @@ -12,17 +12,17 @@ namespace cvtest { void checkIppStatus(); } -#define CV_TEST_INIT \ +#define CV__TEST_INIT \ cv::ipp::setIppStatus(0); \ cv::theRNG().state = cvtest::param_seed; -#define CV_TEST_CLEANUP ::cvtest::checkIppStatus(); -#define CV_TEST_BODY_IMPL(name) \ +#define CV__TEST_CLEANUP ::cvtest::checkIppStatus(); +#define CV__TEST_BODY_IMPL(name) \ { \ CV__TRACE_APP_FUNCTION_NAME(name); \ try { \ - CV_TEST_INIT \ + CV__TEST_INIT \ Body(); \ - CV_TEST_CLEANUP \ + CV__TEST_CLEANUP \ } \ catch (cvtest::SkipTestException& e) \ { \ @@ -54,7 +54,7 @@ void checkIppStatus(); ::testing::Test::TearDownTestCase, \ new ::testing::internal::TestFactoryImpl<\ GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ - void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() CV_TEST_BODY_IMPL( #test_case_name "_" #test_name ) \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() CV__TEST_BODY_IMPL( #test_case_name "_" #test_name ) \ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::Body() #undef TEST_F @@ -80,17 +80,17 @@ void checkIppStatus(); test_fixture::TearDownTestCase, \ new ::testing::internal::TestFactoryImpl<\ GTEST_TEST_CLASS_NAME_(test_fixture, test_name)>);\ - void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::TestBody() CV_TEST_BODY_IMPL( #test_fixture "_" #test_name ) \ + void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::TestBody() CV__TEST_BODY_IMPL( #test_fixture "_" #test_name ) \ void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::Body() -#undef TEST_P -#define TEST_P(test_case_name, test_name) \ +// Don't use directly +#define CV__TEST_P(test_case_name, test_name, bodyMethodName, BODY_IMPL/*(name_str)*/) \ class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ : public test_case_name { \ public: \ GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ private: \ - virtual void Body(); \ + virtual void bodyMethodName(); \ virtual void TestBody(); \ static int AddToRegistry() { \ ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ @@ -112,7 +112,10 @@ void checkIppStatus(); int GTEST_TEST_CLASS_NAME_(test_case_name, \ test_name)::gtest_registering_dummy_ = \ GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ - void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() CV_TEST_BODY_IMPL( #test_case_name "_" #test_name ) \ - void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::Body() + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() BODY_IMPL( #test_case_name "_" #test_name ) \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::bodyMethodName() + +#undef TEST_P +#define TEST_P(test_case_name, test_name) CV__TEST_P(test_case_name, test_name, Body, CV__TEST_BODY_IMPL) #endif // OPENCV_TS_EXT_HPP diff --git a/modules/ts/include/opencv2/ts/ts_perf.hpp b/modules/ts/include/opencv2/ts/ts_perf.hpp index 37e15860dc..37813658df 100644 --- a/modules/ts/include/opencv2/ts/ts_perf.hpp +++ b/modules/ts/include/opencv2/ts/ts_perf.hpp @@ -27,40 +27,49 @@ // declare major namespaces to avoid errors on unknown namespace namespace cv { namespace cuda {} namespace ocl {} } +namespace cvtest { } namespace perf { + +// Tuple stuff from Google Tests +using testing::get; +using testing::make_tuple; +using testing::tuple; +using testing::tuple_size; +using testing::tuple_element; + class TestBase; /*****************************************************************************************\ * Predefined typical frame sizes and typical test parameters * \*****************************************************************************************/ -const cv::Size szQVGA = cv::Size(320, 240); -const cv::Size szVGA = cv::Size(640, 480); -const cv::Size szSVGA = cv::Size(800, 600); -const cv::Size szXGA = cv::Size(1024, 768); -const cv::Size szSXGA = cv::Size(1280, 1024); -const cv::Size szWQHD = cv::Size(2560, 1440); - -const cv::Size sznHD = cv::Size(640, 360); -const cv::Size szqHD = cv::Size(960, 540); -const cv::Size sz240p = szQVGA; -const cv::Size sz720p = cv::Size(1280, 720); -const cv::Size sz1080p = cv::Size(1920, 1080); -const cv::Size sz1440p = szWQHD; -const cv::Size sz2160p = cv::Size(3840, 2160);//UHDTV1 4K -const cv::Size sz4320p = cv::Size(7680, 4320);//UHDTV2 8K - -const cv::Size sz3MP = cv::Size(2048, 1536); -const cv::Size sz5MP = cv::Size(2592, 1944); -const cv::Size sz2K = cv::Size(2048, 2048); - -const cv::Size szODD = cv::Size(127, 61); - -const cv::Size szSmall24 = cv::Size(24, 24); -const cv::Size szSmall32 = cv::Size(32, 32); -const cv::Size szSmall64 = cv::Size(64, 64); -const cv::Size szSmall128 = cv::Size(128, 128); +const static cv::Size szQVGA = cv::Size(320, 240); +const static cv::Size szVGA = cv::Size(640, 480); +const static cv::Size szSVGA = cv::Size(800, 600); +const static cv::Size szXGA = cv::Size(1024, 768); +const static cv::Size szSXGA = cv::Size(1280, 1024); +const static cv::Size szWQHD = cv::Size(2560, 1440); + +const static cv::Size sznHD = cv::Size(640, 360); +const static cv::Size szqHD = cv::Size(960, 540); +const static cv::Size sz240p = szQVGA; +const static cv::Size sz720p = cv::Size(1280, 720); +const static cv::Size sz1080p = cv::Size(1920, 1080); +const static cv::Size sz1440p = szWQHD; +const static cv::Size sz2160p = cv::Size(3840, 2160);//UHDTV1 4K +const static cv::Size sz4320p = cv::Size(7680, 4320);//UHDTV2 8K + +const static cv::Size sz3MP = cv::Size(2048, 1536); +const static cv::Size sz5MP = cv::Size(2592, 1944); +const static cv::Size sz2K = cv::Size(2048, 2048); + +const static cv::Size szODD = cv::Size(127, 61); + +const static cv::Size szSmall24 = cv::Size(24, 24); +const static cv::Size szSmall32 = cv::Size(32, 32); +const static cv::Size szSmall64 = cv::Size(64, 64); +const static cv::Size szSmall128 = cv::Size(128, 128); #define SZ_ALL_VGA ::testing::Values(::perf::szQVGA, ::perf::szVGA, ::perf::szSVGA) #define SZ_ALL_GA ::testing::Values(::perf::szQVGA, ::perf::szVGA, ::perf::szSVGA, ::perf::szXGA, ::perf::szSXGA) @@ -492,7 +501,7 @@ public: template class TestBaseWithParam: public TestBase, public ::testing::WithParamInterface {}; -typedef std::tr1::tuple Size_MatType_t; +typedef tuple Size_MatType_t; typedef TestBaseWithParam Size_MatType; /*****************************************************************************************\ @@ -514,6 +523,13 @@ CV_EXPORTS void PrintTo(const Size& sz, ::std::ostream* os); /*****************************************************************************************\ * Macro definitions for performance tests * \*****************************************************************************************/ + +#define CV__PERF_TEST_BODY_IMPL(name) \ + { \ + CV__TRACE_APP_FUNCTION_NAME("PERF_TEST: " name); \ + RunPerfTestBody(); \ + } + #define PERF_PROXY_NAMESPACE_NAME_(test_case_name, test_name) \ test_case_name##_##test_name##_perf_namespace_proxy @@ -538,7 +554,7 @@ CV_EXPORTS void PrintTo(const Size& sz, ::std::ostream* os); protected:\ virtual void PerfTestBody();\ };\ - TEST_F(test_case_name, test_name){ CV_TRACE_REGION("PERF_TEST: " #test_case_name "_" #test_name); RunPerfTestBody(); }\ + TEST_F(test_case_name, test_name){ CV__PERF_TEST_BODY_IMPL(#test_case_name "_" #test_name); }\ }\ void PERF_PROXY_NAMESPACE_NAME_(test_case_name, test_name)::test_case_name::PerfTestBody() @@ -576,12 +592,20 @@ CV_EXPORTS void PrintTo(const Size& sz, ::std::ostream* os); protected:\ virtual void PerfTestBody();\ };\ - TEST_F(fixture, testname){ CV_TRACE_REGION("PERF_TEST: " #fixture "_" #testname); RunPerfTestBody(); }\ + TEST_F(fixture, testname){ CV__PERF_TEST_BODY_IMPL(#fixture "_" #testname); }\ }\ void PERF_PROXY_NAMESPACE_NAME_(fixture, testname)::fixture::PerfTestBody() // Defines a parametrized performance test. // +// @Note PERF_TEST_P() below violates behavior of original Google Tests - there is no tests instantiation in original TEST_P() +// This macro is intended for usage with separate INSTANTIATE_TEST_CASE_P macro +#define PERF_TEST_P_(test_case_name, test_name) CV__TEST_P(test_case_name, test_name, PerfTestBody, CV__PERF_TEST_BODY_IMPL) + +// Defines a parametrized performance test. +// +// @Note Original TEST_P() macro doesn't instantiate tests with parameters. To keep original usage use PERF_TEST_P_() macro +// // The first parameter is the name of the test fixture class, which // also doubles as the test case name. The second parameter is the // name of the test within the test case. @@ -609,7 +633,7 @@ CV_EXPORTS void PrintTo(const Size& sz, ::std::ostream* os); protected:\ virtual void PerfTestBody();\ };\ - TEST_P(fixture##_##name, name /*perf*/){ CV_TRACE_REGION("PERF_TEST: " #fixture "_" #name); RunPerfTestBody(); }\ + CV__TEST_P(fixture##_##name, name, PerfTestBodyDummy, CV__PERF_TEST_BODY_IMPL){} \ INSTANTIATE_TEST_CASE_P(/*none*/, fixture##_##name, params);\ void fixture##_##name::PerfTestBody() From 78788e1efbe6053efd7178b96fb7d3ef4a044c61 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 22 Sep 2017 15:15:57 +0300 Subject: [PATCH 2/2] dnn(perf): update perf tests --- modules/dnn/perf/perf_convolution.cpp | 27 ++-- modules/dnn/perf/perf_halide_net.cpp | 174 -------------------------- modules/dnn/perf/perf_net.cpp | 149 ++++++++++++++++++++++ modules/dnn/perf/perf_precomp.hpp | 13 +- 4 files changed, 162 insertions(+), 201 deletions(-) delete mode 100644 modules/dnn/perf/perf_halide_net.cpp create mode 100644 modules/dnn/perf/perf_net.cpp diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index 502c5ef822..7429885cdd 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -1,27 +1,15 @@ #include "perf_precomp.hpp" #include -namespace cvtest +namespace { -using std::tr1::tuple; -using std::tr1::get; -using std::tr1::make_tuple; -using std::make_pair; -using namespace perf; -using namespace testing; -using namespace cv; -using namespace cv::dnn; - enum {STRIDE_OFF = 1, STRIDE_ON = 2}; CV_ENUM(StrideSize, STRIDE_OFF, STRIDE_ON); enum {GROUP_OFF = 1, GROUP_2 = 2}; CV_ENUM(GroupSize, GROUP_OFF, GROUP_2); -//Squared Size -#define SSZ(n) cv::Size(n, n) - typedef std::pair InpShapeNumOut; typedef tuple ConvParam; //kernel_size, inp shape, groups, stride typedef TestBaseWithParam ConvolutionPerfTest; @@ -77,11 +65,11 @@ PERF_TEST_P( ConvolutionPerfTest, perf, Combine( Ptr layer = cv::dnn::LayerFactory::createLayerInstance("Convolution", lp); std::vector inputShapes(1, shape(inpBlob)), outShapes, internals; layer->getMemoryShapes(inputShapes, 0, outShapes, internals); - for (int i = 0; i < outShapes.size(); i++) + for (size_t i = 0; i < outShapes.size(); i++) { outBlobs.push_back(Mat(outShapes[i], CV_32F)); } - for (int i = 0; i < internals.size(); i++) + for (size_t i = 0; i < internals.size(); i++) { internalBlobs.push_back(Mat()); if (total(internals[i])) @@ -95,12 +83,13 @@ PERF_TEST_P( ConvolutionPerfTest, perf, Combine( Mat outBlob2D = outBlobs[0].reshape(1, outBlobs[0].size[0]); declare.in(inpBlob2D, wgtBlob2D, WARMUP_RNG).out(outBlob2D).tbb_threads(cv::getNumThreads()); - TEST_CYCLE_N(10) - { + layer->forward(inpBlobs, outBlobs, internalBlobs); /// warmup + + PERF_SAMPLE_BEGIN() layer->forward(inpBlobs, outBlobs, internalBlobs); - } + PERF_SAMPLE_END() SANITY_CHECK_NOTHING(); } -} +} // namespace diff --git a/modules/dnn/perf/perf_halide_net.cpp b/modules/dnn/perf/perf_halide_net.cpp deleted file mode 100644 index 84e6305f52..0000000000 --- a/modules/dnn/perf/perf_halide_net.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2017, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. - -#include "perf_precomp.hpp" - -namespace cvtest -{ - -#ifdef HAVE_HALIDE -using namespace cv; -using namespace dnn; - -static void loadNet(std::string weights, std::string proto, std::string scheduler, - int inWidth, int inHeight, const std::string& outputLayer, - const std::string& framework, int targetId, Net* net) -{ - Mat input(inHeight, inWidth, CV_32FC3); - randu(input, 0.0f, 1.0f); - - weights = findDataFile(weights, false); - if (!proto.empty()) - proto = findDataFile(proto, false); - if (!scheduler.empty()) - scheduler = findDataFile(scheduler, false); - if (framework == "caffe") - { - *net = cv::dnn::readNetFromCaffe(proto, weights); - } - else if (framework == "torch") - { - *net = cv::dnn::readNetFromTorch(weights); - } - else if (framework == "tensorflow") - { - *net = cv::dnn::readNetFromTensorflow(weights); - } - else - CV_Error(Error::StsNotImplemented, "Unknown framework " + framework); - - net->setInput(blobFromImage(input, 1.0, Size(), Scalar(), false)); - net->setPreferableBackend(DNN_BACKEND_HALIDE); - net->setPreferableTarget(targetId); - net->setHalideScheduler(scheduler); - net->forward(outputLayer); -} - -//////////////////////////////////////////////////////////////////////////////// -// CPU target -//////////////////////////////////////////////////////////////////////////////// -PERF_TEST(GoogLeNet, HalidePerfTest) -{ - Net net; - loadNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt", - "", 224, 224, "prob", "caffe", DNN_TARGET_CPU, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(AlexNet, HalidePerfTest) -{ - Net net; - loadNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt", - "dnn/halide_scheduler_alexnet.yml", 227, 227, "prob", "caffe", - DNN_TARGET_CPU, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(ResNet50, HalidePerfTest) -{ - Net net; - loadNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", - "dnn/halide_scheduler_resnet_50.yml", 224, 224, "prob", "caffe", - DNN_TARGET_CPU, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(SqueezeNet_v1_1, HalidePerfTest) -{ - Net net; - loadNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt", - "dnn/halide_scheduler_squeezenet_v1_1.yml", 227, 227, "prob", - "caffe", DNN_TARGET_CPU, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(Inception_5h, HalidePerfTest) -{ - Net net; - loadNet("dnn/tensorflow_inception_graph.pb", "", - "dnn/halide_scheduler_inception_5h.yml", - 224, 224, "softmax2", "tensorflow", DNN_TARGET_CPU, &net); - TEST_CYCLE() net.forward("softmax2"); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(ENet, HalidePerfTest) -{ - Net net; - loadNet("dnn/Enet-model-best.net", "", "dnn/halide_scheduler_enet.yml", - 512, 256, "l367_Deconvolution", "torch", DNN_TARGET_CPU, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} -//////////////////////////////////////////////////////////////////////////////// -// OpenCL target -//////////////////////////////////////////////////////////////////////////////// -PERF_TEST(GoogLeNet_opencl, HalidePerfTest) -{ - Net net; - loadNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt", - "", 227, 227, "prob", "caffe", DNN_TARGET_OPENCL, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(AlexNet_opencl, HalidePerfTest) -{ - Net net; - loadNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt", - "dnn/halide_scheduler_opencl_alexnet.yml", 227, 227, "prob", "caffe", - DNN_TARGET_OPENCL, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(ResNet50_opencl, HalidePerfTest) -{ - Net net; - loadNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", - "dnn/halide_scheduler_opencl_resnet_50.yml", 224, 224, "prob", "caffe", - DNN_TARGET_OPENCL, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - - -PERF_TEST(SqueezeNet_v1_1_opencl, HalidePerfTest) -{ - Net net; - loadNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt", - "dnn/halide_scheduler_opencl_squeezenet_v1_1.yml", 227, 227, "prob", - "caffe", DNN_TARGET_OPENCL, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(Inception_5h_opencl, HalidePerfTest) -{ - Net net; - loadNet("dnn/tensorflow_inception_graph.pb", "", - "dnn/halide_scheduler_opencl_inception_5h.yml", - 224, 224, "softmax2", "tensorflow", DNN_TARGET_OPENCL, &net); - TEST_CYCLE() net.forward("softmax2"); - SANITY_CHECK_NOTHING(); -} - -PERF_TEST(ENet_opencl, HalidePerfTest) -{ - Net net; - loadNet("dnn/Enet-model-best.net", "", "dnn/halide_scheduler_opencl_enet.yml", - 512, 256, "l367_Deconvolution", "torch", DNN_TARGET_OPENCL, &net); - TEST_CYCLE() net.forward(); - SANITY_CHECK_NOTHING(); -} -#endif // HAVE_HALIDE - -} // namespace cvtest diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp new file mode 100644 index 0000000000..55f5ce69e6 --- /dev/null +++ b/modules/dnn/perf/perf_net.cpp @@ -0,0 +1,149 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2017, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "perf_precomp.hpp" +#include "opencv2/core/ocl.hpp" + +#include "opencv2/dnn/shape_utils.hpp" + +namespace +{ + +#ifdef HAVE_HALIDE +#define TEST_DNN_BACKEND DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE +#else +#define TEST_DNN_BACKEND DNN_BACKEND_DEFAULT +#endif +#define TEST_DNN_TARGET DNN_TARGET_CPU, DNN_TARGET_OPENCL + +CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE) +CV_ENUM(DNNTarget, DNN_TARGET_CPU, DNN_TARGET_OPENCL) + +class DNNTestNetwork : public ::perf::TestBaseWithParam< tuple > +{ +public: + dnn::Backend backend; + dnn::Target target; + + dnn::Net net; + + void processNet(std::string weights, std::string proto, std::string halide_scheduler, + int inWidth, int inHeight, const std::string& outputLayer, + const std::string& framework) + { + backend = (dnn::Backend)(int)get<0>(GetParam()); + target = (dnn::Target)(int)get<1>(GetParam()); + + if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL) + { +#if 0 //defined(HAVE_OPENCL) + if (!cv::ocl::useOpenCL()) +#endif + { + throw ::SkipTestException("OpenCL is not available/disabled in OpenCV"); + } + } + + Mat input(inHeight, inWidth, CV_32FC3); + randu(input, 0.0f, 1.0f); + + + weights = findDataFile(weights, false); + if (!proto.empty()) + proto = findDataFile(proto, false); + if (!halide_scheduler.empty() && backend == DNN_BACKEND_HALIDE) + halide_scheduler = findDataFile(std::string("dnn/halide_scheduler_") + (target == DNN_TARGET_OPENCL ? "opencl_" : "") + halide_scheduler, true); + if (framework == "caffe") + { + net = cv::dnn::readNetFromCaffe(proto, weights); + } + else if (framework == "torch") + { + net = cv::dnn::readNetFromTorch(weights); + } + else if (framework == "tensorflow") + { + net = cv::dnn::readNetFromTensorflow(weights); + } + else + CV_Error(Error::StsNotImplemented, "Unknown framework " + framework); + + net.setInput(blobFromImage(input, 1.0, Size(), Scalar(), false)); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + if (backend == DNN_BACKEND_HALIDE) + { + net.setHalideScheduler(halide_scheduler); + } + + MatShape netInputShape = shape(1, 3, inHeight, inWidth); + size_t weightsMemory = 0, blobsMemory = 0; + net.getMemoryConsumption(netInputShape, weightsMemory, blobsMemory); + int64 flops = net.getFLOPS(netInputShape); + + net.forward(outputLayer); // warmup + + std::cout << "Memory consumption:" << std::endl; + std::cout << " Weights(parameters): " << divUp(weightsMemory, 1u<<20) << " Mb" << std::endl; + std::cout << " Blobs: " << divUp(blobsMemory, 1u<<20) << " Mb" << std::endl; + std::cout << "Calculation complexity: " << flops * 1e-9 << " GFlops" << std::endl; + + PERF_SAMPLE_BEGIN() + net.forward(); + PERF_SAMPLE_END() + + SANITY_CHECK_NOTHING(); + } +}; + + +PERF_TEST_P_(DNNTestNetwork, AlexNet) +{ + processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt", + "alexnet.yml", 227, 227, "prob", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, GoogLeNet) +{ + processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt", + "", 224, 224, "prob", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, ResNet50) +{ + processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt", + "resnet_50.yml", 224, 224, "prob", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, SqueezeNet_v1_1) +{ + processNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt", + "squeezenet_v1_1.yml", 227, 227, "prob", "caffe"); +} + +PERF_TEST_P_(DNNTestNetwork, Inception_5h) +{ + processNet("dnn/tensorflow_inception_graph.pb", "", + "inception_5h.yml", + 224, 224, "softmax2", "tensorflow"); +} + +PERF_TEST_P_(DNNTestNetwork, ENet) +{ + processNet("dnn/Enet-model-best.net", "", "enet.yml", + 512, 256, "l367_Deconvolution", "torch"); +} + + +INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, + testing::Combine( + ::testing::Values(TEST_DNN_BACKEND), + DNNTarget::all() + ) +); + +} // namespace diff --git a/modules/dnn/perf/perf_precomp.hpp b/modules/dnn/perf/perf_precomp.hpp index 5cdbc6d179..38e7d616cd 100644 --- a/modules/dnn/perf/perf_precomp.hpp +++ b/modules/dnn/perf/perf_precomp.hpp @@ -1,11 +1,3 @@ -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-declarations" -# if defined __clang__ || defined __APPLE__ -# pragma GCC diagnostic ignored "-Wmissing-prototypes" -# pragma GCC diagnostic ignored "-Wextra" -# endif -#endif - #ifndef __OPENCV_PERF_PRECOMP_HPP__ #define __OPENCV_PERF_PRECOMP_HPP__ @@ -14,4 +6,9 @@ #include #include +using namespace cvtest; +using namespace perf; +using namespace cv; +using namespace dnn; + #endif