Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added performance validation check

Browse Source
pull/3342/head
Alexander Alekhin 11 years ago
parent 4de9605b7b
commit 8ab6852927
2 changed files with 203 additions and 5 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      modules/ts/include/opencv2/ts/ts_perf.hpp
  2. 206
      modules/ts/src/ts_perf.cpp

2
modules/ts/include/opencv2/ts/ts_perf.hpp
Unescape View File

@ -419,9 +419,11 @@ private:
static int64 timeLimitDefault;
static unsigned int iterationsLimitDefault;
unsigned int minIters;
unsigned int nIters;
unsigned int currentIter;
unsigned int runsPerIteration;
unsigned int perfValidationStage;
performance_metrics metrics;
void validateMetrics();

206
modules/ts/src/ts_perf.cpp
Unescape View File

@ -1,5 +1,16 @@
#include "precomp.hpp"
#include <map>
#include <iostream>
#include <fstream>
#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#endif
#ifdef HAVE_CUDA
#include "opencv2/core/cuda.hpp"
#endif
@ -35,11 +46,11 @@ static bool param_verify_sanity;
static bool param_collect_impl;
#endif
extern bool test_ipp_check;
#ifdef HAVE_CUDA
static int param_cuda_device;
#endif
#ifdef ANDROID
static int param_affinity_mask;
static bool log_power_checkpoints;
@ -59,6 +70,8 @@ static void setCurrentThreadAffinityMask(int mask)
}
#endif
static double perf_stability_criteria = 0.03; // 3%
namespace {
class PerfEnvironment: public ::testing::Environment
@ -635,6 +648,82 @@ void performance_metrics::clear()
terminationReason = TERM_UNKNOWN;
}
/*****************************************************************************************\
* Performance validation results
\*****************************************************************************************/
static bool perf_validation_enabled = false;
static std::string perf_validation_results_directory;
static std::map<std::string, float> perf_validation_results;
static std::string perf_validation_results_outfile;
static double perf_validation_criteria = 0.03; // 3 %
static double perf_validation_time_threshold_ms = 0.1;
static int perf_validation_idle_delay_ms = 3000; // 3 sec
static void loadPerfValidationResults(const std::string& fileName)
{
perf_validation_results.clear();
std::ifstream infile(fileName.c_str());
while (!infile.eof())
{
std::string name;
float value = 0;
if (!(infile >> value))
{
if (infile.eof())
break; // it is OK
std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl;
return;
}
infile.ignore(1);
if (!(std::getline(infile, name)))
{
std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl;
return;
}
if (!name.empty() && name[name.size() - 1] == '\r') // CRLF processing on Linux
name.resize(name.size() - 1);
perf_validation_results[name] = value;
}
std::cout << "Performance validation results loaded from " << fileName << " (" << perf_validation_results.size() << " entries)" << std::endl;
}
static void savePerfValidationResult(const std::string& name, float value)
{
perf_validation_results[name] = value;
}
static void savePerfValidationResults()
{
if (!perf_validation_results_outfile.empty())
{
std::ofstream outfile((perf_validation_results_directory + perf_validation_results_outfile).c_str());
std::map<std::string, float>::const_iterator i;
for (i = perf_validation_results.begin(); i != perf_validation_results.end(); ++i)
{
outfile << i->second << ';';
outfile << i->first << std::endl;
}
outfile.close();
std::cout << "Performance validation results saved (" << perf_validation_results.size() << " entries)" << std::endl;
}
}
class PerfValidationEnvironment : public ::testing::Environment
{
public:
virtual ~PerfValidationEnvironment() {}
virtual void SetUp() {}
virtual void TearDown()
{
savePerfValidationResults();
}
};
/*****************************************************************************************\
* ::perf::TestBase
@ -666,6 +755,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
"{ perf_list_impls |false |list available implementation variants and exit}"
"{ perf_run_cpu |false |deprecated, equivalent to --perf_impl=plain}"
"{ perf_strategy |default |specifies performance measuring strategy: default, base or simple (weak restrictions)}"
"{ perf_read_validation_results | |specifies file name with performance results from previous run}"
"{ perf_write_validation_results | |specifies file name to write performance validation results}"
#ifdef ANDROID
"{ perf_time_limit |6.0 |default time limit for a single test (in seconds)}"
"{ perf_affinity_mask |0 |set affinity mask for the main thread}"
@ -789,6 +880,26 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
}
#endif
{
const char* path = getenv("OPENCV_PERF_VALIDATION_DIR");
if (path)
perf_validation_results_directory = path;
}
std::string fileName_perf_validation_results_src = args.get<std::string>("perf_read_validation_results");
if (!fileName_perf_validation_results_src.empty())
{
perf_validation_enabled = true;
loadPerfValidationResults(perf_validation_results_directory + fileName_perf_validation_results_src);
}
perf_validation_results_outfile = args.get<std::string>("perf_write_validation_results");
if (!perf_validation_results_outfile.empty())
{
perf_validation_enabled = true;
::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment());
}
if (!args.check())
{
args.printErrors();
@ -878,7 +989,9 @@ TestBase::TestBase(): testStrategy(PERF_STRATEGY_DEFAULT), declare(this)
{
lastTime = totalTime = timeLimit = 0;
nIters = currentIter = runsPerIteration = 0;
minIters = param_min_samples;
verified = false;
perfValidationStage = 0;
}
#ifdef _MSC_VER
# pragma warning(pop)
@ -1004,7 +1117,7 @@ bool TestBase::next()
has_next = false;
break;
}
if (currentIter < param_min_samples)
if (currentIter < minIters)
{
has_next = true;
break;
@ -1012,14 +1125,96 @@ bool TestBase::next()
calcMetrics();
double criteria = 0.03; // 3%
if (fabs(metrics.mean) > 1e-6)
has_next = metrics.stddev > criteria * fabs(metrics.mean);
has_next = metrics.stddev > perf_stability_criteria * fabs(metrics.mean);
else
has_next = true;
}
} while (false);
if (perf_validation_enabled && !has_next)
{
calcMetrics();
double median_ms = metrics.median * 1000.0f / metrics.frequency;
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
std::string name = (test_info == 0) ? "" :
std::string(test_info->test_case_name()) + "--" + test_info->name();
if (!perf_validation_results.empty() && !name.empty())
{
std::map<std::string, float>::iterator i = perf_validation_results.find(name);
bool isSame = false;
bool found = false;
bool grow = false;
if (i != perf_validation_results.end())
{
found = true;
double prev_result = i->second;
grow = median_ms > prev_result;
isSame = fabs(median_ms - prev_result) <= perf_validation_criteria * fabs(median_ms);
if (!isSame)
{
if (perfValidationStage == 0)
{
printf("Performance is changed (samples = %d, median):\n %.2f ms (current)\n %.2f ms (previous)\n", (int)times.size(), median_ms, prev_result);
}
}
}
else
{
if (perfValidationStage == 0)
printf("New performance result is detected\n");
}
if (!isSame)
{
if (perfValidationStage < 2)
{
if (perfValidationStage == 0 && currentIter <= minIters * 3 && currentIter < nIters)
{
unsigned int new_minIters = std::max(minIters * 5, currentIter * 3);
printf("Increase minIters from %u to %u\n", minIters, new_minIters);
minIters = new_minIters;
has_next = true;
perfValidationStage++;
}
else if (found && currentIter >= nIters &&
median_ms > perf_validation_time_threshold_ms &&
(grow || metrics.stddev > perf_stability_criteria * fabs(metrics.mean)))
{
printf("Performance is unstable, it may be a result of overheat problems\n");
printf("Idle delay for %d ms... \n", perf_validation_idle_delay_ms);
#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
Sleep(perf_validation_idle_delay_ms);
#else
usleep(perf_validation_idle_delay_ms * 1000);
#endif
has_next = true;
minIters = std::min(minIters * 5, nIters);
// reset collected samples
currentIter = 0;
times.clear();
metrics.clear();
perfValidationStage += 2;
}
if (!has_next)
{
printf("Assume that current result is valid\n");
}
}
else
{
printf("Re-measured performance result: %.2f ms\n", median_ms);
}
}
}
if (!has_next && !name.empty())
{
savePerfValidationResult(name, (float)median_ms);
}
}
#ifdef ANDROID
if (log_power_checkpoints)
{
@ -1223,9 +1418,10 @@ void TestBase::validateMetrics()
else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE)
{
double mean = metrics.mean * 1000.0f / metrics.frequency;
double median = metrics.median * 1000.0f / metrics.frequency;
double stddev = metrics.stddev * 1000.0f / metrics.frequency;
double percents = stddev / mean * 100.f;
printf("[ PERFSTAT ] (samples = %d, mean = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, stddev, percents);
printf("[ PERFSTAT ] (samples = %d, mean = %.2f, median = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, median, stddev, percents);
}
else
{

Write Preview
Loading…
Cancel
Save