From 5847b302be47b1e4cb1e5786caa482aa29778971 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Tue, 22 Feb 2022 23:39:38 +0000 Subject: [PATCH] Make it possible to run a benchmark for the given amount of time This is more practical than running it a given number of times, which may result in very long running times for slow functions, such as wxImage resizing tests using wxIMAGE_QUALITY_HIGH. Also show the standard deviation in addition to the average and min/max values. --- tests/benchmarks/bench.cpp | 197 ++++++++++++++++++++++++------------- 1 file changed, 126 insertions(+), 71 deletions(-) diff --git a/tests/benchmarks/bench.cpp b/tests/benchmarks/bench.cpp index 2473db1f38..7cf00fe981 100644 --- a/tests/benchmarks/bench.cpp +++ b/tests/benchmarks/bench.cpp @@ -32,7 +32,7 @@ static const char OPTION_LIST = 'l'; static const char OPTION_SINGLE = '1'; -static const char OPTION_AVG_COUNT = 'a'; +static const char OPTION_RUN_TIME = 't'; static const char OPTION_NUM_RUNS = 'n'; static const char OPTION_NUMERIC_PARAM = 'p'; static const char OPTION_STRING_PARAM = 's'; @@ -64,13 +64,17 @@ public: const wxString& GetStringParameter() const { return m_strParam; } private: + // output the results of a single benchmark if successful or just return + // false if anything went wrong + bool RunSingleBenchmark(Bench::Function* func); + // list all registered benchmarks void ListBenchmarks(); // command lines options/parameters wxSortedArrayString m_toRun; - long m_numRuns, - m_avgCount, + long m_numRuns, // number of times to run a single benchmark or 0 + m_runTime, // minimum time to run a single benchmark if m_numRuns == 0 m_numParam; wxString m_strParam; }; @@ -99,8 +103,8 @@ wxString Bench::GetStringParameter() BenchApp::BenchApp() { - m_avgCount = 10; - m_numRuns = 10000; // just some default (TODO: switch to time-based one) + m_numRuns = 0; // this means to use m_runTime + m_runTime = 500; // default minimum m_numParam = 0; } @@ -132,12 +136,13 @@ void BenchApp::OnInitCmdLine(wxCmdLineParser& parser) "single", "run the benchmark once only"); - parser.AddOption(OPTION_AVG_COUNT, - "avg-count", + parser.AddOption(OPTION_RUN_TIME, + "run-time", wxString::Format ( - "number of times to run benchmarking loop (default: %ld)", - m_avgCount + "maximum time to run each benchmark in ms " + "(default: %ld, set to 0 to disable)", + m_runTime ), wxCMD_LINE_VAL_NUMBER); parser.AddOption(OPTION_NUM_RUNS, @@ -145,7 +150,7 @@ void BenchApp::OnInitCmdLine(wxCmdLineParser& parser) wxString::Format ( "number of times to run each benchmark in a loop " - "(default: %ld)", + "(default: %ld, 0 means to run until max time passes)", m_numRuns ), wxCMD_LINE_VAL_NUMBER); @@ -188,25 +193,26 @@ bool BenchApp::OnCmdLineParsed(wxCmdLineParser& parser) return false; } - bool numRunsSpecified = false; - if ( parser.Found(OPTION_AVG_COUNT, &m_avgCount) ) - numRunsSpecified = true; - if ( parser.Found(OPTION_NUM_RUNS, &m_numRuns) ) - numRunsSpecified = true; + const bool runTimeSpecified = parser.Found(OPTION_RUN_TIME, &m_runTime); + const bool numRunsSpecified = parser.Found(OPTION_NUM_RUNS, &m_numRuns); parser.Found(OPTION_NUMERIC_PARAM, &m_numParam); parser.Found(OPTION_STRING_PARAM, &m_strParam); if ( parser.Found(OPTION_SINGLE) ) { - if ( numRunsSpecified ) + if ( runTimeSpecified || numRunsSpecified ) { wxFprintf(stderr, "Incompatible options specified.\n"); return false; } - m_avgCount = m_numRuns = 1; } + else if ( numRunsSpecified && !runTimeSpecified ) + { + // If only the number of runs is specified, use it only. + m_runTime = 0; + } // construct sorted array for quick verification of benchmark names wxSortedArrayString benchmarks; @@ -235,6 +241,20 @@ bool BenchApp::OnCmdLineParsed(wxCmdLineParser& parser) int BenchApp::OnRun() { int rc = EXIT_SUCCESS; + + wxString params; + if ( m_numParam ) + params += wxString::Format("N=%ld", m_numParam); + if ( !m_strParam.empty() ) + { + if ( !params.empty() ) + params += " and "; + params += wxString::Format("s=\"%s\"", m_strParam); + } + + if ( !params.empty() ) + wxPrintf("Benchmarks are running with non-default %s\n", params); + for ( Bench::Function *func = Bench::Function::GetFirst(); func; func = func->GetNext() ) @@ -242,68 +262,103 @@ int BenchApp::OnRun() if ( m_toRun.Index(func->GetName()) == wxNOT_FOUND ) continue; - wxString params; - if ( m_numParam ) - params += wxString::Format(" with N=%ld", m_numParam); - if ( !m_strParam.empty() ) + if ( !RunSingleBenchmark(func) ) { - if ( !params.empty() ) - params += " and"; - params += wxString::Format(" with s=\"%s\"", m_strParam); - } - - wxPrintf("Benchmarking %s%s: ", func->GetName(), params); - - long timeMin = LONG_MAX, - timeMax = 0, - timeTotal = 0; - bool ok = func->Init(); - for ( long a = 0; ok && a < m_avgCount; a++ ) - { - wxStopWatch sw; - for ( long n = 0; n < m_numRuns && ok; n++ ) - { - ok = func->Run(); - } - - sw.Pause(); - - const long t = sw.Time(); - if ( t < timeMin ) - timeMin = t; - if ( t > timeMax ) - timeMax = t; - timeTotal += t; - } - - func->Done(); - - if ( !ok ) - { - wxPrintf("ERROR\n"); + wxFprintf(stderr, "ERROR running %s\n", func->GetName()); rc = EXIT_FAILURE; } - else - { - wxPrintf("%ldms total, ", timeTotal); - - long times = m_avgCount; - if ( m_avgCount > 2 ) - { - timeTotal -= timeMin + timeMax; - times -= 2; - } - - wxPrintf("%.2f avg (min=%ld, max=%ld)\n", - (float)timeTotal / times, timeMin, timeMax); - } - - fflush(stdout); } return rc; } +bool BenchApp::RunSingleBenchmark(Bench::Function* func) +{ + if ( !func->Init() ) + return false; + + wxPrintf("Benchmarking %s: ", func->GetName()); + fflush(stdout); + + // We use the algorithm for iteratively computing the mean and the + // standard deviation of the sequence of values described in Knuth's + // "The Art of Computer Programming, Volume 2: Seminumerical + // Algorithms", section 4.2.2. + // + // The algorithm defines the sequences M(k) and S(k) as follows: + // + // M(1) = x(1), M(k) = M(k-1) + (x(k) - M(k-1)) / k + // S(1) = 0, S(k) = S(k-1) + (x(k) - M(k-1))*(x(k) - M(k)) + // + // where x(k) is the k-th value. Then the mean is simply the last value + // of the first sequence M(N) and the standard deviation is + // sqrt(S(N)/(N-1)). + + wxStopWatch swTotal; + if ( !func->Run() ) + return false; + + double timeMin = DBL_MAX, + timeMax = 0; + + double m = swTotal.TimeInMicro().ToDouble(); + double s = 0; + + long n = 0; + for ( ;; ) + { + // One termination condition is reaching the maximum number of runs. + if ( ++n == m_numRuns ) + break; + + double t; + { + wxStopWatch swThis; + if ( !func->Run() ) + return false; + + t = swThis.TimeInMicro().ToDouble(); + } + + if ( t < timeMin ) + timeMin = t; + if ( t > timeMax ) + timeMax = t; + + const double lastM = m; + m += (t - lastM) / n; + s += (t - lastM)*(t - m); + + // The other termination condition is that we are running for at least + // m_runTime milliseconds. + if ( m_runTime && swTotal.Time() >= m_runTime ) + break; + } + + func->Done(); + + // For a single run there is no standard deviation and min/max don't make + // much sense. + if ( n == 1 ) + { + wxPrintf("single run took %.0fus\n", m); + } + else + { + s = sqrt(s / (n - 1)); + + wxPrintf + ( + "%ld runs, %.0fus avg, %.0f std dev (%.0f/%.0f min/max)\n", + n, m, s, timeMin, timeMax + ); + } + + fflush(stdout); + + return true; +} + int BenchApp::OnExit() { #if wxUSE_GUI