Make it possible to run a benchmark for the given amount of time

This is more practical than running it a given number of times, which
may result in very long running times for slow functions, such as
wxImage resizing tests using wxIMAGE_QUALITY_HIGH.

Also show the standard deviation in addition to the average and min/max
values.

tests/benchmarks/bench.cpp

@@ -32,7 +32,7 @@
 static const char OPTION_LIST = 'l';
 static const char OPTION_SINGLE = '1';
 
-static const char OPTION_AVG_COUNT = 'a';
+static const char OPTION_RUN_TIME = 't';
 static const char OPTION_NUM_RUNS = 'n';
 static const char OPTION_NUMERIC_PARAM = 'p';
 static const char OPTION_STRING_PARAM = 's';
@@ -64,13 +64,17 @@ public:
     const wxString& GetStringParameter() const { return m_strParam; }
 
 private:
+    // output the results of a single benchmark if successful or just return
+    // false if anything went wrong
+    bool RunSingleBenchmark(Bench::Function* func);
+
     // list all registered benchmarks
     void ListBenchmarks();
 
     // command lines options/parameters
     wxSortedArrayString m_toRun;
-    long m_numRuns,
+    long m_numRuns, // number of times to run a single benchmark or 0
-         m_avgCount,
+         m_runTime, // minimum time to run a single benchmark if m_numRuns == 0
          m_numParam;
     wxString m_strParam;
 };
@@ -99,8 +103,8 @@ wxString Bench::GetStringParameter()
 
 BenchApp::BenchApp()
 {
-    m_avgCount = 10;
+    m_numRuns = 0; // this means to use m_runTime
-    m_numRuns = 10000; // just some default (TODO: switch to time-based one)
+    m_runTime = 500; // default minimum
     m_numParam = 0;
 }
 
@@ -132,12 +136,13 @@ void BenchApp::OnInitCmdLine(wxCmdLineParser& parser)
                      "single",
                      "run the benchmark once only");
 
-    parser.AddOption(OPTION_AVG_COUNT,
+    parser.AddOption(OPTION_RUN_TIME,
-                     "avg-count",
+                     "run-time",
                      wxString::Format
                      (
-                        "number of times to run benchmarking loop (default: %ld)",
+                        "maximum time to run each benchmark in ms "
-                        m_avgCount
+                        "(default: %ld, set to 0 to disable)",
+                        m_runTime
                      ),
                      wxCMD_LINE_VAL_NUMBER);
     parser.AddOption(OPTION_NUM_RUNS,
@@ -145,7 +150,7 @@ void BenchApp::OnInitCmdLine(wxCmdLineParser& parser)
                      wxString::Format
                      (
                          "number of times to run each benchmark in a loop "
-                         "(default: %ld)",
+                         "(default: %ld, 0 means to run until max time passes)",
                          m_numRuns
                      ),
                      wxCMD_LINE_VAL_NUMBER);
@@ -188,25 +193,26 @@ bool BenchApp::OnCmdLineParsed(wxCmdLineParser& parser)
         return false;
     }
 
-    bool numRunsSpecified = false;
+    const bool runTimeSpecified = parser.Found(OPTION_RUN_TIME, &m_runTime);
-    if ( parser.Found(OPTION_AVG_COUNT, &m_avgCount) )
+    const bool numRunsSpecified = parser.Found(OPTION_NUM_RUNS, &m_numRuns);
-        numRunsSpecified = true;
-    if ( parser.Found(OPTION_NUM_RUNS, &m_numRuns) )
-        numRunsSpecified = true;
     parser.Found(OPTION_NUMERIC_PARAM, &m_numParam);
     parser.Found(OPTION_STRING_PARAM, &m_strParam);
     if ( parser.Found(OPTION_SINGLE) )
     {
-        if ( numRunsSpecified )
+        if ( runTimeSpecified || numRunsSpecified )
         {
             wxFprintf(stderr, "Incompatible options specified.\n");
 
             return false;
         }
 
-        m_avgCount =
         m_numRuns = 1;
     }
+    else if ( numRunsSpecified && !runTimeSpecified )
+    {
+        // If only the number of runs is specified, use it only.
+        m_runTime = 0;
+    }
 
     // construct sorted array for quick verification of benchmark names
     wxSortedArrayString benchmarks;
@@ -235,6 +241,20 @@ bool BenchApp::OnCmdLineParsed(wxCmdLineParser& parser)
 int BenchApp::OnRun()
 {
     int rc = EXIT_SUCCESS;
+
+    wxString params;
+    if ( m_numParam )
+        params += wxString::Format("N=%ld", m_numParam);
+    if ( !m_strParam.empty() )
+    {
+        if ( !params.empty() )
+            params += " and ";
+        params += wxString::Format("s=\"%s\"", m_strParam);
+    }
+
+    if ( !params.empty() )
+        wxPrintf("Benchmarks are running with non-default %s\n", params);
+
     for ( Bench::Function *func = Bench::Function::GetFirst();
           func;
           func = func->GetNext() )
@@ -242,66 +262,101 @@ int BenchApp::OnRun()
         if ( m_toRun.Index(func->GetName()) == wxNOT_FOUND )
             continue;
 
-        wxString params;
+        if ( !RunSingleBenchmark(func) )
-        if ( m_numParam )
-            params += wxString::Format(" with N=%ld", m_numParam);
-        if ( !m_strParam.empty() )
         {
-            if ( !params.empty() )
+            wxFprintf(stderr, "ERROR running %s\n", func->GetName());
-                params += " and";
+            rc = EXIT_FAILURE;
-            params += wxString::Format(" with s=\"%s\"", m_strParam);
+        }
     }
 
-        wxPrintf("Benchmarking %s%s: ", func->GetName(), params);
+    return rc;
-
-        long timeMin = LONG_MAX,
-             timeMax = 0,
-             timeTotal = 0;
-        bool ok = func->Init();
-        for ( long a = 0; ok && a < m_avgCount; a++ )
-        {
-            wxStopWatch sw;
-            for ( long n = 0; n < m_numRuns && ok; n++ )
-            {
-                ok = func->Run();
 }
 
-            sw.Pause();
+bool BenchApp::RunSingleBenchmark(Bench::Function* func)
+{
+    if ( !func->Init() )
+        return false;
+
+    wxPrintf("Benchmarking %s: ", func->GetName());
+    fflush(stdout);
+
+    // We use the algorithm for iteratively computing the mean and the
+    // standard deviation of the sequence of values described in Knuth's
+    // "The Art of Computer Programming, Volume 2: Seminumerical
+    // Algorithms", section 4.2.2.
+    //
+    // The algorithm defines the sequences M(k) and S(k) as follows:
+    //
+    //  M(1) = x(1), M(k) = M(k-1) + (x(k) - M(k-1)) / k
+    //  S(1) = 0,    S(k) = S(k-1) + (x(k) - M(k-1))*(x(k) - M(k))
+    //
+    // where x(k) is the k-th value. Then the mean is simply the last value
+    // of the first sequence M(N) and the standard deviation is
+    // sqrt(S(N)/(N-1)).
+
+    wxStopWatch swTotal;
+    if ( !func->Run() )
+        return false;
+
+    double timeMin = DBL_MAX,
+           timeMax = 0;
+
+    double m = swTotal.TimeInMicro().ToDouble();
+    double s = 0;
+
+    long n = 0;
+    for ( ;; )
+    {
+        // One termination condition is reaching the maximum number of runs.
+        if ( ++n == m_numRuns )
+            break;
+
+        double t;
+        {
+            wxStopWatch swThis;
+            if ( !func->Run() )
+                return false;
+
+            t = swThis.TimeInMicro().ToDouble();
+        }
 
-            const long t = sw.Time();
         if ( t < timeMin )
             timeMin = t;
         if ( t > timeMax )
             timeMax = t;
-            timeTotal += t;
+
+        const double lastM = m;
+        m += (t - lastM) / n;
+        s += (t - lastM)*(t - m);
+
+        // The other termination condition is that we are running for at least
+        // m_runTime milliseconds.
+        if ( m_runTime && swTotal.Time() >= m_runTime )
+            break;
     }
 
     func->Done();
 
-        if ( !ok )
+    // For a single run there is no standard deviation and min/max don't make
+    // much sense.
+    if ( n == 1 )
     {
-            wxPrintf("ERROR\n");
+        wxPrintf("single run took %.0fus\n", m);
-            rc = EXIT_FAILURE;
     }
     else
     {
-            wxPrintf("%ldms total, ", timeTotal);
+        s = sqrt(s / (n - 1));
 
-            long times = m_avgCount;
+        wxPrintf
-            if ( m_avgCount > 2 )
+        (
-            {
+            "%ld runs, %.0fus avg, %.0f std dev (%.0f/%.0f min/max)\n",
-                timeTotal -= timeMin + timeMax;
+            n, m, s, timeMin, timeMax
-                times -= 2;
+        );
-            }
-
-            wxPrintf("%.2f avg (min=%ld, max=%ld)\n",
-                     (float)timeTotal / times, timeMin, timeMax);
     }
 
     fflush(stdout);
-    }
 
-    return rc;
+    return true;
 }
 
 int BenchApp::OnExit()