optimized wxStringTokenizer: it's now slightly faster in wchar_t build and much faster in UTF-8 build

git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@47707 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

include/wx/tokenzr.h

@@ -72,12 +72,12 @@ public:
     // get current tokenizer state
         // returns the part of the string which remains to tokenize (*not* the
         // initial string)
-    wxString GetString() const { return m_string.substr(m_pos); }
+    wxString GetString() const { return wxString(m_pos, m_string.end()); }
 
         // returns the current position (i.e. one index after the last
         // returned token or 0 if GetNextToken() has never been called) in the
         // original string
-    size_t GetPosition() const { return m_pos; }
+    size_t GetPosition() const { return m_pos - m_string.begin(); }
 
     // misc
         // get the current mode - can be different from the one passed to the
@@ -111,10 +111,24 @@ public:
 protected:
     bool IsOk() const { return m_mode != wxTOKEN_INVALID; }
 
-    wxString m_string,              // the string we tokenize
-             m_delims;              // all possible delimiters
+    bool DoHasMoreTokens() const;
 
-    size_t   m_pos;                 // the current position in m_string
+    enum MoreTokensState
+    {
+        MoreTokens_Unknown,
+        MoreTokens_Yes,
+        MoreTokens_No
+    };
+
+    MoreTokensState m_hasMoreTokens;
+
+    wxString m_string;              // the string we tokenize
+    wxString::const_iterator m_stringEnd;
+    // FIXME-UTF8: use wxWcharBuffer
+    wxWxCharBuffer m_delims;        // all possible delimiters
+    size_t m_delimsLen;
+
+    wxString::const_iterator m_pos; // the current position in m_string
 
     wxStringTokenizerMode m_mode;   // see wxTOKEN_XXX values