1. changed wxStringTokenizer to not modify the string we're iterating over

but to just update our position in it (makes the code much more clear)
2. added GetLastDelimiter() to make up for lack of mode combining
   wxTOKEN_RET_EMPTY_ALL and RET_DELIMS
3. documented it and added unit tests for it


git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@36552 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

include/wx/tokenzr.h

@@ -58,12 +58,16 @@ public:
     void Reinit(const wxString& str);
 
     // tokens access
-        // count them
+        // return the number of remaining tokens
     size_t CountTokens() const;
         // did we reach the end of the string?
     bool HasMoreTokens() const;
         // get the next token, will return empty string if !HasMoreTokens()
     wxString GetNextToken();
+        // get the delimiter which terminated the token last retrieved by
+        // GetNextToken() or NUL if there had been no tokens yet or the last
+        // one wasn't terminated (but ran to the end of the string)
+    wxChar GetLastDelimiter() const { return m_lastDelim; }
 
     // get current tokenizer state
         // returns the part of the string which remains to tokenize (*not* the
@@ -79,6 +83,9 @@ public:
         // get the current mode - can be different from the one passed to the
         // ctor if it was wxTOKEN_DEFAULT
     wxStringTokenizerMode GetMode() const { return m_mode; }
+        // do we return empty tokens?
+    bool AllowEmpty() const { return m_mode != wxTOKEN_STRTOK; }
+
 
     // backwards compatibility section from now on
     // -------------------------------------------
@@ -104,14 +111,14 @@ public:
 protected:
     bool IsOk() const { return m_mode != wxTOKEN_INVALID; }
 
-    wxString m_string,              // the (rest of) string to tokenize
-             m_delims;              // all delimiters
+    wxString m_string,              // the string we tokenize
+             m_delims;              // all possible delimiters
 
-    size_t   m_pos;                 // the position in the original string
+    size_t   m_pos;                 // the current position in m_string
 
     wxStringTokenizerMode m_mode;   // see wxTOKEN_XXX values
 
-    bool     m_hasMore;             // do we have more (possible empty) tokens?
+    wxChar   m_lastDelim;           // delimiter after last token or '\0'
 };
 
 // ----------------------------------------------------------------------------