This commit was manufactured by cvs2svn to create tag 'TOKENZR_ORIG'.

git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/tags/TOKENZR_ORIG@2882 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

include/wx/tokenzr.h

@@ -0,0 +1,57 @@
+/////////////////////////////////////////////////////////////////////////////
+// Name:        tokenzr.h
+// Purpose:     String tokenizer
+// Author:      Guilhem Lavaux
+// Modified by:
+// Created:     04/22/98
+// RCS-ID:      $Id$
+// Copyright:   (c) Guilhem Lavaux
+// Licence:     wxWindows licence
+/////////////////////////////////////////////////////////////////////////////
+
+#ifndef _WX_TOKENZRH
+#define _WX_TOKENZRH
+
+#ifdef __GNUG__
+#pragma interface
+#endif
+
+#include "wx/object.h"
+#include "wx/string.h"
+#include "wx/filefn.h"
+
+class WXDLLEXPORT wxStringTokenizer : public wxObject
+{
+public:
+    wxStringTokenizer(const wxString& to_tokenize,
+                      const wxString& delims = " \t\r\n",
+                      bool ret_delim = FALSE);
+    wxStringTokenizer() { m_retdelims = FALSE;}
+    virtual ~wxStringTokenizer();
+
+    int CountTokens() const;
+    bool HasMoreTokens();
+
+    wxString NextToken();
+    wxString GetNextToken() { return NextToken(); };
+
+    wxString GetString() const { return m_string; }
+
+    void SetString(const wxString& to_tokenize,
+                   const wxString& delims = " \t\r\n",
+                   bool ret_delim = FALSE)
+    {
+        m_string = to_tokenize;
+        m_delims = delims;
+        m_retdelims = ret_delim;
+    }
+
+protected:
+    off_t FindDelims(const wxString& str, const wxString& delims) const;
+    void EatLeadingDelims();
+
+    wxString m_string, m_delims;
+    bool m_retdelims;
+};
+
+#endif // _WX_TOKENZRH

src/common/tokenzr.cpp

@@ -0,0 +1,138 @@
+/////////////////////////////////////////////////////////////////////////////
+// Name:        tokenzr.cpp
+// Purpose:     String tokenizer
+// Author:      Guilhem Lavaux
+// Modified by:
+// Created:     04/22/98
+// RCS-ID:      $Id$
+// Copyright:   (c) Guilhem Lavaux
+// Licence:     wxWindows licence
+/////////////////////////////////////////////////////////////////////////////
+
+#ifdef __GNUG__
+    #pragma implementation "tokenzr.h"
+#endif
+
+// For compilers that support precompilation, includes "wx.h".
+#include "wx/wxprec.h"
+
+#ifdef __BORLANDC__
+    #pragma hdrstop
+#endif
+
+#include "wx/tokenzr.h"
+
+wxStringTokenizer::wxStringTokenizer(const wxString& to_tokenize,
+                                     const wxString& delims,
+                                     bool ret_delims)
+{
+    m_string = to_tokenize;
+    m_delims = delims;
+    m_retdelims = ret_delims;
+}
+
+wxStringTokenizer::~wxStringTokenizer()
+{
+}
+
+off_t wxStringTokenizer::FindDelims(const wxString& str, const wxString& delims) const
+{
+    for ( size_t i = 0; i < str.Length(); i++ )
+    {
+        wxChar c = str[i];
+
+        for ( size_t j = 0; j < delims.Length() ; j++ )
+        {
+            if ( delims[j] == c )
+                return i;
+        }
+    }
+
+    return -1;
+}
+
+int wxStringTokenizer::CountTokens() const
+{
+    wxString p_string = m_string;
+    bool found = TRUE;
+    int pos, count = 1;
+
+    if (p_string.Length() == 0)
+        return 0;
+
+    while (found)
+    {
+        pos = FindDelims(p_string, m_delims);
+        if (pos != -1)
+        {
+            count++;
+            p_string = p_string(pos+1, p_string.Length());
+        }
+        else
+        {
+            found = FALSE;
+        }
+    }
+
+    return count;
+}
+
+bool wxStringTokenizer::HasMoreTokens()
+{
+    return !m_string.IsEmpty();
+}
+
+// needed to fix leading whitespace / mult. delims bugs
+void wxStringTokenizer::EatLeadingDelims()
+{
+    int pos;
+
+    // while leading delims trim 'em from the left
+    while ( ( pos = FindDelims(m_string, m_delims)) == 0 )
+    {
+        m_string = m_string.Mid((size_t)1);
+    }
+}
+
+wxString wxStringTokenizer::NextToken()
+{
+    off_t pos, pos2;
+    wxString r_string;
+
+    if ( m_string.IsEmpty() )
+        return m_string;
+
+    if ( !m_retdelims )
+        EatLeadingDelims();
+
+    pos = FindDelims(m_string, m_delims);
+    if (pos == -1)
+    {
+        r_string = m_string;
+        m_string = wxEmptyString;
+
+        return r_string;
+    }
+
+    if (m_retdelims)
+    {
+        if (!pos)
+        {
+            pos++;
+            pos2 = 1;
+        }
+        else
+        {
+            pos2 = pos;
+        }
+    }
+    else
+    {
+        pos2 = pos + 1;
+    }
+
+    r_string = m_string.Left((size_t)pos);
+    m_string = m_string.Mid((size_t)pos2);
+
+    return r_string;
+}

wxPython/wxPython/lib/PyCrust/PyCrust.py

@@ -1,216 +0,0 @@
-#!/usr/bin/env python
-"""PyCrust is a python shell application.
-"""
-
-__author__ = "Patrick K. O'Brien <pobrien@orbtech.com>"
-__cvsid__ = "$Id$"
-__date__ = "July 1, 2001"
-__version__ = "$Revision$"[11:-2]
-
-from wxPython.wx import *
-
-from version import VERSION
-from shell import Shell
-
-ID_AUTOCOMP = NewId()
-ID_AUTOCOMP_SHOW = NewId()
-ID_AUTOCOMP_INCLUDE_MAGIC = NewId()
-ID_AUTOCOMP_INCLUDE_SINGLE = NewId()
-ID_AUTOCOMP_INCLUDE_DOUBLE = NewId()
-ID_CALLTIPS = NewId()
-ID_CALLTIPS_SHOW = NewId()
-
-
-class Frame(wxFrame):
-    """Main window for the PyCrust application."""
-    def __init__(self, parent, id, title):
-        """Create the main frame object for the application."""
-        wxFrame.__init__(self, parent, id, title)
-        intro = 'Welcome To PyCrust %s - The Flakiest Python Shell' % VERSION
-        self.CreateStatusBar()
-        self.SetStatusText(intro)
-        self.icon = wxIcon('PyCrust.ico', wxBITMAP_TYPE_ICO)
-        self.SetIcon(self.icon)
-        self.createMenus()
-        # Create the shell, which will create a default interpreter.
-        locals = {'__app__': 'PyCrust Application'}
-        self.shell = Shell(parent=self, id=-1, introText=intro, locals=locals)
-        # Override the shell so that status messages go to the status bar.
-        self.shell.setStatusText = self.SetStatusText
-
-    def createMenus(self):
-        m = self.fileMenu = wxMenu()
-        m.AppendSeparator()
-        m.Append(wxID_EXIT, 'E&xit', 'Exit PyCrust')
-
-        m = self.editMenu = wxMenu()
-        m.Append(wxID_UNDO, '&Undo \tCtrl+Z', 'Undo the last action')
-        m.Append(wxID_REDO, '&Redo \tCtrl+Y', 'Redo the last undone action')
-        m.AppendSeparator()
-        m.Append(wxID_CUT, 'Cu&t \tCtrl+X', 'Cut the selection')
-        m.Append(wxID_COPY, '&Copy \tCtrl+C', 'Copy the selection')
-        m.Append(wxID_PASTE, '&Paste \tCtrl+V', 'Paste')
-        m.AppendSeparator()
-        m.Append(wxID_CLEAR, 'Cle&ar \tDel', 'Delete the selection')
-        m.Append(wxID_SELECTALL, 'Select A&ll \tCtrl+A', 'Select all text')
-
-        m = self.autocompMenu = wxMenu()
-        m.Append(ID_AUTOCOMP_SHOW, 'Show Auto Completion', \
-                 'Show auto completion during dot syntax', checkable=1)
-        m.Append(ID_AUTOCOMP_INCLUDE_MAGIC, 'Include Magic Attributes', \
-                 'Include attributes visible to __getattr__ and __setattr__', checkable=1)
-        m.Append(ID_AUTOCOMP_INCLUDE_SINGLE, 'Include Single Underscores', \
-                 'Include attibutes prefixed by a single underscore', checkable=1)
-        m.Append(ID_AUTOCOMP_INCLUDE_DOUBLE, 'Include Double Underscores', \
-                 'Include attibutes prefixed by a double underscore', checkable=1)
-
-        m = self.calltipsMenu = wxMenu()
-        m.Append(ID_CALLTIPS_SHOW, 'Show Call Tips', \
-                 'Show call tips with argument specifications', checkable=1)
-
-        m = self.optionsMenu = wxMenu()
-        m.AppendMenu(ID_AUTOCOMP, '&Auto Completion', self.autocompMenu, \
-                     'Auto Completion Options')
-        m.AppendMenu(ID_CALLTIPS, '&Call Tips', self.calltipsMenu, \
-                     'Call Tip Options')
-
-        m = self.helpMenu = wxMenu()
-        m.AppendSeparator()
-        m.Append(wxID_ABOUT, '&About...', 'About PyCrust')
-
-        b = self.menuBar = wxMenuBar()
-        b.Append(self.fileMenu, '&File')
-        b.Append(self.editMenu, '&Edit')
-        b.Append(self.optionsMenu, '&Options')
-        b.Append(self.helpMenu, '&Help')
-        self.SetMenuBar(b)
-
-        EVT_MENU(self, wxID_EXIT, self.OnExit)
-        EVT_MENU(self, wxID_UNDO, self.OnUndo)
-        EVT_MENU(self, wxID_REDO, self.OnRedo)
-        EVT_MENU(self, wxID_CUT, self.OnCut)
-        EVT_MENU(self, wxID_COPY, self.OnCopy)
-        EVT_MENU(self, wxID_PASTE, self.OnPaste)
-        EVT_MENU(self, wxID_CLEAR, self.OnClear)
-        EVT_MENU(self, wxID_SELECTALL, self.OnSelectAll)
-        EVT_MENU(self, wxID_ABOUT, self.OnAbout)
-        EVT_MENU(self, ID_AUTOCOMP_SHOW, self.OnAutoCompleteShow)
-        EVT_MENU(self, ID_AUTOCOMP_INCLUDE_MAGIC, self.OnAutoCompleteIncludeMagic)
-        EVT_MENU(self, ID_AUTOCOMP_INCLUDE_SINGLE, self.OnAutoCompleteIncludeSingle)
-        EVT_MENU(self, ID_AUTOCOMP_INCLUDE_DOUBLE, self.OnAutoCompleteIncludeDouble)
-        EVT_MENU(self, ID_CALLTIPS_SHOW, self.OnCallTipsShow)
-
-        EVT_UPDATE_UI(self, wxID_UNDO, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, wxID_REDO, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, wxID_CUT, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, wxID_COPY, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, wxID_PASTE, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, wxID_CLEAR, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, ID_AUTOCOMP_SHOW, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, ID_AUTOCOMP_INCLUDE_MAGIC, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, ID_AUTOCOMP_INCLUDE_SINGLE, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, ID_AUTOCOMP_INCLUDE_DOUBLE, self.OnUpdateMenu)
-        EVT_UPDATE_UI(self, ID_CALLTIPS_SHOW, self.OnUpdateMenu)
-
-    def OnExit(self, event):
-        self.Close(true)
-
-    def OnUndo(self, event):
-        self.shell.Undo()
-
-    def OnRedo(self, event):
-        self.shell.Redo()
-
-    def OnCut(self, event):
-        self.shell.Cut()
-
-    def OnCopy(self, event):
-        self.shell.Copy()
-
-    def OnPaste(self, event):
-        self.shell.Paste()
-
-    def OnClear(self, event):
-        self.shell.Clear()
-
-    def OnSelectAll(self, event):
-        self.shell.SelectAll()
-
-    def OnAbout(self, event):
-        """Display an About PyCrust window."""
-        title = 'About PyCrust'
-        text = 'PyCrust %s\n\n' % VERSION + \
-               'Yet another Python shell, only flakier.\n\n' + \
-               'Half-baked by Patrick K. O\'Brien,\n' + \
-               'the other half is still in the oven.\n\n' + \
-               'Shell Revision: %s\n' % self.shell.revision + \
-               'Interpreter Revision: %s\n' % self.shell.interp.revision
-        dialog = wxMessageDialog(self, text, title, wxOK | wxICON_INFORMATION)
-        dialog.ShowModal()
-        dialog.Destroy()
-
-    def OnAutoCompleteShow(self, event):
-        self.shell.autoComplete = event.IsChecked()
-
-    def OnAutoCompleteIncludeMagic(self, event):
-        self.shell.autoCompleteIncludeMagic = event.IsChecked()
-
-    def OnAutoCompleteIncludeSingle(self, event):
-        self.shell.autoCompleteIncludeSingle = event.IsChecked()
-
-    def OnAutoCompleteIncludeDouble(self, event):
-        self.shell.autoCompleteIncludeDouble = event.IsChecked()
-
-    def OnCallTipsShow(self, event):
-        self.shell.autoCallTip = event.IsChecked()
-
-    def OnUpdateMenu(self, event):
-        """Update menu items based on current status."""
-        id = event.GetId()
-        if id == wxID_UNDO:
-            event.Enable(self.shell.CanUndo())
-        elif id == wxID_REDO:
-            event.Enable(self.shell.CanRedo())
-        elif id == wxID_CUT:
-            event.Enable(self.shell.CanCut())
-        elif id == wxID_COPY:
-            event.Enable(self.shell.CanCopy())
-        elif id == wxID_PASTE:
-            event.Enable(self.shell.CanPaste())
-        elif id == wxID_CLEAR:
-            event.Enable(self.shell.CanCut())
-        elif id == ID_AUTOCOMP_SHOW:
-            event.Check(self.shell.autoComplete)
-        elif id == ID_AUTOCOMP_INCLUDE_MAGIC:
-            event.Check(self.shell.autoCompleteIncludeMagic)
-        elif id == ID_AUTOCOMP_INCLUDE_SINGLE:
-            event.Check(self.shell.autoCompleteIncludeSingle)
-        elif id == ID_AUTOCOMP_INCLUDE_DOUBLE:
-            event.Check(self.shell.autoCompleteIncludeDouble)
-        elif id == ID_CALLTIPS_SHOW:
-            event.Check(self.shell.autoCallTip)
-            
-
-class App(wxApp):
-    def OnInit(self):
-        parent = None
-        id = -1
-        title = 'PyCrust'
-        self.frame = Frame(parent, id, title)
-        self.frame.Show(true)
-        self.SetTopWindow(self.frame)
-        return true
-
-
-def main():
-    import sys
-    application = App(0)
-    # Add the application object to the sys module's namespace.
-    # This allows a shell user to do:
-    # >>> import sys
-    # >>> sys.application.whatever
-    sys.application = application
-    application.MainLoop()
-
-if __name__ == '__main__':
-    main()

wxPython/wxPython/lib/editor/py_editor.py

@@ -1,211 +0,0 @@
-# (C)opyright by Dirk Holtwick, 1999
-# ----------------------------------
-# holtwick@spirito.de
-# http://www.spirito.de/pyde
-
-from editor import *
-from string import *
-from keyword import *
-from tokenizer import *
-
-"""
-This module will be loaded by the main
-window. It implements some methods that
-are typical for Python sources.
-"""
-
-class wxPyEditor(wxEditor):
-
-    # ------------------------------------------------------------------
-
-    def __init__(self, parent, id,
-                 pos=wxDefaultPosition, size=wxDefaultSize, style=0):
-        wxEditor.__init__(self, parent, id, pos, size, style)
-        self.SetFontTab([
-            wxNamedColour('black'),
-            wxNamedColour('blue'),
-            wxNamedColour('red'),
-            wxNamedColour('darkgreen'),
-            wxNamedColour('brown')
-            ])
-
-    # ------------------------------------------------------------------
-
-    def OnUpdateHighlight(self, line = -1):
-        if line>=0:
-            t   = self.text[line].text
-            syn = []
-
-            toks = Tokenizer(t).tokens()
-            for type, string, begin, end in toks:
-                if type == "KEY":
-                    syn.append((begin,  1))
-                    syn.append((end,    0))
-                elif type == "COMMENT":
-                    syn.append((begin,  2))
-                elif type == "STRING":
-                    syn.append((begin,  3))
-                    syn.append((end,    0))
-                elif type == "NUMBER":
-                    syn.append((begin,  4))
-                    syn.append((end,    0))
-                elif type == "NAME":
-                    if string=="self":
-                        syn.append((begin,  4))
-                        syn.append((end,    0))
-                else:
-                    pass
-            self.text[line].syntax = syn
-
-    # ------------------------------------------------------------------
-
-    def OnUpdateSyntax(self, line = -1):
-        if line>=0:
-            """
-            tx, syn, m = self.text[line]
-            pre = 0
-            for i in range(0,len(tx)):
-                if tx[i] != " ":
-                    pre = i
-                    break
-            t = tx[pre:]
-
-            t = Tokenizer(t).line()
-
-            t = tx[:pre] + t
-            self.text[line] = t, syn, m
-            """
-            self.OnUpdateHighlight(line)
-
-    # ------------------------------------------------------------------
-
-    def OnTabulator(self, event):
-        add = +1
-        if event.ShiftDown():
-            add = -1
-        t = self.GetTextLine(self.cy)
-        if strip(t):
-            indent = self.GetIndent(t)
-#           print indent
-            t = t[indent:]
-            tabs = indent / self.tabsize
-#            for i in range(0,tabs+add):
-            t = (" " * 4 * (tabs+add)) + t
-            self.SetTextLine(self.cy, t)
-        elif add>0:
-            self.InsertText("    ")
-
-    # ------------------------------------------------------------------
-
-    def FindQuote(self, lineno, quote_type='"""', direction=1):
-        """find line containing the matching quote"""
-        l =lineno +direction
-        while (l < len(self.text)-1) and (l >= 0):
-            if find(self.text[l].text, quote_type) >=0: return l
-            l =l +direction
-        return None
-
-    def FindNextLine(self, lineno, direction=1):
-        """get the next line of code (skipping comment lines and empty lines)"""
-        l =lineno +direction
-        while (l < len(self.text)-1) and (l >= 0):
-            str =lstrip(self.text[l].text)
-            if (len(str) >0) and (str[0] !="#"): return l
-            l =l +direction
-        return None
-
-    def Fold(self):
-        l       = self.GetLine(self.cy)
-        line    = self.text[l]
-        t       = line.text
-
-        # fold ...
-        if line.editable:
-
-            # 3*quotes
-            qpos =find(t, '"""')
-            if qpos >=0: qtype ='"""'
-            else:
-                qpos =find(t, "'''")
-                if qpos >=0: qtype ="'''"
-
-            if (qpos >=0) and (find(t[qpos+3:], qtype) <0):
-                closing_quote =self.FindQuote(l, qtype)
-                if closing_quote !=None:
-                    line.editable = not line.editable
-                    l =l +1
-                    while l <= closing_quote:
-                       self.text[l].visible =self.text[l].visible +1
-                       l =l +1
-
-            else:  # try normal fold on leading whitespace
-                lim = self.GetIndent(t)
-                lnext =self.FindNextLine(l)
-                if (lnext !=None) \
-                and (self.GetIndent(self.text[lnext].text) >lim):
-                    line.editable =FALSE
-                    lstart =l +1
-                    l =self.FindNextLine(l)
-                    while (l !=None) \
-                    and (self.GetIndent(self.text[l].text) >lim):
-                        l =self.FindNextLine(l)
-                    if l ==None:
-                        # fold till the end
-                        l =len(self.text)
-                    for line in self.text[lstart:l]:
-                        line.visible =line.visible +1
-
-        # ... or unfold
-        else:
-            lim = line.visible + 1
-            line.editable = not line.editable
-
-            l = l + 1
-            line = self.text[l]
-            while (l < (len(self.text) -1)) and (line.visible>=lim):
-                line.visible = line.visible - 1
-                l = l + 1
-                line = self.text[l]
-
-    def FoldAll(self):
-        self.CalcLines()
-        self.cx = 0
-        self.cy = len(self.lines) - 1
-        prev_indent =0
-        # following loop is exited in two cases:
-        # when self.cy becomes 0 (topmost level is not folded by FoldAll)
-        # or when FindNextLine() returns None (all remaining lines till
-        #               the beginning of the text are empty or comments)
-        while self.cy:
-            t = self.GetTextLine(self.cy)
-            # indent-based folding
-            indent =self.GetIndent(t)
-            if indent <prev_indent:
-                self.Fold()
-            prev_indent =indent
-            # triple-quote folding
-            qpos =find(t, '"""')
-            if qpos >=0: qtype ='"""'
-            else:
-                qpos =find(t, "'''")
-                if qpos >=0: qtype ="'''"
-            if (qpos >=0) and (find(t[qpos+3:], qtype) <0):
-                closing_quote =self.FindQuote(self.cy, qtype, -1)
-                if closing_quote !=None:
-                    # XXX potential bug: unmatched triple quotes
-                    self.cy =closing_quote
-                    self.Fold()
-            self.cy =self.FindNextLine(self.cy, -1)
-        if self.cy ==None: self.cy =0
-
-    # ------------------------------------------------------------------
-
-    def OnFold(self):
-        self.Fold()
-
-    # ------------------------------------------------------------------
-
-    def OnInit(self):
-        #self.FoldAll()
-        pass
-

wxPython/wxPython/lib/editor/tokenizer.py

@@ -1,60 +0,0 @@
-from tokenize import *
-from keyword import *
-from string import *
-
-class Tokenizer:
-    """
-    Simple class to create a list of token-tuples like:
-
-    	(type, string, first, last)
-
-    Example:
-    t = Tokenizer('def hallo(du): # juchee')
-    print t.tokens()
-    """
-
-    def __init__(self, text):
-    	self.text = text
-    	self.toks = []
-    	try:
-    	    tokenize(self.readline, self.get)
-	except TokenError:
-	    pass
-
-    def tokens(self):
-    	return self.toks
-
-    def get(self, type, string, begin, end, l):
-    	#print begin,end
-    	h1, b = begin
-	h2, e = end
-	tname = tok_name[type]
-   	if iskeyword(string):
-    	    tname = "KEY"
-    	self.toks.append( (tname, string, b, e) )
-
-    def readline(self):
-    	t = self.text
-	self.text = ""
-	return t
-
-    def line(self):
-    	pre = ""
-    	out = ""
-	for type, string, begin, end in self.toks:
-    	    if (pre in ["NAME","KEY"]) and (not string in [".",",","("]):
-	    	out = out + " "
-
-	    if type in ["NAME","KEY"]:
-		out = out + string
-	    elif type=="OP":
-	    	if string in [",",":"]:
-		    out = out + string + " "
-		else:
-       		    out = out + string
-	    else:
-    	        out = out + string
-    	    pre = type
-	return out
-
-