Added wxEditor from Dirc Holtwic

Some tweaks and fixes
Some distrib updates


git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@5161 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

utils/wxPython/lib/editor/tokenizer.py

@@ -0,0 +1,60 @@
+from tokenize import *
+from keyword import *
+from string import *
+
+class Tokenizer:
+    """
+    Simple class to create a list of token-tuples like:
+
+    	(type, string, first, last)
+
+    Example:
+    t = Tokenizer('def hallo(du): # juchee')
+    print t.tokens()
+    """
+
+    def __init__(self, text):
+    	self.text = text
+    	self.toks = []
+    	try:
+    	    tokenize(self.readline, self.get)
+	except TokenError:
+	    pass
+
+    def tokens(self):
+    	return self.toks
+
+    def get(self, type, string, begin, end, l):
+    	#print begin,end
+    	h1, b = begin
+	h2, e = end
+	tname = tok_name[type]
+   	if iskeyword(string):
+    	    tname = "KEY"
+    	self.toks.append(tname, string, b, e)
+
+    def readline(self):
+    	t = self.text
+	self.text = ""
+	return t
+
+    def line(self):
+    	pre = ""
+    	out = ""
+	for type, string, begin, end in self.toks:
+    	    if (pre in ["NAME","KEY"]) and (not string in [".",",","("]):
+	    	out = out + " "
+
+	    if type in ["NAME","KEY"]:
+		out = out + string
+	    elif type=="OP":
+	    	if string in [",",":"]:
+		    out = out + string + " "
+		else:
+       		    out = out + string
+	    else:
+    	        out = out + string
+    	    pre = type
+	return out
+
+