1b55cabf50
Some tweaks and fixes Some distrib updates git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@5161 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775
61 lines
1.2 KiB
Python
61 lines
1.2 KiB
Python
from tokenize import *
|
|
from keyword import *
|
|
from string import *
|
|
|
|
class Tokenizer:
|
|
"""
|
|
Simple class to create a list of token-tuples like:
|
|
|
|
(type, string, first, last)
|
|
|
|
Example:
|
|
t = Tokenizer('def hallo(du): # juchee')
|
|
print t.tokens()
|
|
"""
|
|
|
|
def __init__(self, text):
|
|
self.text = text
|
|
self.toks = []
|
|
try:
|
|
tokenize(self.readline, self.get)
|
|
except TokenError:
|
|
pass
|
|
|
|
def tokens(self):
|
|
return self.toks
|
|
|
|
def get(self, type, string, begin, end, l):
|
|
#print begin,end
|
|
h1, b = begin
|
|
h2, e = end
|
|
tname = tok_name[type]
|
|
if iskeyword(string):
|
|
tname = "KEY"
|
|
self.toks.append(tname, string, b, e)
|
|
|
|
def readline(self):
|
|
t = self.text
|
|
self.text = ""
|
|
return t
|
|
|
|
def line(self):
|
|
pre = ""
|
|
out = ""
|
|
for type, string, begin, end in self.toks:
|
|
if (pre in ["NAME","KEY"]) and (not string in [".",",","("]):
|
|
out = out + " "
|
|
|
|
if type in ["NAME","KEY"]:
|
|
out = out + string
|
|
elif type=="OP":
|
|
if string in [",",":"]:
|
|
out = out + string + " "
|
|
else:
|
|
out = out + string
|
|
else:
|
|
out = out + string
|
|
pre = type
|
|
return out
|
|
|
|
|