DocView and ActiveGrid IDE updates from Morgan Hua:

New Features: In Tab-View mode, Ctrl-number will take the user to
    the numbered tab view.  Modified files now show an '*' astrisk in
    the view title.  Debugger framework can now support PHP debugging.
    Not important for python development, but at least that means the
    debugger framework is more generalized.


git-svn-id: https://svn.wxwidgets.org/svn/wx/wxWidgets/trunk@38852 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

wxPython/samples/ide/activegrid/util/aglogging.py

@@ -19,6 +19,7 @@ import logging.config
 from activegrid.util.lang import *
 import activegrid.util.objutils as objutils
 import activegrid.util.sysutils as sysutils
+import activegrid.util.appdirs as appdirs
 
 LEVEL_FATAL = logging.FATAL
 LEVEL_ERROR = logging.ERROR
@@ -27,38 +28,47 @@ LEVEL_INFO = logging.INFO
 LEVEL_DEBUG = logging.DEBUG
 
 EXCEPTION_INFO = 'exceptionInfo'
+loggingInitialized = False
 
 LOG_MODE_IDE = 1
 LOG_MODE_TESTRUN = 2
 LOG_MODE_RUN = 3
-def initLogging(mode):
-    configFile = None
-    if (mode == LOG_MODE_IDE):
-        configFile = os.getenv("AG_LOGCONFIG_IDE")
-    elif (mode == LOG_MODE_TESTRUN):
-        configFile = os.getenv("AG_LOGCONFIG_TESTRUN")
-    else:
-        configFile = os.getenv("AG_LOGCONFIG_RUN")
-    if ((configFile == None) or not os.path.exists(configFile)):
+def initLogging(mode, force=False):
+    global ag_debugLogger, loggingInitialized
+    if (force or not loggingInitialized):
+        loggingInitialized = True
+        configFile = None
         if (mode == LOG_MODE_IDE):
-            configFile = "IDELog"
+            configFile = os.getenv("AG_LOGCONFIG_IDE")
         elif (mode == LOG_MODE_TESTRUN):
-            configFile = "TestRunLog"
+            configFile = os.getenv("AG_LOGCONFIG_PYTESTRUN")
         else:
-            configFile = "RunLog"
-        configFile = sysutils.mainModuleDir + "/py" + configFile + ".ini"
-    if (os.path.exists(configFile)):
-        fileConfig(configFile)
-    else:
-        defaultStream = sys.stderr
-        if (mode == LOG_MODE_RUN):
-            defaultStream = sys.stdout
-        handler = logging.StreamHandler(defaultStream)
-        handler.setLevel(logging.INFO)
-        handler.setFormatter(logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s"))
-        logging.getLogger().addHandler(handler)
-    return configFile
-
+            configFile = os.getenv("AG_LOGCONFIG_RUN")
+        if ((configFile == None) or not os.path.exists(configFile)):
+            if (mode == LOG_MODE_IDE):
+                configFile = "IDELog"
+            elif (mode == LOG_MODE_TESTRUN):
+                configFile = "TestRunLog"
+            else:
+                configFile = "RunLog"
+            configFile = os.path.join(appdirs.getSystemDir(appdirs.AG_LOGS_DIR), "py" + configFile + ".ini")
+        if (os.path.exists(configFile)):
+            print "Using logging configuration file: %s" % configFile
+            fileConfig(configFile)
+        else:
+            print "*** Cannot find logging configuration file (%s) -- setting default logging level to WARN ***" % (configFile)
+            defaultStream = sys.stderr
+            if (mode == LOG_MODE_RUN):
+                defaultStream = sys.stdout
+            handler = logging.StreamHandler(defaultStream)
+            handler.setLevel(logging.DEBUG)
+            handler.setFormatter(logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s"))
+            logging.getLogger().addHandler(handler)
+            logging.getLogger().setLevel(logging.WARN)
+        ag_debugLogger = logging.getLogger("activegrid.debug")
+        ag_debugLogger.setLevel(logging.DEBUG)
+        return configFile
+    
 ag_debugLogger = logging.getLogger("activegrid.debug")
 
 def log(logger, level, msg, *params):
@@ -181,21 +191,18 @@ def addExceptionInfo(e, key, value):
     if not e.exceptionInfo.has_key(key): # Never overwrite exception info since we assume earlier info is more specific
         e.exceptionInfo[key] = value
             
-def reportException(out=None, stacktrace=False, diffable=False, exception=None):
+def reportException(exception, out=None, stacktrace=False, diffable=False):
     exstr = exceptionToString(exception, stacktrace, diffable)
     if (out == None):
         print exstr
     else:
         print >> out, exstr
 
-def exceptionToString(exception=None, stacktrace=False, diffable=False):
-    if (exception == None):
-        extype, val, t = sys.exc_info()
-    else:
-        extype = objutils.typeToString(exception)
-        val = exception
-        if (stacktrace):
-            e,v,t = sys.exc_info()
+def exceptionToString(exception, stacktrace=False, diffable=False):
+    extype = objutils.typeToString(exception)
+    val = exception
+    if (stacktrace):
+        e,v,t = sys.exc_info()
     if (diffable):
         exstr = removeFileRefs(str(val))
     else:

wxPython/samples/ide/activegrid/util/appdirs.py

@@ -2,7 +2,7 @@
 # Name:         appdirs.py
 # Purpose:      Utilities for retrieving special application dirs
 #
-# Author:       Kevin Ollivier
+# Author:       Kevin Ollivier, Jeff Norton
 #
 # Created:      8/27/05
 # CVS-ID:       $Id$
@@ -10,50 +10,117 @@
 # License:      wxWindows License
 #----------------------------------------------------------------------------
 
-# NOTE: This was made a separate file because it depends upon the 
-# wx.StandardPaths module, and thus, on wxWidgets, unlike other
-# utils modules. I wanted to ensure this module is never loaded
-# from the web server, etc.
-
+from activegrid.util.lang import *
 import sys
 import os
 import string
-import wx
+import activegrid.util.sysutils as sysutils
 
-def isWindows():
-    return os.name == 'nt'
-
-def _generateDocumentsDir():
-    path = ""
-    if sys.platform == "win32":
-        from win32com.shell import shell, shellcon
-        path=shell.SHGetFolderPath(0, shellcon.CSIDL_PERSONAL, None, 0)
-    elif sys.platform == "darwin":
-        import macfs, MACFS
-        fsspec_disk, fsspec_desktop = macfs.FindFolder( MACFS.kOnSystemDisk, MACFS.kDocumentsFolderType, 0)
-        path = macfs.FSSpec((fsspec_disk, fsspec_desktop, '')).as_pathname()
-    
-    if path == "":
-        path = os.path.expanduser("~")
-
-    return path
+def _getSystemDir(kind):
+    if (kind == AG_LOGS_DIR):
+        return os.path.join(getSystemDir(AG_SYSTEM_DIR) , "logs")
+    elif (kind == AG_DEMOS_DIR):
+        return os.path.join(getSystemDir(AG_SYSTEM_DIR), "demos")
+    else:
+        path = ""
+        if (sysutils.isServer()):
+            path = os.getenv("ACTIVEGRID_SERVER_HOME")
+            if ((path is None) or (len(path) < 1)):
+                path = sysutils.mainModuleDir
+        else:
+            path = os.getenv("AG_DOCUMENTS_DIR")
+            if ((path is None) or (len(path) < 1)):
+                if sysutils.isWindows():
+                    ifDefPy()
+                    try:
+                        from win32com.shell import shell, shellcon
+                        path = shell.SHGetFolderPath(0, shellcon.CSIDL_PERSONAL, None, 0)
+                    except:
+                        pass
+                    endIfDef()
+                    if ((path is None) or (len(path) < 1)):
+                        homedrive = asString(os.getenv("HOMEDRIVE"))
+                        homepath = os.getenv("HOMEPATH")
+##                        if ((homedrive is not None) and (len(homedrive) > 0) and (homepath is not None) and (len(homepath) > 0)):
+                        path = os.path.join(homedrive, homepath, "MYDOCU~1")
+                else:
+                    ifDefPy()
+                    if sys.platform == "darwin":
+                        try:
+                            import macfs
+                            import MACFS
+                            fsspec_disk, fsspec_desktop = macfs.FindFolder(MACFS.kOnSystemDisk, MACFS.kDocumentsFolderType, 0)
+                            path = macfs.FSSpec((fsspec_disk, fsspec_desktop, '')).as_pathname()
+                        except:
+                            pass
+                    endIfDef()
+                
+                ifDefPy()
+                if ((path is None) or (len(path) < 1)):
+                    path = os.path.expanduser("~")
+                endIfDef()
+                if ((path is None) or (len(path) < 1)):
+                    path = "/"
+                path = os.path.join(path, "ActiveGrid")
             
-documents_folder = _generateDocumentsDir()
+        return path
+        
+
+AG_SYSTEM_DIR = 0
+AG_LOGS_DIR = 1
+AG_DEMOS_DIR = 2
+
+__systemDir = None
+__logsDir = None
+__demosDir = None
+
+def getSystemDir(kind=0):
+    if (kind == AG_SYSTEM_DIR):
+        global __systemDir
+        if (__systemDir is None):
+            __systemDir = _getSystemDir(kind)
+        return __systemDir
+    elif (kind == AG_LOGS_DIR):
+        global __logsDir
+        if (__logsDir is None):
+            __logsDir = _getSystemDir(kind)
+        return __logsDir
+    elif (kind == AG_DEMOS_DIR):
+        global __demosDir
+        if (__demosDir is None):
+            __demosDir = _getSystemDir(kind)
+        return __demosDir
+    return None
+
 
 # NOTE: We don't set this at startup because wxStandardPaths needs a running
 # application object. This makes sure the wxApp will always be created when
 # we get the folder.
+ifDefPy()
 def getAppDataFolder():
-    # wxStandardPaths requires a running app
-    if wx.GetApp() and wx.Platform != "__WXGTK__":
-        data_folder = wx.StandardPaths.Get().GetUserDataDir()
-        if not os.path.exists(data_folder):
-            os.mkdir(data_folder)
-        return data_folder
-    else:
-        # wxBug: on *nix, it wants to point to ~/.appname, but
-        # so does wxConfig... For now, redirect this to ~/.appbuilder
-        # when this is fixed, we'll migrate settings to the correct place
-        return os.path.join(os.path.expanduser("~"), ".appbuilder")
+    try:
+        # NOTE: cannot import wx from the server
+        import wx
+        # wxStandardPaths requires a running app
+        if wx.GetApp() and wx.Platform != "__WXGTK__":
+            data_folder = wx.StandardPaths.Get().GetUserDataDir()
+            if not os.path.exists(data_folder):
+                os.mkdir(data_folder)
+            return data_folder
+    except:
+        pass
+    # wxBug: on *nix, it wants to point to ~/.appname, but
+    # so does wxConfig... For now, redirect this to ~/.appbuilder
+    # when this is fixed, we'll migrate settings to the correct place
+    return os.path.join(os.path.expanduser("~"), ".appbuilder")
+endIfDef()
 
-    return ""
+ifDefPy()
+def createSystemDirs():
+    if (not os.path.exists(getSystemDir())):
+        os.mkdir(getSystemDir())
+    if (not os.path.exists(getSystemDir(AG_LOGS_DIR))):
+        os.mkdir(getSystemDir(AG_LOGS_DIR))
+    if (not os.path.exists(getSystemDir(AG_DEMOS_DIR))):
+        os.mkdir(getSystemDir(AG_DEMOS_DIR))
+endIfDef()

wxPython/samples/ide/activegrid/util/datetimeparser.py

@@ -0,0 +1,118 @@
+#----------------------------------------------------------------------------
+# Name:         datetimeparser.py
+#
+# Purpose:      - Instantiate datetime.datetime/date instance from a string
+#                 date representation.
+#                 Uses dateutil from http://labix.org/python-dateutil.
+#
+#               - Creates string representation of datetime/date instance.
+#
+#
+# Author:       Simon Toens
+#
+# Created:      28-Feb-06
+# CVS-ID:
+# Copyright:    (c) 2005 ActiveGrid, Inc.
+# License:      wxWindows License
+#----------------------------------------------------------------------------
+
+import datetime
+
+try:
+    import dateutil.parser
+    DATEUTIL_INSTALLED = True
+except ImportError:
+    DATEUTIL_INSTALLED = False
+
+ISO_8601_DATE_FORMAT = "%Y-%m-%d"
+ISO_8601_TIME_FORMAT = "%H:%M:%S"
+ISO_8601_DATETIME_FORMAT = "%s %s" %(ISO_8601_DATE_FORMAT,
+                                     ISO_8601_TIME_FORMAT)
+
+DEFAULT_DATETIME = datetime.datetime(1, 1, 1, 0, 0, 0, 0)
+
+
+def format(dateobj, formatstr=None):
+    if (formatstr != None and _isDateTimeObject(dateobj)):
+        return dateobj.strftime(str(formatstr))
+    return str(dateobj)
+
+
+def parse(datestr, formatstr=None, asdate=False, astime=False):
+    """Instantiates and returns a datetime instance from the datestr datetime
+       representation.
+       
+       Optionally, a format string may be used. The format is only loosely
+       interpreted, its only purpose beeing to determine if the year is first
+       or last in datestr, and whether the day is in front or follows the
+       month. If no formatstr is passed in, dateutil tries its best to parse
+       the datestr. The default date format is YYYY-mm-dd HH:SS.
+       
+       If asdate is True, returns a date instance instead of a datetime
+       instance, if astime is True, returns a time instance instead of a
+       datetime instance."""
+    
+       
+    dayfirst, yearfirst = _getDayFirstAndYearFirst(formatstr)
+
+    rtn = None
+
+    try:
+        if DATEUTIL_INSTALLED:
+            rtn = dateutil.parser.parse(str(datestr), fuzzy=True,
+                                        dayfirst=dayfirst, yearfirst=yearfirst,
+                                        default=DEFAULT_DATETIME)
+        else:
+            rtn = DEFAULT_DATETIME
+    except:
+        rtn = DEFAULT_DATETIME
+
+    if (asdate and isinstance(rtn, datetime.datetime)):
+        rtn = datetime.date(rtn.year, rtn.month, rtn.day)
+    elif (astime and isinstance(rtn, datetime.datetime)):
+        rtn = datetime.time(rtn.hour, rtn.minute, rtn.second, rtn.microsecond)
+
+    return rtn
+
+
+def _isDateTimeObject(obj):
+    return (isinstance(obj, datetime.datetime) or
+            isinstance(obj, datetime.date) or
+            isinstance(obj, datetime.time))
+
+
+def _getDayFirstAndYearFirst(formatstr):
+    dayFirst = False
+    yearFirst = False
+    
+    gotYear = False
+    gotMonth = False
+    gotDay = False
+
+    if (formatstr == None):
+        formatstr = ""
+    
+    for c in formatstr:
+        if (c.lower() == "y"):
+            if (gotYear):
+                continue
+            if (not gotDay and not gotMonth):
+                yearFirst = True
+            gotYear = True
+
+        elif (c.lower() == "m"):
+            if (gotMonth):
+                continue
+            if (not gotDay):
+                dayFirst = False
+            gotMonth = True
+            
+        elif (c.lower() == "d"):
+            if (gotDay):
+                continue
+            if (not gotMonth):
+                dayFirst = True
+            gotDay = True
+        
+
+    return dayFirst, yearFirst

wxPython/samples/ide/activegrid/util/fileutils.py

@@ -19,6 +19,7 @@ import zipfile
 
 import activegrid.util.aglogging as aglogging
 import activegrid.util.sysutils as sysutils
+import activegrid.util.utillang as utillang
 from activegrid.util.lang import *
 
 global fileutilsLogger
@@ -31,6 +32,65 @@ fileutilsLogger = logging.getLogger("activegrid.util.fileutils")
 aglogging.setLevelFatal(fileutilsLogger)
 #logging.getLogger().addHandler(logging.StreamHandler(sys.stderr))
 
+def addRef(varname):
+    return "${%s}" % varname
+
+AG_SYSTEM_VAR_NAMES = [] # all AG System vars, with ${} syntax
+
+AG_SYSTEM_VAR = "AG_SYSTEM"
+AG_SYSTEM_VAR_REF = addRef(AG_SYSTEM_VAR)
+AG_SYSTEM_VAR_NAMES.append(AG_SYSTEM_VAR_REF)
+
+AG_SYSTEM_STATIC_VAR = "AG_SYSTEM_STATIC"
+AG_SYSTEM_STATIC_VAR_REF = addRef(AG_SYSTEM_STATIC_VAR)
+AG_SYSTEM_VAR_NAMES.append(AG_SYSTEM_STATIC_VAR_REF)
+
+AG_APP_VAR = "AG_APP"
+AG_APP_STATIC_VAR = "AG_APP_STATIC"
+
+# _initAGSystemVars needs to be called to initialize the following two
+# containers:
+EXPANDED_AG_SYSTEM_VARS = {} # ${varname} -> value (path)
+# ${varname}, ordered from longest to shortest path value
+AG_SYSTEM_VARS_LENGTH_ORDER = [] 
+
+def _initAGSystemVars():
+    if (len(EXPANDED_AG_SYSTEM_VARS) > 0):
+        return
+    
+    for v in AG_SYSTEM_VAR_NAMES:
+        EXPANDED_AG_SYSTEM_VARS[v] = os.path.abspath(expandVars(v))
+        AG_SYSTEM_VARS_LENGTH_ORDER.append(v)
+        
+    AG_SYSTEM_VARS_LENGTH_ORDER.sort(_sortByValLength)
+
+
+def parameterizePathWithAGSystemVar(inpath):
+    """Returns parameterized path if path starts with a known AG directory. Otherwise returns path as it was passed in."""
+    _initAGSystemVars()
+    path = inpath
+    if not sysutils.isWindows():
+        # ensure we have forward slashes
+        path = path.replace("\\", "/")
+        
+    path = os.path.abspath(path)
+
+    for varname in AG_SYSTEM_VARS_LENGTH_ORDER:
+        varval = EXPANDED_AG_SYSTEM_VARS[varname]
+        if path.startswith(varval):
+            return path.replace(varval, varname)
+        
+    return inpath
+
+def startsWithAgSystemVar(path):
+    """Returns True if path starts with a known AG system env var, False otherwise."""
+    for varname in AG_SYSTEM_VAR_NAMES:
+        if path.startswith(varname):
+            return True
+    return False
+        
+def _sortByValLength(v1, v2):
+    return len(EXPANDED_AG_SYSTEM_VARS[v2]) - len(EXPANDED_AG_SYSTEM_VARS[v1])
 
 def makeDirsForFile(filename):
     d = os.path.dirname(filename)
@@ -44,7 +104,7 @@ def createFile(filename, mode='w'):
     f = file(filename, mode)
     return f
 
-def compareFiles(file1, file2):
+def compareFiles(file1, file2, ignore=None):
 ##    result = filecmp.cmp(file1, file2)
 ##    if result:
 ##        return 0
@@ -62,6 +122,9 @@ def compareFiles(file1, file2):
         elif (len(line2) == 0):
             return -1
         elif (line1 != line2):
+            if (ignore != None):
+                if (line1.startswith(ignore) or line2.startswith(ignore)):
+                    continue
             line1 = line1.replace(" ", "")
             line2 = line2.replace(" ", "")
             if (line1 != line2):
@@ -81,7 +144,10 @@ def compareFiles(file1, file2):
                         continue
                 return -1
 
-def expandVars(value):
+def expandKnownAGVars(value):
+    return expandVars(value, includeEnv=False)
+
+def expandVars(value, includeEnv=True):
     """Syntax: ${myvar,default="default value"}"""
     import activegrid.runtime as runtime
     sx = value.find("${")
@@ -97,16 +163,19 @@ def expandVars(value):
                     defaultValue = value[defsx+10:endx-1]
             if (defaultValue == None):
                 varname = value[sx+2:endx]
-            if (varname == "AG_SYSTEM"):
+            if (varname == AG_SYSTEM_VAR):
                 varval = runtime.appInfo.getSystemDir()
-            elif (varname == "AG_SYSTEM_STATIC"):
+            elif (varname == AG_SYSTEM_STATIC_VAR):
                 varval = runtime.appInfo.getSystemStaticDir()
-            elif (varname == "AG_APP"):
+            elif (varname == AG_APP_VAR):
                 varval = runtime.appInfo.getAppDir()
-            elif (varname == "AG_APP_STATIC"):
+            elif (varname == AG_APP_STATIC_VAR):
                 varval = runtime.appInfo.getAppStaticDir()
             else:
-                varval = os.getenv(varname)
+                if (includeEnv):
+                    varval = os.getenv(varname)
+                else:
+                    varval = None
             if ((varval == None) and (defaultValue != None)):
                 varval = defaultValue
             if (varval == None):
@@ -148,22 +217,21 @@ def convertSourcePath(path, to, otherdir=None):
         return otherdir + path[ix+7:]
 
 
-def visit(directory, files, extension):
+def visit(directory, files, extension, maxLevel=None, level=1):
     testdirs = os.listdir(directory)
     for thing in testdirs:
         fullpath = os.path.join(directory, thing)
-        if (os.path.isdir(fullpath)):
-            visit(fullpath, files, extension)
+        if (os.path.isdir(fullpath) and (maxLevel == None or level < maxLevel)):
+            visit(fullpath, files, extension, maxLevel, level+1)
         elif thing.endswith(extension):
             fullname = os.path.normpath(os.path.join(directory, thing))
             if not fullname in files:
                 files.append(fullname)
  
-def listFilesByExtensionInPath(path=[], extension='.lyt'):
-    #Collect input and output arguments into one bunch
+def listFilesByExtensionInPath(path=[], extension='.lyt', maxLevel=None):
     retval = []
     for directory in path:
-        visit(directory, retval, extension)
+        visit(directory, retval, extension, maxLevel)
     return retval
 
 def getFileLastModificationTime(fileName):
@@ -311,6 +379,21 @@ def remove(file):
         shutil.rmtree(file)
 endIfDef()
 
+def getUserTempDir():
+    systemTempDir = utillang.getSystemTempDir()
+    userName = sysutils.getUserName()
+    userNameNoSpace = userName.replace('_','__').replace(' ','_')
+    userTempDir = systemTempDir + os.sep + "activegrid_" + userNameNoSpace
+    return userTempDir
+
+def createUserTempDir():
+    userTempDir = getUserTempDir()
+    if not os.path.exists(userTempDir):
+        os.makedirs(userTempDir)
+        os.chmod(userTempDir, 0700)
+
+createUserTempDir()
+
 ifDefPy()
 import warnings
 warnings.filterwarnings("ignore", message="tmpnam is a potential security risk to your program")

wxPython/samples/ide/activegrid/util/objutils.py

@@ -16,7 +16,8 @@ import sys
 import os
 import __builtin__
 import types
-import xml.sax.saxutils as saxutils
+import activegrid.util.utillang as utillang
+import activegrid.util.datetimeparser as datetimeparser
 from types import *
 from activegrid.util.lang import *
 
@@ -65,6 +66,18 @@ def setStaticAttr(obj, attr, value):
         classDesc = obj.__class__
     setattr(classDesc, attr, value)
 
+def hasAttrFast(obj, name):
+    if hasRawAttr(obj, name):
+        return True
+    if hasattr(obj, '_complexType'):
+        complexType=obj._complexType
+        element=complexType.findElement(name)
+        if element:
+            return True
+    if hasattr(obj, name):
+        return True
+    return False
+
 def moduleForName(moduleName):
     module = None
     pathList = moduleName.split('.')
@@ -114,15 +127,22 @@ def newInstance(className, objargs=None):
         if ((len(objargs) < 1) or (objargs[0].lower() == "false") or (not objargs[0])):
             return False
         return True
-    if className == "str" or className == "unicode": # don"t strip: blanks are significant
+    if className == "str" or className == "unicode": # don't strip: blanks are significant
         if len(objargs) > 0:
             try:
-                return saxutils.unescape(objargs[0]).encode()
+                return utillang.unescape(objargs[0]).encode()
             except:
                 return "?"
         else:
             return ""
-
+            
+    if className == "date":
+        return datetimeparser.parse(objargs[0], asdate=True)
+    if className == "datetime":
+        return datetimeparser.parse(objargs[0])
+    if className == "time":
+        return datetimeparser.parse(objargs[0], astime=True)
+        
     classtype = classForName(className)
     if (classtype == None):
         raise Exception("Could not find class %s" % className)
@@ -135,23 +155,35 @@ def newInstance(className, objargs=None):
 def getClassProperty(classType, propertyName):
     return getattr(classType, propertyName)
     
-def toDiffableRepr(value, exclude=None):
+def toDiffableRepr(value, maxLevel=None):
     if (value == None):
         return "None"
+    if (maxLevel == None):
+        maxLevel = 8
+    maxLevel -= 1
+    if (maxLevel < 0):
+        return typeToString(value, PRINT_OBJ_DIFFABLE)
+##    if ((exclude != None) and not isinstance(value, (basestring, int))):
+##        for v in exclude:
+##            if (v is value):
+##                return "<recursive reference>"
+##        exclude.append(value)
 ##    elif (isinstance(value, ObjectType) and hasattr(value, "__dict__")):
 ##        if (exclude == None):
 ##            exclude = []
 ##        s = "%s(%s)" % (type(value), toDiffableString(value.__dict__, exclude))
-    elif (not isinstance(value, (BooleanType, ClassType, ComplexType, DictType, DictionaryType, 
+    if (not isinstance(value, (BooleanType, ClassType, ComplexType, DictType, DictionaryType, 
                                FloatType, IntType, ListType, LongType, StringType, TupleType, 
                                UnicodeType, BufferType, BuiltinFunctionType, BuiltinMethodType,
                                CodeType, FrameType, FunctionType, GeneratorType, InstanceType,
                                LambdaType, MethodType, ModuleType, SliceType, TracebackType,
                                TypeType, XRangeType))):
-        if (hasattr(value, "__str__")):
+        if (hasattr(value, "_toDiffableString")):
+            s = value._toDiffableString(maxLevel)
+        elif (hasattr(value, "__str__")):
             s = str(value)
         elif (hasattr(value, "__dict__")):
-            s = "%s(%s)" % (type(value), toDiffableString(value.__dict__, exclude))
+            s = "%s(%s)" % (type(value), toDiffableString(value.__dict__, maxLevel))
         else:
             s = str(type(value))
         ix2 = s.find(" object at 0x")
@@ -173,33 +205,31 @@ def toDiffableRepr(value, exclude=None):
                 else:
                     items.append("'%s'" % v)
             else:
-                items.append(toDiffableString(v, exclude))
+                items.append(toDiffableString(v, maxLevel))
         s = "[" + ", ".join(items) + "]"
     elif (isinstance(value, dict)):
-        if (exclude == None):
-            exclude = []
         items = []
         for key, val in value.iteritems():
             if (isinstance(val, UnicodeType)):
-                items.append("'%s': u'%s'" % (key, toDiffableString(val, exclude)))
+                items.append("'%s': u'%s'" % (key, toDiffableString(val, maxLevel)))
             elif (isinstance(val, basestring)):
-                items.append("'%s': '%s'" % (key, toDiffableString(val, exclude)))
+                items.append("'%s': '%s'" % (key, toDiffableString(val, maxLevel)))
             else:
-                items.append("'%s': %s" % (key, toDiffableString(val, exclude)))
+                items.append("'%s': %s" % (key, toDiffableString(val, maxLevel)))
         s = "{" + ", ".join(items) + "}"
     else:
         s = str(value)
     return s
     
-def toDiffableString(value, exclude=None):
-    if (value == None):
-        return "None"
-    if ((exclude != None) and not isinstance(value, (basestring, int))):
-        for v in exclude:
-            if (v is value):
-                return "<recursive reference>"
-        exclude.append(value)
-    s = toDiffableRepr(value)
+def toDiffableString(value, maxLevel=None):
+##    if (value == None):
+##        return "None"
+##    if ((exclude != None) and not isinstance(value, (basestring, int))):
+##        for v in exclude:
+##            if (v is value):
+##                return "<recursive reference>"
+##        exclude.append(value)
+    s = toDiffableRepr(value, maxLevel)
     ds = ""
     i = s.find(" at 0x") 
     start = 0

wxPython/samples/ide/activegrid/util/parser.py

@@ -0,0 +1,380 @@
+#----------------------------------------------------------------------------
+# Name:         parser.py
+# Purpose:      parsing utilities
+#
+# Author:       Jeff Norton
+#
+# Created:      8/9/05
+# CVS-ID:       $Id$
+# Copyright:    (c) 2004-2005 ActiveGrid, Inc.
+# License:      wxWindows License
+#----------------------------------------------------------------------------
+
+import re
+from activegrid.util.lang import *
+ifDefPy()
+import string
+import array
+endIfDef()
+
+XPATH_ROOT_VAR = '__rootObj__'
+GETOBJECTPARTNAMES  =   ["primaryRef", "ref", "orderings", "limit"]
+
+class Tokenizer(object):
+    
+    TOKEN_IDENT = 1
+    TOKEN_STRING = 2
+    TOKEN_OP = 3
+    TOKEN_WS = 4
+##    TOKEN_PLACEHOLDER = 5
+    
+    def __init__(self, text, identStart=None, tokenSep=None, ignoreWhitespace=True):
+        """
+Turn a string into individual tokens.  Three types of tokens are recognized:
+    TOKEN_IDENT:   identifiers (those that start with the identStart pattern)
+    TOKEN_STRING:  quoted string
+    TOKEN_OP:      everything else
+Tokens are separated by white space or the tokenSep pattern.  
+Constructor parameters:
+    text:  The string to tokenize
+    identStart:  A regular expression describing characters which start an identifier
+                 The default expression accepts letters, "_", and "/".
+    tokenSep:    A regular expression describing the characters which end a token 
+                 (in addition to whitespace).  The default expression accepts
+                 anything except alpha-numerics, "_", "/", and ":".
+Usage:
+    Invoke getNextToken (or next) to get the next token.  The instance variables
+    token, and tokenVal will be populated with the current token type (TOKEN_IDENT,
+    TOKEN_STRING, or TOEKN_OP) and value respectively.  nextToken and nextTokenVal
+    will also be available for lookahead.   The next method is similar to
+    getNextToken but also returns the token value.  A value of None signals end
+    of stream.                                           
+        """
+        self.ignoreWhitespace=ignoreWhitespace
+        ifDefPy()
+        if (isinstance(text, array.array)):
+            text = text.tostring()
+        endIfDef()
+        self.text = asString(text)
+        self.textIndex = 0
+        self.textLen = len(self.text)
+        self.token = None
+        self.tokenVal = None
+        self.nextToken = None
+        self.nextTokenVal = None
+        if (identStart == None):
+            identStart = "[a-zA-Z_/]"
+        if (tokenSep == None):
+            tokenSep = "[^a-zA-Z0-9_/:]"
+        self.identStart = re.compile(identStart)
+        self.tokenSep = re.compile(tokenSep)
+        self.getNextToken() # Prime the pump
+        
+    def isEscaped(text, index):
+        if ((index > 0) and (text[index-1] == '\\') and ((index < 2) or (text[index-2] != '\\'))):
+            return True
+        return False
+    isEscaped = staticmethod(isEscaped)
+
+    def findClosingQuote(text, index, char):
+        index = index + 1
+        while True:
+            endIndex = text.find(char, index)
+            if (endIndex < 1):
+                return -1
+            if (Tokenizer.isEscaped(text, endIndex)):
+                index = endIndex+1
+            else:
+                break
+        return endIndex + 1
+    findClosingQuote = staticmethod(findClosingQuote)
+
+    def _findClosing(self, char):
+        if (self.textIndex >= self.textLen):
+            raise Exception("The text \"%s\" has an unmatched string starting at %d" % (self.text, self.textIndex))
+        index = Tokenizer.findClosingQuote(self.text, self.textIndex, char)
+        if (index < 0):
+            raise Exception("The text \"%s\" has an unmatched string starting at %d" % (self.text, self.textIndex-1))
+        return index
+        
+    def next(self):
+        self.getNextToken()
+        if (self.token == None):
+            raise StopIteration()
+        return self.tokenVal
+
+    def getNextToken(self):
+        self.token = self.nextToken
+        self.tokenVal = self.nextTokenVal
+        while (self.textIndex < self.textLen):
+            c = self.text[self.textIndex]
+            if (c not in string.whitespace):
+                if (c == '"' or c == "'" or c == '`'):
+                    endIndex = self._findClosing(c)
+                    self.nextToken = self.TOKEN_STRING
+                    self.nextTokenVal = self.text[self.textIndex:endIndex]
+                    self.textIndex = endIndex
+                    return
+                elif (self.identStart.search(c)):
+                    endMatch = self.tokenSep.search(self.text, self.textIndex+1)
+                    if (endMatch):
+                        endIndex = endMatch.start()
+                    else:
+                        endIndex = self.textLen
+                    self.nextToken = self.TOKEN_IDENT
+                    self.nextTokenVal = self.text[self.textIndex:endIndex]
+                    self.textIndex = endIndex
+                    return
+                else:
+                    self.nextToken = self.TOKEN_OP
+                    endIndex = self.textIndex + 1
+                    if (c == '<' or c == '>' or c == '!' or c == '='):
+                        if ((endIndex < self.textLen) and (self.text[endIndex] == '=')):
+                            endIndex += 1
+                    elif ((c == '%') and (endIndex < self.textLen)):
+                        c = self.text[endIndex]
+                        if (c in ['d', 'i', 'o', 'u', 'x', 'X', 'e', 'E', 'f', 'F', 'g', 'G', 'c', 'r', 's', '%']):
+                            endIndex += 1
+##                            self.nextToken = self.TOKEN_PLACEHOLDER # Should really be this but no one can handle it yet
+                    self.nextTokenVal = self.text[self.textIndex:endIndex]
+                    self.textIndex = endIndex
+                    return
+            elif not self.ignoreWhitespace:
+                self.nextToken=self.TOKEN_WS
+                self.nextTokenVal=""
+                while c in string.whitespace:
+                    self.nextTokenVal+=c
+                    self.textIndex+=1
+                    if self.textIndex==len(self.text):
+                        break
+                    c=self.text[self.textIndex]
+                return
+            self.textIndex += 1
+        self.nextToken = None
+        self.nextTokenVal = None
+
+def isXPathNonVar(var):
+    """Returns true iff var is a string ("foo" or 'foo') or a number."""
+    if (var.startswith("'") and var.endswith("'")) or \
+            (var.startswith('"') and var.endswith('"')):
+        return True
+
+    # list from XPathToCode, below
+    if var.lower() in ["count", "empty", "true", "false", "null", "and", "or", \
+            "like", "not"]:
+        return True
+
+    try:
+        t=int(var)
+        return True
+    except TypeError, e:
+        pass
+    except ValueError, e:
+        pass
+
+    return False
+
+def xpathToCode(xpaths, convertBracket=True):
+    if ((xpaths == None) or (len(xpaths) < 1)):
+        return "True"
+    if (not isinstance(xpaths, (list, tuple))):
+        xpaths = [xpaths]
+    result = []
+    for xpath in xpaths:
+        t = Tokenizer(xpath, "[a-zA-Z0-9_/:\.]", "[^a-zA-Z0-9_/:\.]", ignoreWhitespace=False)
+        expr = []
+        lastToken=None
+        while t.nextToken != None:
+            t.getNextToken()
+            if (t.token == Tokenizer.TOKEN_WS):
+                expr.append(" ")
+            elif (t.token == Tokenizer.TOKEN_OP):
+                if (t.tokenVal == "="):
+                    expr.append("==")
+                elif (t.tokenVal == "[" and convertBracket):
+                    expr.append("(")
+                elif (t.tokenVal == "]" and convertBracket):
+                    expr.append(")")
+                else:
+                    expr.append(t.tokenVal)
+            elif (t.token == Tokenizer.TOKEN_IDENT):
+                if (t.tokenVal == "and"):
+                    expr.append(" and ")
+                elif (t.tokenVal == "or"):
+                    expr.append(" or ")
+                elif (t.tokenVal == "not"):
+                    expr.append(" not ")
+                elif (t.tokenVal == "like"):
+                    # REVIEW stoens@activegrid.com 02-Nov-05 --
+                    # This is very limited support for like:
+                    # typically like queries look like this: "foo like 'blah%'".
+                    # So translate this into "foo.startswith(blah)".
+                    # We should use a regular expression to support '%'s in
+                    # arbitrary places in the string. After 1.1.
+                    if t.nextToken and t.nextTokenVal.endswith("%'"):
+                        t.getNextToken() # throw away the "like" token
+                        last = len(expr) - 1
+                        expr[last] = "%s.startswith(%s')"\
+                            % (expr[last], t.tokenVal[:-2])
+                    else:
+                        # old behavior
+                        expr.append(t.tokenVal)
+                    
+                elif (t.tokenVal == "count"):
+                    expr.append("len")
+                elif (t.tokenVal == 'empty'):
+                    expr.append('ctx.isEmptyPath')
+                elif (t.tokenVal == 'true'):
+                    expr.append(_parseConstantFunction(t, 'True'))
+                elif (t.tokenVal == 'false'):
+                    expr.append(_parseConstantFunction(t, 'False'))
+                elif (t.tokenVal == 'null'):
+                    expr.append(_parseConstantFunction(t, 'None'))
+                elif (-1!=t.tokenVal.find(':')):
+                    serviceDef, args=_parseServiceFunction(t)
+
+                    # XXX handle serviceDef, args being None
+
+                    for i in range(len(args)):
+                        args[i]=xpathToCode(args[i], False)
+                    jargs="[%s]" % (",".join(args))
+
+                    # XXX should be processmodel.DATASERVICE_PROCESS_NAME, not "dataservice"
+                    if serviceDef[0]=='dataservice':
+                        expr.append("runtimesupport.invokeDataServiceWrapper(%s, %s, ctx, locals())" % \
+                                (serviceDef, jargs))
+                    else:
+                        expr.append("runtimesupport.invokeServiceWrapper(%s, %s, ctx)" % \
+                                (serviceDef, jargs))
+                else:
+                    if (lastToken==')' or lastToken==']'):
+                        wasFunc=True
+                    else:
+                        wasFunc=False
+                    if (t.tokenVal.startswith('/')) and not wasFunc:
+                        expr.append(XPATH_ROOT_VAR)
+                    expr.append(t.tokenVal.replace('/','.'))
+                lastToken=t.tokenVal
+            else: 
+                expr.append(t.tokenVal)
+                
+
+        if (len(expr) == 2 and expr[0]==" "):
+            expr = "".join(expr)
+            result.append(expr)
+        elif (len(expr) > 1):
+            expr = "".join(expr)
+            result.append("(%s)" % expr)
+        elif (len(expr) > 0):
+            result.append(expr[0])
+        
+    return " and ".join(result)
+
+def _parseArgs(t):
+    args=[]
+    argcon=""
+
+    if t.tokenVal!='(':
+        return []
+    if t.nextTokenVal==')':
+        t.getNextToken()
+        return []
+
+    depth=1
+
+    while(depth!=0):
+        if not t.nextToken:
+            raise Exception("parameters list with no closing ) after token: %s" % t.tokenVal)
+        t.getNextToken()
+
+        if t.tokenVal=='(':
+            depth+=1
+        if t.tokenVal==')':
+            depth-=1
+
+        if depth==0 or (depth==1 and t.tokenVal==','):
+            args.append(argcon)
+            argcon=""
+        else:
+            argcon+=t.tokenVal
+    return args
+
+def _parseServiceFunction(t):
+    """Parses what appears to be a service function call into serviceDefs and args lists.
+
+    Returns None, None if the serviceFunction appears to be invalid.
+    """
+    if t.nextTokenVal!='(':
+        return t.tokenVal, None
+
+    serviceDef=t.tokenVal.split(':')
+    t.getNextToken()
+    args=_parseArgs(t)
+
+    return serviceDef, args
+    
+def _parseConstantFunction(t, outputValue):
+    firstVal = t.tokenVal
+    if t.nextTokenVal != '(':
+        return firstVal
+    t.getNextToken()
+    if t.nextTokenVal != ')':
+        return "%s%s" % (firstVal, '(')
+    t.getNextToken()
+    return outputValue
+
+def parseDSPredicate(ctx, str, vars, valueList=None):
+    from activegrid.util.utillang import evalCode
+    from activegrid.util.utillang import ObjAsDict
+
+    if valueList == None:
+        valueList = []
+    indexVar=0
+    oldIndexVar=0
+    sourceStr=str
+    inlinedPredicate=[]
+    qualifications=[]
+    while True:
+        oldIndexVar = indexVar
+        dollarCurlForm = False
+        quoted = False
+        indexVar = sourceStr.find("bpws:getVariableData", indexVar)
+        if indexVar == -1:
+            indexVar = sourceStr.find("${", oldIndexVar)
+            if indexVar == -1:
+                break
+            dollarCurlForm = True
+        if indexVar > 0 and sourceStr[indexVar-1] in ('"',"'"):
+            quoted = True
+        if not dollarCurlForm:
+            openParen = sourceStr.find("(", indexVar)
+            if openParen == -1:
+                break
+            closeParen = sourceStr.find(")", openParen)
+            if closeParen == -1:
+                break
+        else:
+            openParen = indexVar+1
+            closeParen = sourceStr.find("}", openParen)
+            if closeParen == -1:
+                break
+        varRef = sourceStr[openParen+1: closeParen]
+        if varRef.startswith('"') or varRef.startswith("'"):
+            varRef = varRef[1:]
+        if varRef.endswith('"') or varRef.endswith("'"):
+            varRef = varRef[:-1]
+        if isinstance(vars, dict) or isinstance(vars, ObjAsDict):
+            varRefCode = xpathToCode(varRef)
+            value = evalCode(varRefCode, vars)
+        else:
+            value = ctx.evalPath(vars, varRef)
+        inlinedPredicate.append(sourceStr[oldIndexVar:indexVar])
+        if quoted:
+            inlinedPredicate.append("%s" % value)
+        else:
+            inlinedPredicate.append('%s')
+            valueList.append(value)
+        indexVar = closeParen+1
+    inlinedPredicate.append(sourceStr[oldIndexVar:])
+    qualifications.append(''.join(inlinedPredicate))
+    return qualifications, valueList

wxPython/samples/ide/activegrid/util/strutils.py

@@ -21,3 +21,91 @@ def caseInsensitiveCompare(s1, s2):
         return -1
     else:
         return 1
+
+def multiSplit(stringList, tokenList=[" "]):
+    """Splits strings in stringList by tokens, returns list of string."""
+    if not stringList: return []
+    if isinstance(tokenList, basestring):
+        tokenList = [tokenList]
+    if isinstance(stringList, basestring):
+        stringList = [stringList]
+    rtnList = stringList
+    for token in tokenList:
+        rtnList = rtnList[:]
+        for string in rtnList:
+            if string.find(token) > -1:
+                rtnList.remove(string)
+                names = string.split(token)
+                for name in names:
+                    name = name.strip()
+                    if name:
+                        rtnList.append(name)
+    return rtnList
+
+QUOTES = ("\"", "'")
+
+def _findArgStart(argStr):
+    i = -1
+    for c in argStr:
+        i += 1
+        if (c == " "):
+            continue
+        elif (c == ","):
+            continue
+        return i
+    return None
+
+def _findArgEnd(argStr):
+    quotedArg = True
+    argEndChar = argStr[0]
+    if (not argEndChar in QUOTES):
+        argEndChar = ","
+        quotedArg = False
+    i = -1
+    firstChar = True
+    for c in argStr:
+        i+= 1
+        if (firstChar):
+            firstChar = False
+            if (quotedArg):
+                continue
+        if (c == argEndChar):
+            if (quotedArg):
+                return min(i+1, len(argStr))
+            else:
+                return i
+    return i
+
+def parseArgs(argStr, stripQuotes=False):
+    """
+    Given a str representation of method arguments, returns list arguments (as
+    strings).
+    
+    Input: "('[a,b]', 'c', 1)" -> Output: ["'[a,b]'", "'c'", "1"].
+
+    If stripQuotes, removes quotes from quoted arg.
+    """
+    if (argStr.startswith("(")):
+        argStr = argStr[1:]
+        if (argStr.endswith(")")):
+            argStr = argStr[:-1]
+        else:
+            raise AssertionError("Expected argStr to end with ')'")
+
+    rtn = []
+    argsStr = argStr.strip()
+    while (True):
+        startIndex = _findArgStart(argStr)
+        if (startIndex == None):
+            break
+        argStr = argStr[startIndex:]
+        endIndex = _findArgEnd(argStr)
+        if (endIndex == len(argStr) - 1):
+            rtn.append(argStr.strip())
+            break        
+        t = argStr[:endIndex].strip()
+        if (stripQuotes and t[0] in QUOTES and t[-1] in QUOTES):
+            t = t[1:-1]
+        rtn.append(t)
+        argStr = argStr[endIndex:]
+    return rtn

wxPython/samples/ide/activegrid/util/sysutils.py

@@ -12,6 +12,7 @@
 
 import sys
 import os
+import time
 
 # this will be set to true in IDE.py when we are running release builds.
 isRelease = False
@@ -26,6 +27,12 @@ isRelease = False
 
 MAINMODULE_DIR = "AG_MAINMODULE_DIR"
 IS_RELEASE = "AG_IS_RELEASE"
+IS_COMMERCIAL = "AG_IS_COMMERCIAL"
+AG_SYSTEM_START_TIME_ENV_NAME = "AG_SYSTEM_START_TIME"
+
+def isCommercial():
+    
+    return os.path.exists(os.path.join(mainModuleDir,"commercial.txt")) or 'true' == (str(os.getenv(IS_COMMERCIAL)).lower())
  
 def isRelease():
     return 'true' == (str(os.getenv(IS_RELEASE)).lower())
@@ -39,7 +46,16 @@ def setRelease(value):
 def isWindows():
     return os.name == 'nt'
 
+__isServer = False
 
+def setServerMode(isServer):
+    global __isServer
+    __isServer = isServer
+    
+def isServer():
+    global __isServer
+    return __isServer
+    
 def _generateMainModuleDir():
     mainModuleDir = os.getenv(MAINMODULE_DIR)
     if mainModuleDir:  # if environment variable set, return it
@@ -85,3 +101,16 @@ def getCommandNameForExecPath(execPath):
         return '"%s"' % execPath
     return execPath
 
+def getUserName():
+    if isWindows():
+        return os.getenv('USERNAME')
+    else:
+        # 06-Feb-06 stoens@activegrid.com --
+        # this blows up the linux cc runs with "Inappropriate ioctl for device"
+        #return os.getlogin()
+        return os.getenv('USER')        
+
+def getCurrentTimeAsFloat():
+    return time.time()
+
+systemStartTime = getCurrentTimeAsFloat()

wxPython/samples/ide/activegrid/util/utillang.py

@@ -0,0 +1,146 @@
+#----------------------------------------------------------------------------
+# Name:         utillang.py
+# Purpose:      Provide language specific utilities 
+#
+# Author:       Joel Hare
+#
+# Created:      8/23/05
+# CVS-ID:       $Id$
+# Copyright:    (c) 2004-2005 ActiveGrid, Inc.
+# License:      wxWindows License
+#----------------------------------------------------------------------------
+
+import os
+import sys
+import UserDict
+import tempfile
+import xml.sax.saxutils as saxutils
+
+import activegrid.util.parser as parser
+
+PY2WEB_codepages = {
+     'cp1251' : 'CP-1251',
+     'koi8_r' : 'KOI8-R',
+}
+
+def evalXPath(xpath, data, specialEntries=None):
+    codeStr = parser.xpathToCode(xpath)
+    return evalCode(codeStr, data, specialEntries)
+
+def evalCode(codeStr, data, specialEntries=None):
+    if isinstance(data, ObjAsDict):
+        namespace = data
+    elif isinstance(data, dict):
+        namespace = dict(data)
+    else:
+        namespace = ObjAsDict(data)
+    if specialEntries:
+        for key, value in specialEntries.items():
+            namespace.addSpecialEntry(key, value)
+    return eval(codeStr, {}, namespace)
+
+def deriveCharset():
+    charset = None
+    encodingString = sys.getdefaultencoding()
+    if encodingString != 'ascii':
+        charset = PY2WEB_codepages.get(encodingString.lower())
+        if charset == None:
+            charset = encodingString
+    return charset
+
+def toUTF8(value):
+    """
+    Converts all unicode and non-string values to utf-8.
+    This assumes string instances are already encoded in utf-8.
+    Note that us-ascii is a subset of utf-8.
+    """
+    if isinstance(value, unicode):
+        return value.encode('utf-8')
+    return str(value)
+
+def toUnicode(value):
+    """
+    Converts all strings non-string values to unicode.
+    This assumes string instances are encoded in utf-8.
+    Note that us-ascii is a subset of utf-8.
+    """
+    if not isinstance(value, unicode):
+        if not isinstance(value, str):
+            return unicode(value)
+        return unicode(value, 'utf-8')
+    return value
+    
+
+def getSystemTempDir():
+    return tempfile.gettempdir()
+
+def getEnvVar(name, defaultVal=None):
+    if os.environ.has_key(name):
+        return os.environ[name]
+    return defaultVal
+
+class ObjAsDict(UserDict.DictMixin):
+    """
+    Passing this to eval as the local variables dictionary allows the
+    evaluated code to access properties in the wrapped object
+    """
+    def __init__(self, obj):
+        self.obj = obj
+        self.specialEntries = {}
+
+    def __getitem__(self, key):
+        try:
+            return getattr(self.obj, key)
+        except AttributeError, e:
+            if self.specialEntries.has_key(key):
+                return self.specialEntries[key]
+            raise KeyError(e.args)
+    def __setitem__(self, key, item): setattr(self.obj, key, item)
+    def __delitem__(self, key): delattr(self.obj, key)
+    def keys(self):
+        ret=[]
+        for i in list(dir(self.obj)+self.specialEntries.keys()):
+            if i=="__doc__" or i=="__module__":
+                pass
+            elif i not in ret:
+                ret.append(i)
+        return ret
+
+    def addSpecialEntry(self, key, value):
+        self.specialEntries[key] = value
+
+global saxXMLescapeDoubleQuote
+saxXMLescapeDoubleQuote = {'"':'"'}
+
+global saxXMLescapesAllQuotes
+# IE doesn't support ' but it doesn't seem like we should need this escaped at all so I took it out.
+saxXMLescapesAllQuotes = {'"':'"', "'":"'"}
+
+global saxXMLunescapes
+saxXMLunescapes = {'"':'"', "'":"'"}
+
+def escape(data, extraEscapes=None):
+    """Escape ', ", &, <, and > in a string of data.
+
+    Basically, everything that saxutils.escape does (and this calls that, at
+    least for now), but with " and ' added as well.
+
+    TODO: make this faster; saxutils.escape() is really slow
+    """
+    
+    global saxXMLescapeDoubleQuote
+    if (extraEscapes == None):
+        extraEscapes = saxXMLescapeDoubleQuote
+    return saxutils.escape(data, extraEscapes)
+
+def unescape(data):
+    """Unescape ', ", &, <, and > in a string of data.
+
+    Basically, everything that saxutils.unescape does (and this calls that, at
+    least for now), but with " and ' added as well.
+
+    TODO: make this faster; saxutils.unescape() is really slow
+    """
+    
+    global saxXMLunescapes
+    return saxutils.unescape(data, saxXMLunescapes)

wxPython/samples/ide/activegrid/util/xmlmarshaller.py

@@ -2,7 +2,7 @@
 # Name:         xmlmarshaller.py
 # Purpose:
 #
-# Authors:       John Spurling, Joel Hare, Alan Mullendore
+# Authors:      John Spurling, Joel Hare, Jeff Norton, Alan Mullendore
 #
 # Created:      7/28/04
 # CVS-ID:       $Id$
@@ -16,14 +16,18 @@ import logging
 ifDefPy()
 import xml.sax
 import xml.sax.handler
+import xml.sax.saxutils
+import datetime
 endIfDef()
-import xml.sax.saxutils as saxutils
+import activegrid.util.utillang as utillang
 import activegrid.util.objutils as objutils
+import activegrid.util.sysutils as sysutils
 import activegrid.util.aglogging as aglogging
 
 MODULE_PATH = "__main__"
 
 ## ToDO remove maxOccurs "unbounded" resolves to -1 hacks after bug 177 is fixed
+##unboundedVal = 2147483647 # value used for maxOccurs == "unbounded"
 
 """
 Special attributes that we recognize:
@@ -109,7 +113,6 @@ __xmlcdatacontent__ = "messyContent"
 
 global xmlMarshallerLogger
 xmlMarshallerLogger = logging.getLogger("activegrid.util.xmlmarshaller.marshal")
-xmlMarshallerLogger.setLevel(aglogging.LEVEL_WARN)
 # INFO  : low-level info
 # DEBUG : debugging info
 
@@ -184,6 +187,7 @@ DICT_ITEM_VALUE_NAME = "value"
 ################################################################################
 
 def setattrignorecase(object, name, value):
+##    print "[setattrignorecase] name = %s, value = %s" % (name, value)
     if (name not in object.__dict__):
         namelow = name.lower()
         for attr in object.__dict__:
@@ -193,27 +197,95 @@ def setattrignorecase(object, name, value):
     object.__dict__[name] = value
 
 def getComplexType(obj):
+    if (hasattr(obj, "_instancexsdcomplextype")):
+        return obj._instancexsdcomplextype
     if (hasattr(obj, "__xsdcomplextype__")):
         return obj.__xsdcomplextype__
     return None
 
-def _objectfactory(objname, objargs=None, objclass=None):
-    "dynamically create an object based on the objname and return it."
-##    print "[objectfactory] objname [%s]" % (objname)
+def _objectfactory(objtype, objargs=None, objclass=None):
+    "dynamically create an object based on the objtype and return it."
     if not isinstance(objargs, list):
         objargs = [objargs]
     if (objclass != None):
+        obj = None
         if (len(objargs) > 0):
             if (hasattr(objclass, "__xmlcdatacontent__")):
                 obj = objclass()
                 contentAttr = obj.__xmlcdatacontent__
                 obj.__dict__[contentAttr] = str(objargs[0])
-                return obj
-            return objclass(*objargs)
+            else:
+                obj = objclass(*objargs)
         else:
-            return objclass()
-    return objutils.newInstance(objname, objargs)
+            obj = objclass()
+        if ((obj != None) and (hasattr(obj, 'postUnmarshal'))):
+            obj.postUnmarshal()
+        return obj
+    return objutils.newInstance(objtype, objargs)
 
+class GenericXMLObject(object):
+    def __init__(self, content=None):
+        if content != None:
+            self._content = content
+            self.__xmlcontent__ = '_content'
+
+    def __str__(self):
+        return "GenericXMLObject(%s)" % objutils.toDiffableString(self.__dict__)
+
+    def setXMLAttributes(self, xmlName, attrs=None, children=None, nsMap=None, defaultNS=None):
+        if xmlName != None:
+            i = xmlName.rfind(':')
+            if i < 0:
+                self.__xmlname__ = xmlName
+                if defaultNS != None:
+                    self.__xmldefaultnamespace__ = str(defaultNS)
+            else:
+                self.__xmlname__ = xmlName[i+1:]
+                prefix = xmlName[:i]
+                if nsMap.has_key(prefix):
+                    self.__xmldefaultnamespace__ = str(nsMap[prefix])
+        if attrs != None:
+            for attrname, attr in attrs.items():
+                attrname = str(attrname)
+                if attrname == XMLNS or attrname.startswith(XMLNS_PREFIX):
+                    pass
+                elif attrname == "objtype":
+                    pass
+                else:
+                    if not hasattr(self, '__xmlattributes__'):
+                        self.__xmlattributes__ = []
+                    i = attrname.rfind(':')
+                    if i >= 0:
+                        prefix = attrname[:i]
+                        attrname = attrname[i+1:]
+                        if not hasattr(self, '__xmlattrnamespaces__'):
+                            self.__xmlattrnamespaces__ = {}
+                        if self.__xmlattrnamespaces__.has_key(prefix):
+                            alist = self.__xmlattrnamespaces__[prefix]
+                        else:
+                            alist = []
+                        alist.append(attrname)
+                        self.__xmlattrnamespaces__[prefix] = alist
+                    self.__xmlattributes__.append(attrname)
+            if hasattr(self, '__xmlattributes__'):
+                self.__xmlattributes__.sort()
+        if children != None and len(children) > 0:
+            childList = []
+            flattenList = {}
+            for childname, child in children:
+                childstr = str(childname)
+                if childstr in childList:
+                    if not flattenList.has_key(childstr):
+                        flattenList[childstr] = (childstr,)
+                else:
+                    childList.append(childstr)
+            if len(flattenList) > 0:
+                self.__xmlflattensequence__ = flattenList
+                
+    def initialize(self, arg1=None):
+        pass
+            
+    
 class Element:
     def __init__(self, name, attrs=None, xsname=None):
         self.name = name
@@ -222,9 +294,11 @@ class Element:
         self.children = []
         self.objclass = None
         self.xsname = xsname
+        self.objtype = None
         
     def getobjtype(self):
-        objtype = self.attrs.get("objtype")
+#        objtype = self.attrs.get("objtype")
+        objtype = self.objtype
         if (objtype == None):
             if (len(self.children) > 0):
                 objtype = "dict"
@@ -238,16 +312,35 @@ class NsElement(object):
         self.targetNS = None
         self.defaultNS = None
         self.prefix = None
-        
-    def isEmpty(self):
-        return ((self.nsMap == {}) and (self.targetNS == None) and (self.defaultNS == None))
 
+    def __str__(self):
+        if self.prefix == None:
+            strVal = 'prefix = None; '
+        else:
+            strVal = 'prefix = "%s"; ' % (self.prefix)
+        if self.targetNS == None:
+            strVal += 'targetNS = None; '
+        else:
+            strVal += 'targetNS = "%s"; ' % (self.targetNS)
+        if self.defaultNS == None:
+            strVal += 'defaultNS = None; '
+        else:
+            strVal += 'defaultNS = "%s"; ' % (self.defaultNS)
+        if len(self.nsMap) == 0:
+            strVal += 'nsMap = None; '
+        else:
+            strVal += 'nsMap = {'
+            for ik, iv in self.nsMap.iteritems():
+                strVal += '%s=%s; ' % (ik,iv)
+            strVal += '}'
+        return strVal
+               
     def setKnownTypes(self, masterKnownTypes, masterKnownNamespaces, parentNSE):
         # if we're a nested element, extend our parent element's mapping
         if parentNSE != None:
             self.knownTypes = parentNSE.knownTypes.copy()
             # but if we have a different default namespace, replace the parent's default mappings
-            if parentNSE.defaultNS != self.defaultNS:
+            if (self.defaultNS != None) and (parentNSE.defaultNS != self.defaultNS):
                 newKT = self.knownTypes.copy()
                 for tag in newKT:
                     if tag.find(':') < 0:
@@ -283,7 +376,6 @@ class NsElement(object):
                     self.knownTypes[knownTagName] = mapClass
             else:                                           # e.g. "ItemSearchRequest"
                 self.knownTypes[tag] = mapClass
-##                print 'mapping <%s> to class "%s"' % (tag, mapClass.__name__)
 
     def expandQName(self, eName, attrName, attrValue):
         bigValue = attrValue
@@ -298,38 +390,57 @@ class NsElement(object):
                 if shortNs == attrNS:
                     bigValue = '%s:%s' % (longNs, attrNCName)
                     break
-##        print '[expandQName] input attrName = "%s" and attrValue "%s"; output = "%s"' % (attrName, attrValue, bigValue)
         return bigValue
 
 class XMLObjectFactory(xml.sax.ContentHandler):
-    def __init__(self, knownTypes=None, knownNamespaces=None):
+    def __init__(self, knownTypes=None, knownNamespaces=None, xmlSource=None, createGenerics=False):
         self.rootelement = None
-        if (knownTypes == None):
-            self.knownTypes = {}
+        if xmlSource == None:
+            self.xmlSource = "unknown"
         else:
-            self.knownTypes = knownTypes
-        if (knownNamespaces == None):
-            self.knownNamespaces = {}
-        else:
-            self.knownNamespaces = knownNamespaces
+            self.xmlSource = xmlSource
+        self.createGenerics = createGenerics
         self.skipper = False
         self.elementstack = []
         self.nsstack = []
         self.collectContent = None
+        if (knownNamespaces == None):
+            self.knownNamespaces = {}
+        else:
+            self.knownNamespaces = knownNamespaces
+        self.reversedNamespaces = {}
+        for longns, shortns in self.knownNamespaces.iteritems():
+            self.reversedNamespaces[shortns] = longns
+        self.knownTypes = {}
+        if (knownTypes != None):
+            for tag, cls in knownTypes.iteritems():
+                i = tag.rfind(':')
+                if i >= 0:
+                    shortns = tag[:i]
+                    tag = tag[i+1:]
+                    if not self.reversedNamespaces.has_key(shortns):
+                        errorString = 'Error unmarshalling XML document from source "%s": knownTypes specifies an unmapped short namespace "%s" for element "%s"' % (self.xmlSource, shortns, tag)
+                        raise UnmarshallerException(errorString)
+                    longns = self.reversedNamespaces[shortns]
+                    tag = '%s:%s' % (longns, tag)
+                self.knownTypes[tag] = cls
+        #printKnownTypes(self.knownTypes, 'Unmarshaller.XMLObjectFactory.__init__')
         xml.sax.handler.ContentHandler.__init__(self)
 
     def appendElementStack(self, newElement, newNS):
         self.elementstack.append(newElement)
-        if (newNS.isEmpty()):
-            if (len(self.nsstack) > 0):
-                newNS = self.nsstack[-1]
-            else:
-                newNS.knownTypes = self.knownTypes.copy()
-        else:
-            if (len(self.nsstack) > 0):
-                newNS.setKnownTypes(self.knownTypes, self.knownNamespaces, self.nsstack[-1])
-            else:
-                newNS.setKnownTypes(self.knownTypes, self.knownNamespaces, None)
+        if (len(self.nsstack) > 0):
+            oldNS = self.nsstack[-1]
+            if newNS.defaultNS == None:
+                newNS.defaultNS = oldNS.defaultNS
+            if newNS.targetNS == None:
+                newNS.targetNS = oldNS.targetNS
+            if len(newNS.nsMap) == 0:
+                newNS.nsMap = oldNS.nsMap
+            elif len(oldNS.nsMap) > 0:
+                map = oldNS.nsMap.copy()
+                map.update(newNS.nsMap)
+                newNS.nsMap = map
         self.nsstack.append(newNS)
         return newNS
     
@@ -353,11 +464,16 @@ class XMLObjectFactory(xml.sax.ContentHandler):
             strVal += '>'
             self.collectContent.content += strVal
         xsname = name
-        if name.find(':') > -1:  # Strip namespace prefixes for now until actually looking them up in xsd
-            name = name[name.rfind(":") + 1:]
+        i = name.rfind(':')
+        if i >= 0:
+            nsname = name[:i]
+            name = name[i+1:]
+        else:
+            nsname = None
         element = Element(name, attrs.copy(), xsname=xsname)
         # if the element has namespace attributes, process them and add them to our stack
         nse = NsElement()
+        objtype = None
         for k in attrs.getNames():
             if k.startswith('xmlns'):
                 longNs = attrs[k]
@@ -371,8 +487,28 @@ class XMLObjectFactory(xml.sax.ContentHandler):
                     nse.nsMap[shortNs] = longNs
             elif k == 'targetNamespace':
                 nse.targetNS = attrs.getValue(k)
+            elif k == 'objtype':
+                objtype = attrs.getValue(k)
         nse = self.appendElementStack(element, nse)
-        element.objclass = nse.knownTypes.get(xsname)
+        if nsname != None:
+            if nse.nsMap.has_key(nsname):
+                longname = '%s:%s' % (nse.nsMap[nsname], name)
+##            elif objtype == None:
+##                errorString = 'Error unmarshalling XML document from source "%s": tag "%s" at line "%d", column "%d" has an undefined namespace' % (self.xmlSource, xsname, self._locator.getLineNumber(), self._locator.getColumnNumber())
+##                raise UnmarshallerException(errorString)
+            elif self.reversedNamespaces.has_key(nsname):
+                longname = '%s:%s' % (self.reversedNamespaces[nsname], name)
+            else:
+                longname = xsname
+        elif nse.defaultNS != None:
+            longname = '%s:%s' % (nse.defaultNS, name)
+        else:
+            longname = name
+        element.objtype = objtype
+        element.objclass = self.knownTypes.get(longname)
+        if element.objclass == None and len(self.knownNamespaces) == 0:
+            # handles common case where tags are unqualified and knownTypes are too, but there's a defaultNS
+            element.objclass = self.knownTypes.get(name)
         if (hasattr(element.objclass, "__xmlcontent__")):
             self.collectContent = element
 
@@ -387,8 +523,9 @@ class XMLObjectFactory(xml.sax.ContentHandler):
     def endElement(self, name):
 ##        print "[endElement] </%s>" % name
         xsname = name
-        if name.find(":") > -1:  # Strip namespace prefixes for now until actually looking them up in xsd
-            name = name[name.find(":") + 1:]
+        i = name.rfind(':')
+        if i >= 0:  # Strip namespace prefixes for now until actually looking them up in xsd
+            name = name[i+1:]
         if self.skipper:
             if xsname == "xs:annotation" or xsname == "xsd:annotation":     # here too
                 self.skipper = False
@@ -405,34 +542,36 @@ class XMLObjectFactory(xml.sax.ContentHandler):
         element, nse = self.popElementStack()
         if ((len(self.elementstack) > 1) and (self.elementstack[-1].getobjtype() == "None")):
             parentElement = self.elementstack[-2]
-##            print "[endElement] %s: found parent with objtype==None: using its grandparent" % name
         elif (len(self.elementstack) > 0):
             parentElement = self.elementstack[-1]
         objtype = element.getobjtype()
-##        print "element objtype is: ", objtype
         if (objtype == "None"):
-##            print "[endElement] %s: skipping a (objtype==None) end tag" % name
             return
         constructorarglist = []
         if (len(element.content) > 0):
             strippedElementContent = element.content.strip()
             if (len(strippedElementContent) > 0):
                 constructorarglist.append(element.content)
+        # If the element requires an object, but none is known, use the GenericXMLObject class
+        if ((element.objclass == None) and (element.attrs.get("objtype") == None) and ((len(element.attrs) > 0) or (len(element.children) > 0))):
+            if self.createGenerics:
+                element.objclass = GenericXMLObject
         obj = _objectfactory(objtype, constructorarglist, element.objclass)
+        if element.objclass == GenericXMLObject:
+            obj.setXMLAttributes(str(xsname), element.attrs, element.children, nse.nsMap, nse.defaultNS)
         complexType = getComplexType(obj)
         if (obj != None):
             if (hasattr(obj, "__xmlname__") and getattr(obj, "__xmlname__") == "sequence"):
                 self.elementstack[-1].children = oldChildren
                 return
         if (len(element.attrs) > 0) and not isinstance(obj, list):
-##            print "[endElement] %s: element has attrs and the obj is not a list" % name
             for attrname, attr in element.attrs.items():
                 if attrname == XMLNS or attrname.startswith(XMLNS_PREFIX):
                     if attrname.startswith(XMLNS_PREFIX):
                         ns = attrname[XMLNS_PREFIX_LENGTH:]
                     else:
                         ns = ""
-                    if complexType != None:
+                    if complexType != None or element.objclass == GenericXMLObject:
                         if not hasattr(obj, "__xmlnamespaces__"):
                             obj.__xmlnamespaces__ = {ns:attr}
                         elif ns not in obj.__xmlnamespaces__:
@@ -447,7 +586,6 @@ class XMLObjectFactory(xml.sax.ContentHandler):
                         xsdElement = complexType.findElement(attrname)
                         if (xsdElement != None):
                             type = xsdElement.type
-##                            print 'Unmarshalling element "%s", attribute "%s" with type "%s"' % (name, xsdElement.name, type)
                             if (type != None):
                                 if (type == TYPE_QNAME):
                                     attr = nse.expandQName(name, attrname, attr)
@@ -455,11 +593,15 @@ class XMLObjectFactory(xml.sax.ContentHandler):
                                 ### ToDO remove maxOccurs hack after bug 177 is fixed
                                 if attrname == "maxOccurs" and attr == "unbounded":
                                     attr = "-1"
-                                attr = _objectfactory(type, attr)
+                                try:
+                                    attr = _objectfactory(type, attr)
+                                except Exception, exceptData:
+                                    errorString = 'Error unmarshalling attribute "%s" at line %d, column %d in XML document from source "%s": %s' % (attrname, self._locator.getLineNumber(), self._locator.getColumnNumber(), self.xmlSource, str(exceptData))
+                                    raise UnmarshallerException(errorString)
                     try:
                         setattrignorecase(obj, _toAttrName(obj, attrname), attr)
                     except AttributeError:
-                        errorString = 'Error unmarshalling XML document at line %i, column %i: The object type of attribute "%s" of XML element "%s": not specified or known' % (self._locator.getLineNumber(), self._locator.getColumnNumber(), attrname, name)
+                        errorString = 'Error setting value of attribute "%s" at line %d, column %d in XML document from source "%s": object type of XML element "%s" is not specified or known' % (attrname, self._locator.getLineNumber(), self._locator.getColumnNumber(), self.xmlSource, name)
                         raise UnmarshallerException(errorString)
 ##                    obj.__dict__[_toAttrName(obj, attrname)] = attr
         # stuff any child attributes meant to be in a sequence via the __xmlflattensequence__
@@ -474,14 +616,12 @@ class XMLObjectFactory(xml.sax.ContentHandler):
                         flattenDict[str(xmlnametuple)] = sequencename
                     else:
                         for xmlname in xmlnametuple:
-##                            print "[endElement]: adding flattenDict[%s] = %s" % (xmlname, sequencename)
                             flattenDict[xmlname] = sequencename
             else:
                 raise Exception("Invalid type for __xmlflattensequence___ : it must be a dict")
 
         # reattach an object"s attributes to it
         for childname, child in element.children:
-##            print "[endElement] childname is: ", childname, "; child is: ", child
             if (childname in flattenDict):
                 sequencename = _toAttrName(obj, flattenDict[childname])
                 if (not hasattr(obj, sequencename)):
@@ -499,12 +639,13 @@ class XMLObjectFactory(xml.sax.ContentHandler):
                 else:
                     obj[childname] = child
             else:
-##                print "childname = %s, obj = %s, child = %s" % (childname, repr(obj), repr(child))
-                try:
-                    setattrignorecase(obj, _toAttrName(obj, childname), child)
-                except AttributeError:
-                    raise MarshallerException("Error unmarshalling child element \"%s\" of XML element \"%s\": object type not specified or known" % (childname, name))
-##                obj.__dict__[_toAttrName(obj, childname)] = child
+                # don't replace a good attribute value with a bad one
+                childAttrName = _toAttrName(obj, childname)
+                if (not hasattr(obj, childAttrName)) or (getattr(obj, childAttrName) == None) or (getattr(obj, childAttrName) == []) or (not isinstance(child, GenericXMLObject)):
+                    try:
+                        setattrignorecase(obj, childAttrName, child)
+                    except AttributeError:
+                        raise MarshallerException("Error unmarshalling child element \"%s\" of XML element \"%s\": object type not specified or known" % (childname, name))
 
         if (complexType != None):
             for element in complexType.elements:
@@ -524,7 +665,6 @@ class XMLObjectFactory(xml.sax.ContentHandler):
         if (len(self.elementstack) > 0):
 ##            print "[endElement] appending child with name: ", name, "; objtype: ", objtype
             parentElement.children.append((name, obj))
-##            print "parentElement now has ", len(parentElement.children), " children"
         else:
             self.rootelement = obj
             
@@ -539,7 +679,12 @@ def _toAttrName(obj, name):
                 break
 ##    if (name.startswith("__") and not name.endswith("__")):
 ##        name = "_%s%s" % (obj.__class__.__name__, name)
-    return name
+    return str(name)
+    
+def printKnownTypes(kt, where):
+    print 'KnownTypes from %s' % (where)
+    for tag, cls in kt.iteritems():
+        print '%s => %s' % (tag, str(cls))
 
 __typeMappingXsdToLang = {
     "string": "str",
@@ -569,7 +714,7 @@ def xsdToLangType(xsdType):
     if xsdType.startswith(XMLSCHEMA_XSD_URL):
         xsdType = xsdType[len(XMLSCHEMA_XSD_URL)+1:]
     elif xsdType.startswith(AG_URL):
-        xsdType = xsdType[len(AG_URL)+1:]        
+        xsdType = xsdType[len(AG_URL)+1:]
     langType = __typeMappingXsdToLang.get(xsdType)
     if (langType == None):
         raise Exception("Unknown xsd type %s" % xsdType)
@@ -588,8 +733,11 @@ def _getXmlValue(langValue):
     else:
         return str(langValue)
 
-def unmarshal(xmlstr, knownTypes=None, knownNamespaces=None, xmlSource=None):
-    objectfactory = XMLObjectFactory(knownTypes, knownNamespaces)
+def unmarshal(xmlstr, knownTypes=None, knownNamespaces=None, xmlSource=None, createGenerics=False):
+    objectfactory = XMLObjectFactory(knownTypes, knownNamespaces, xmlSource, createGenerics)
+    # on Linux, pyXML's sax.parseString fails when passed unicode
+    if (not sysutils.isWindows()):
+        xmlstr = str(xmlstr)
     try:
         xml.sax.parseString(xmlstr, objectfactory)
     except xml.sax.SAXParseException, errorData:
@@ -600,17 +748,19 @@ def unmarshal(xmlstr, knownTypes=None, knownNamespaces=None, xmlSource=None):
     return objectfactory.getRootObject()
 
 def marshal(obj, elementName=None, prettyPrint=False, marshalType=True, indent=0, knownTypes=None, knownNamespaces=None, encoding=-1):
-##    print '[marshal] entered with elementName = "%s"' % (elementName)
     worker = XMLMarshalWorker(prettyPrint=prettyPrint, marshalType=marshalType, knownTypes=knownTypes, knownNamespaces=knownNamespaces)    
     if obj != None and hasattr(obj, '__xmldeepexclude__'):
         worker.xmldeepexclude = obj.__xmldeepexclude__
     xmlstr = "".join(worker._marshal(obj, elementName, indent=indent))
-    if (isinstance(encoding, basestring)):
-        return '<?xml version="1.0" encoding="%s"?>\n%s' % (encoding, xmlstr.encode(encoding))
-    elif (encoding == None):
+    aglogging.info(xmlMarshallerLogger, "marshal produced string of type %s", type(xmlstr))
+    if (encoding == None):
         return xmlstr
-    else:
-        return '<?xml version="1.0" encoding="%s"?>\n%s' % (sys.getdefaultencoding(), xmlstr)
+    if (not isinstance(encoding, basestring)):
+        encoding = sys.getdefaultencoding()
+    if (not isinstance(xmlstr, unicode)):
+        xmlstr = xmlstr.decode()
+    xmlstr = u'<?xml version="1.0" encoding="%s"?>\n%s' % (encoding, xmlstr)
+    return xmlstr.encode(encoding)
 
 class XMLMarshalWorker(object):
     def __init__(self, marshalType=True, prettyPrint=False, knownTypes=None, knownNamespaces=None):
@@ -695,7 +845,7 @@ class XMLMarshalWorker(object):
             newNS.prefix = self.nsstack[-1].prefix
         else:
             newNS.prefix = ''
-        if hasattr(obj, "__xmldefaultnamespace__"):
+        if obj != None and hasattr(obj, "__xmldefaultnamespace__"):
             longPrefixNS = getattr(obj, "__xmldefaultnamespace__")
             if longPrefixNS == defaultLongNS:
                 newNS.prefix = ''
@@ -705,13 +855,12 @@ class XMLMarshalWorker(object):
                         if v == longPrefixNS:
                             newNS.prefix = k + ':'
                             break;
-##                    print '[appendNSStack] found longPrefixNS in nameSpaces = "%s"' % (newNS.prefix)
                 except:
                     if (longPrefixNS in asDict(self.knownNamespaces)):
                         newNS.prefix = self.knownNamespaces[longPrefixNS] + ':'
                     else:
                         raise MarshallerException('Error marshalling __xmldefaultnamespace__ ("%s") not defined in namespace stack' % (longPrefixNS))
-        if hasattr(obj, "targetNamespace"):
+        if obj != None and hasattr(obj, "targetNamespace"):
             newNS.targetNS = obj.targetNamespace
         elif len(self.nsstack) > 0:
             newNS.targetNS = self.nsstack[-1].targetNS
@@ -749,9 +898,11 @@ class XMLMarshalWorker(object):
             
     def _marshal(self, obj, elementName=None, nameSpacePrefix="", indent=0):
         if (obj != None):
-            xmlMarshallerLogger.debug("--> _marshal: elementName=%s%s, type=%s, obj=%s, indent=%d" % (nameSpacePrefix, elementName, type(obj), str(obj), indent))
+            aglogging.debug(xmlMarshallerLogger, "--> _marshal: elementName=%s%s, type=%s, obj=%s, indent=%d", nameSpacePrefix, elementName, type(obj), str(obj), indent)
         else:
-            xmlMarshallerLogger.debug("--> _marshal: elementName=%s%s, obj is None, indent=%d" % (nameSpacePrefix, elementName, indent))
+            aglogging.debug(xmlMarshallerLogger, "--> _marshal: elementName=%s%s, obj is None, indent=%d", nameSpacePrefix, elementName, indent)
+        if ((obj != None) and (hasattr(obj, 'preMarshal'))):
+            obj.preMarshal()
         excludeAttrs = []
         excludeAttrs.extend(self.xmldeepexclude)
         if hasattr(obj, "__xmlexclude__"):
@@ -768,8 +919,8 @@ class XMLMarshalWorker(object):
             newline = ""
             increment = 0
         ## Determine the XML element name. If it isn"t specified in the
-        ## parameter list, look for it in the __xmlname__ Lang
-        ## attribute, else use the default generic BASETYPE_ELEMENT_NAME.
+        ## parameter list, look for it in the __xmlname__ attribute,
+        ## else use the default generic BASETYPE_ELEMENT_NAME.
         nameSpaceAttrs = self.appendNSStack(obj)
         nameSpacePrefix = self.getNSPrefix()       
         if not elementName:
@@ -779,13 +930,15 @@ class XMLMarshalWorker(object):
                 elementName = nameSpacePrefix + BASETYPE_ELEMENT_NAME
         else:
             elementName = nameSpacePrefix + elementName
-##        print '[XMLMarshalWorker._marshal] elementName "%s"; nameSpaceAttrs is "%s"' % (elementName, nameSpaceAttrs)
     
         if (hasattr(obj, "__xmlsequencer__")) and (obj.__xmlsequencer__ != None):
             if (XMLSCHEMA_XSD_URL in self.nsstack[-1].nameSpaces.values()):
                 for kShort, vLong in self.nsstack[-1].nameSpaces.iteritems():
                     if vLong == XMLSCHEMA_XSD_URL:
-                        xsdPrefix = kShort + ':'
+                        if kShort != DEFAULT_NAMESPACE_KEY:
+                            xsdPrefix = kShort + ':'
+                        else:
+                            xsdPrefix = ''
                         break
             else:
                 xsdPrefix = 'xs:'
@@ -793,7 +946,6 @@ class XMLMarshalWorker(object):
         else:
             elementAdd = None
                    
-    ##    print "marshal: entered with elementName: ", elementName
         members_to_skip = []
         ## Add more members_to_skip based on ones the user has selected
         ## via the __xmlexclude__ and __xmldeepexclude__ attributes.
@@ -806,7 +958,6 @@ class XMLMarshalWorker(object):
             xmlattributes = obj.__xmlattributes__
             members_to_skip.extend(xmlattributes)
             for attr in xmlattributes:
-##                print 'Processing element "%s"; attribute "%s"' % (elementName, attr)
                 internalAttrName = attr
                 ifDefPy()
                 if (attr.startswith("__") and not attr.endswith("__")): 
@@ -814,7 +965,6 @@ class XMLMarshalWorker(object):
                 endIfDef()
                 # Fail silently if a python attribute is specified to be
                 # an XML attribute but is missing.
-##            print "marshal:   processing attribute ", internalAttrName
                 attrNameSpacePrefix = ""
                 if hasattr(obj, "__xmlattrnamespaces__"):
                     for nameSpaceKey, nameSpaceAttributes in getattr(obj, "__xmlattrnamespaces__").iteritems():
@@ -856,8 +1006,7 @@ class XMLMarshalWorker(object):
                 else:
                     value = objutils.toDiffableRepr(value)
     
-                objattrs += ' %s%s="%s"' % (attrNameSpacePrefix, attr, saxutils.escape(value))
-    ##            print "marshal:   new objattrs is: ", objattrs
+                objattrs += ' %s%s="%s"' % (attrNameSpacePrefix, attr, utillang.escape(value))
         if (obj == None):
             xmlString = [""]
         elif isinstance(obj, bool):
@@ -873,9 +1022,18 @@ class XMLMarshalWorker(object):
             objTypeStr = self._genObjTypeStr("float")
             xmlString = ['%s<%s%s>%s</%s>%s' % (prefix, elementName, objTypeStr, str(obj), elementName, newline)]
         elif isinstance(obj, unicode): # have to check before basestring - unicode is instance of base string
-            xmlString = ['%s<%s>%s</%s>%s' % (prefix, elementName, saxutils.escape(obj.encode()), elementName, newline)]
+            xmlString = ['%s<%s>%s</%s>%s' % (prefix, elementName, utillang.escape(obj.encode()), elementName, newline)]
         elif isinstance(obj, basestring):
-            xmlString = ['%s<%s>%s</%s>%s' % (prefix, elementName, saxutils.escape(obj), elementName, newline)]
+            xmlString = ['%s<%s>%s</%s>%s' % (prefix, elementName, utillang.escape(obj), elementName, newline)]
+        elif isinstance(obj, datetime.datetime):
+            objTypeStr = self._genObjTypeStr("datetime")
+            xmlString = ['%s<%s%s>%s</%s>%s' % (prefix, elementName, objTypeStr, str(obj), elementName, newline)]
+        elif isinstance(obj, datetime.date):
+            objTypeStr = self._genObjTypeStr("date")
+            xmlString = ['%s<%s%s>%s</%s>%s' % (prefix, elementName, objTypeStr, str(obj), elementName, newline)]
+        elif isinstance(obj, datetime.time):
+            objTypeStr = self._genObjTypeStr("time")
+            xmlString = ['%s<%s%s>%s</%s>%s' % (prefix, elementName, objTypeStr, str(obj), elementName, newline)]
         elif isinstance(obj, list):
             if len(obj) < 1:
                 xmlString = ""
@@ -910,13 +1068,15 @@ class XMLMarshalWorker(object):
         elif hasattr(obj, "__xmlcontent__"):
             contentValue = getattr(obj, obj.__xmlcontent__)
             if contentValue == None: 
-                contentValue = ''
+                xmlString = ["%s<%s%s%s/>%s" % (prefix, elementName, nameSpaceAttrs, objattrs, newline)]        
             else:
-                contentValue = saxutils.escape(contentValue)
-            xmlString = ["%s<%s%s%s>%s</%s>%s" % (prefix, elementName, nameSpaceAttrs, objattrs, contentValue, elementName, newline)]        
+                contentValue = utillang.escape(contentValue)
+                xmlString = ["%s<%s%s%s>%s</%s>%s" % (prefix, elementName, nameSpaceAttrs, objattrs, contentValue, elementName, newline)]        
         else:
             # Only add the objtype if the element tag is unknown to us.
-            if (self.isKnownType(elementName) == True): 
+            if (isinstance(obj, GenericXMLObject)):
+                objTypeStr = ""
+            elif (self.isKnownType(elementName) == True):
                 objTypeStr = ""
             else:
                 objTypeStr = self._genObjTypeStr("%s.%s" % (obj.__class__.__module__, className))
@@ -929,7 +1089,7 @@ class XMLMarshalWorker(object):
             if hasattr(obj, "__xmlbody__"):
                 xmlbody = getattr(obj, obj.__xmlbody__)
                 if xmlbody != None:
-                    xmlMemberString.append(xmlbody)           
+                    xmlMemberString.append(utillang.escape(xmlbody))
             else:
                 if hasattr(obj, "__xmlattrgroups__"):
                     attrGroups = obj.__xmlattrgroups__.copy()
@@ -975,21 +1135,17 @@ class XMLMarshalWorker(object):
                             xmlname = None
                             if (len(xmlnametuple) == 1):
                                 xmlname = xmlnametuple[0]
-    ##                        ix = 0
                             if not isinstance(value, (list, tuple)):
                               value = [value]
                             for seqitem in value:
-    ##                            xmlname = xmlnametuple[ix]
-    ##                            ix += 1
-    ##                            if (ix >= len(xmlnametuple)):
-    ##                                ix = 0
                                 xmlMemberString.extend(self._marshal(seqitem, xmlname, subElementNameSpacePrefix, indent=indent+increment))
                         else:
                             if (hasattr(obj, "__xmlrename__") and name in asDict(obj.__xmlrename__)):
                                 xmlname = obj.__xmlrename__[name]
                             else:
                                 xmlname = name
-                            xmlMemberString.extend(self._marshal(value, xmlname, subElementNameSpacePrefix, indent=indent+increment))
+                            if (value != None):
+                                xmlMemberString.extend(self._marshal(value, xmlname, subElementNameSpacePrefix, indent=indent+increment))
                     if (eName != "__nogroup__"):
                         xmlMemberString.append("%s</%s>%s" % (prefix, eName, newline))
                         prefix = prefix[:-increment]
@@ -1022,8 +1178,8 @@ class XMLMarshalWorker(object):
                     xmlString.append("><![CDATA[%s]]></%s>%s" % (cdataContent, elementName, newline))
                 else:
                     xmlString.append("/>%s" % newline)
-    ##        return xmlString
-        xmlMarshallerLogger.debug("<-- _marshal: %s" % str(xmlString))
+        if aglogging.isEnabledForDebug(xmlMarshallerLogger):
+            aglogging.debug(xmlMarshallerLogger, "<-- _marshal: %s", objutils.toDiffableString(xmlString))
         #print "<-- _marshal: %s" % str(xmlString)
         self.popNSStack()
         return xmlString

wxPython/samples/ide/activegrid/util/xmlutils.py

@@ -22,35 +22,44 @@ import activegrid.util.aglogging as aglogging
 
 xmlLogger = logging.getLogger("activegrid.util.xml")
     
-def load(fileName, knownTypes=None, knownNamespaces=None):
+def load(fileName, knownTypes=None, knownNamespaces=None, createGenerics=False):
     loadedObject = None
     fileObject = file(fileName)
     timeStart = time.time()
+    xml = ""
     try:
         xml = fileObject.read()
-        loadedObject = unmarshal(xml, knownTypes=knownTypes, knownNamespaces=knownNamespaces, xmlSource=fileName)
+        loadedObject = unmarshal(xml, knownTypes=knownTypes, knownNamespaces=knownNamespaces, xmlSource=fileName, createGenerics=createGenerics)
         loadedObject.fileName = os.path.abspath(fileName)
         if hasattr(loadedObject, 'initialize'):
             loadedObject.initialize()
     finally:
         fileObject.close()
-        timeDone = time.time()
-        aglogging.info(xmlLogger, ('Load statistics for file %s: elapsed time = %f secs' % (fileName, timeDone-timeStart)))
+        if xmlLogger.isEnabledFor(aglogging.LEVEL_INFO):
+            timeDone = time.time()
+            aglogging.info(xmlLogger, ('Load statistics for file %s (%d bytes): elapsed time = %f secs' % (fileName, len(xml), timeDone-timeStart)))
     return loadedObject
 
-def loadURI(uri, knownTypes=None, knownNamespaces=None, xmlSource=None):
+def loadURI(uri, knownTypes=None, knownNamespaces=None, xmlSource=None, createGenerics=False):
     loadedObject = None
-    xml = urllib.urlopen(uri).read()
-    loadedObject = unmarshal(xml, knownTypes=knownTypes, knownNamespaces=knownNamespaces, xmlSource=xmlSource)
-    loadedObject.fileName = uri
-    if hasattr(loadedObject, 'initialize'):
-        loadedObject.initialize()
+    timeStart = time.time()
+    xml = ""
+    try:
+        xml = urllib.urlopen(uri).read()
+        loadedObject = unmarshal(xml, knownTypes=knownTypes, knownNamespaces=knownNamespaces, xmlSource=xmlSource, createGenerics=createGenerics)
+        loadedObject.fileName = uri
+        if hasattr(loadedObject, 'initialize'):
+            loadedObject.initialize()
+    finally:
+        if xmlLogger.isEnabledFor(aglogging.LEVEL_INFO):
+            timeDone = time.time()
+            aglogging.info(xmlLogger, ('Load statistics for URI %s (%d bytes): elapsed time = %f secs' % (uri, len(xml), timeDone-timeStart)))
     return loadedObject
 
-def unmarshal(xml, knownTypes=None, knownNamespaces=None, xmlSource=None):
+def unmarshal(xml, knownTypes=None, knownNamespaces=None, xmlSource=None, createGenerics=False):
     if (knownTypes == None): 
         knownTypes, knownNamespaces = getAgKnownTypes()
-    return xmlmarshaller.unmarshal(xml, knownTypes=knownTypes, knownNamespaces=knownNamespaces, xmlSource=xmlSource)    
+    return xmlmarshaller.unmarshal(xml, knownTypes=knownTypes, knownNamespaces=knownNamespaces, xmlSource=xmlSource, createGenerics=createGenerics)    
 
 def save(fileName, objectToSave, prettyPrint=True, marshalType=True, knownTypes=None, knownNamespaces=None, encoding='utf-8'):
     if hasattr(objectToSave, '_xmlReadOnly') and objectToSave._xmlReadOnly == True:
@@ -155,41 +164,6 @@ def getAgVersion(fileName):
             version = xml[i+1:j]
     return version
 
-def escape(data):
-    """Escape ', ", &, <, and > in a string of data.
-
-    Basically, everything that saxutils.escape does (and this calls that, at
-    least for now), but with " added as well.
-
-    XXX TODO make this faster; saxutils.escape() is really slow
-    """
-
-    import xml.sax.saxutils as saxutils
-
-    data=saxutils.escape(data)
-    data=data.replace("\"", "&quot;")
-
-    # IE doesn't support &apos;
-    # data=data.replace("\'", "&apos;")
-    data=data.replace("\'", "&#039;")
-
-    return data
-
-def unescape(data):
-    """Unescape ', ", &, <, and > in a string of data.
-
-    Basically, everything that saxutils.unescape does (and this calls that, at
-    least for now), but with " added as well.
-
-    XXX TODO make this faster; saxutils.unescape() is really slow
-    """
-
-    import xml.sax.saxutils as saxutils
-
-    data=data.replace("&quot;", "\"")
-    data=data.replace("&apos;", "\'")
-    return saxutils.unescape(data)
-
     
 AG_NS_URL = "http://www.activegrid.com/ag.xsd"
 BPEL_NS_URL = "http://schemas.xmlsoap.org/ws/2003/03/business-process"
@@ -197,7 +171,9 @@ HTTP_WSDL_NS_URL = "http://schemas.xmlsoap.org/wsdl/http/"
 MIME_WSDL_NS_URL = "http://schemas.xmlsoap.org/wsdl/mime/"
 SOAP_NS_URL = "http://schemas.xmlsoap.org/wsdl/soap/"
 SOAP12_NS_URL = "http://schemas.xmlsoap.org/wsdl/soap12/"
+SOAP_NS_ENCODING = "http://schemas.xmlsoap.org/soap/encoding/"
 WSDL_NS_URL = "http://schemas.xmlsoap.org/wsdl/"
+WSSE_NS_URL = "http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"
 XFORMS_NS_URL = "http://www.w3c.org/xform.xsd"
 XMLSCHEMA_NS_URL = "http://www.w3.org/2001/XMLSchema"
 XSI_NS_URL = "http://www.w3.org/2001/XMLSchema-instance"
@@ -209,7 +185,8 @@ KNOWN_NAMESPACES = { AG_NS_URL          :  "ag",
                      MIME_WSDL_NS_URL   :  "mime",
                      SOAP_NS_URL        :  "soap",
                      SOAP12_NS_URL      :  "soap12",
-                     WSDL_NS_URL        :  "wsdl", 
+                     WSDL_NS_URL        :  "wsdl",
+                     WSSE_NS_URL        :  "wsse", 
                      XFORMS_NS_URL      :  "xforms",                             
                      XMLSCHEMA_NS_URL   :  "xs",
                      XACML_NS_URL       :  "xacml",
@@ -226,19 +203,23 @@ def getAgXsdToClassName():
             "ag:body"            : "activegrid.model.processmodel.Body",
             "ag:category_substitutions"    : "activegrid.server.layoutrenderer.CategorySubstitutions",
             "ag:command"         : "activegrid.model.wsdl.Command",
+            "ag:setElement"      : "activegrid.model.processmodel.SetElementOperation",
             "ag:css"             : "activegrid.server.layoutrenderer.CSS", 
-            "ag:cssRule"         : "activegrid.model.processmodel.CssRule",
             "ag:databaseService" : "activegrid.server.deployment.DatabaseService",
             "ag:datasource"      : "activegrid.data.dataservice.DataSource",
             "ag:dataObjectList"  : "activegrid.data.datalang.DataObjectList",
             "ag:debug"           : "activegrid.model.processmodel.DebugOperation",
             "ag:deployment"      : "activegrid.server.deployment.Deployment",
+            "ag:formData"        : "activegrid.model.processmodel.FormData",
+            "ag:formVar"         : "activegrid.model.processmodel.FormVar",
             "ag:generator"       : "activegrid.server.layoutrenderer.SerializableGenerator", 
             "ag:head"            : "activegrid.server.layoutrenderer.Head", 
             "ag:hr"              : "activegrid.model.processmodel.HorizontalRow",
             "ag:identity"        : "activegrid.model.identitymodel.Identity",
             "ag:identityref"     : "activegrid.server.deployment.IdentityRef",
             "ag:image"           : "activegrid.model.processmodel.Image",
+            "ag:inputPart"       : "activegrid.model.processmodel.InputPart",
+            "ag:keystore"        : "activegrid.model.identitymodel.KeyStore",
             "ag:label"           : "activegrid.model.processmodel.Label",
             "ag:layout"          : "activegrid.server.layoutrenderer.Layout", 
             "ag:layouts"         : "activegrid.server.layoutrenderer.Layouts", 
@@ -246,9 +227,11 @@ def getAgXsdToClassName():
             "ag:localService"    : "activegrid.server.deployment.LocalService",
             "ag:parameter"       : "activegrid.server.layoutrenderer.Parameter",
             "ag:parameters"      : "activegrid.server.layoutrenderer.Parameters",
+            "ag:postInitialize"  : "activegrid.model.processmodel.PostInitialize",
             "ag:processref"      : "activegrid.server.deployment.ProcessRef",
             "ag:query"           : "activegrid.model.processmodel.Query",
             "ag:soapService"     : "activegrid.server.deployment.SoapService",
+            "ag:redirect"        : "activegrid.server.layoutrenderer.Redirect", 
             "ag:requiredFile"    : "activegrid.server.layoutrenderer.RequiredFile", 
             "ag:resource"        : "activegrid.model.identitymodel.IDResource",
             "ag:restService"     : "activegrid.server.deployment.RestService",
@@ -355,27 +338,38 @@ def getAgXsdToClassName():
             "xforms:xforms"      : "activegrid.model.processmodel.XFormsRoot",
             "xs:all"             : "activegrid.model.schema.XsdSequence",
             "xs:any"             : "activegrid.model.schema.XsdAny",
+            "xs:anyAttribute"    : "activegrid.model.schema.XsdAnyAttribute",
             "xs:attribute"       : "activegrid.model.schema.XsdAttribute",
+            "xs:choice"          : "activegrid.model.schema.XsdChoice",
             "xs:complexContent"  : "activegrid.model.schema.XsdComplexContent",
             "xs:complexType"     : "activegrid.model.schema.XsdComplexType",
+            "xs:documentation"   : "activegrid.model.schema.XsdDocumentation",
             "xs:element"         : "activegrid.model.schema.XsdElement",
-            "xs:enumeration"     : "activegrid.model.schema.XsdEnumeration",
+            "xs:enumeration"     : "activegrid.model.schema.XsdFacetEnumeration",
             "xs:extension"       : "activegrid.model.schema.XsdExtension",
+            "xs:fractionDigits"  : "activegrid.model.schema.XsdFacetFractionDigits",
             "xs:field"           : "activegrid.model.schema.XsdKeyField",
             "xs:import"          : "activegrid.model.schema.XsdInclude",
             "xs:include"         : "activegrid.model.schema.XsdInclude",
             "xs:key"             : "activegrid.model.schema.XsdKey",
             "xs:keyref"          : "activegrid.model.schema.XsdKeyRef",
-            "xs:length"          : "activegrid.model.schema.XsdLength",
+            "xs:length"          : "activegrid.model.schema.XsdFacetLength",
             "xs:list"            : "activegrid.model.schema.XsdList",
-            "xs:maxLength"       : "activegrid.model.schema.XsdMaxLength",
+            "xs:maxExclusive"    : "activegrid.model.schema.XsdFacetMaxExclusive",
+            "xs:maxInclusive"    : "activegrid.model.schema.XsdFacetMaxInclusive",
+            "xs:maxLength"       : "activegrid.model.schema.XsdFacetMaxLength",
+            "xs:minExclusive"    : "activegrid.model.schema.XsdFacetMinExclusive",
+            "xs:minInclusive"    : "activegrid.model.schema.XsdFacetMinInclusive",
+            "xs:minLength"       : "activegrid.model.schema.XsdFacetMinLength",
+            "xs:pattern"         : "activegrid.model.schema.XsdFacetPattern",
             "xs:restriction"     : "activegrid.model.schema.XsdRestriction",
             "xs:schema"          : "activegrid.model.schema.Schema",
             "xs:selector"        : "activegrid.model.schema.XsdKeySelector",              
             "xs:sequence"        : "activegrid.model.schema.XsdSequence",
             "xs:simpleContent"   : "activegrid.model.schema.XsdSimpleContent",
             "xs:simpleType"      : "activegrid.model.schema.XsdSimpleType",
-            "xs:totalDigits"     : "activegrid.model.schema.XsdTotalDigits",
+            "xs:totalDigits"     : "activegrid.model.schema.XsdFacetTotalDigits",
+            "xs:whiteSpace"      : "activegrid.model.schema.XsdFacetWhiteSpace",
         }
     return agXsdToClassName