Index: external/Zope2/lib/python/DocumentTemplate/DT_In.py
diff -c external/Zope2/lib/python/DocumentTemplate/DT_In.py:1.1.1.2 external/Zope2/lib/python/DocumentTemplate/DT_In.py:1.2
*** external/Zope2/lib/python/DocumentTemplate/DT_In.py:1.1.1.2	Thu Jun 22 12:04:53 2000
--- external/Zope2/lib/python/DocumentTemplate/DT_In.py	Fri Jul  7 14:39:05 2000
***************
*** 382,391 ****
  
  ''' #'
  
! __rcs_id__='$Id: DT_In.py,v 1.1.1.2 2000/06/22 11:04:53 tdickenson Exp $'
! __version__='$Revision: 1.1.1.2 $'[11:-2]
  
! from DT_Util import ParseError, parse_params, name_param, str
  from DT_Util import render_blocks, InstanceDict, ValidationError
  from string import find, atoi, join, split
  import ts_regex
--- 382,391 ----
  
  ''' #'
  
! __rcs_id__='$Id: DT_In.py,v 1.2 2000/07/07 13:39:05 tdickenson Exp $'
! __version__='$Revision: 1.2 $'[11:-2]
  
! from DT_Util import ParseError, parse_params, name_param, str, join_unicode
  from DT_Util import render_blocks, InstanceDict, ValidationError
  from string import find, atoi, join, split
  import ts_regex
***************
*** 614,620 ****
                      if index==first: kw['sequence-start']=0
  
  
!                 result=join(result, '')
  
          finally:
              if cache: pop()
--- 614,620 ----
                      if index==first: kw['sequence-start']=0
  
  
!             result = join_unicode(result, '')
  
          finally:
              if cache: pop()
***************
*** 692,698 ****
                      finally: pop()
                      if index==0: kw['sequence-start']=0
  
!                 result=join(result, '')
  
          finally:
              if cache: pop()
--- 692,698 ----
                      finally: pop()
                      if index==0: kw['sequence-start']=0
  
!                 result=join_unicode(result, '')
  
          finally:
              if cache: pop()
Index: external/Zope2/lib/python/DocumentTemplate/DT_Util.py
diff -c external/Zope2/lib/python/DocumentTemplate/DT_Util.py:1.1.1.2 external/Zope2/lib/python/DocumentTemplate/DT_Util.py:1.5
*** external/Zope2/lib/python/DocumentTemplate/DT_Util.py:1.1.1.2	Thu Jun 22 12:04:53 2000
--- external/Zope2/lib/python/DocumentTemplate/DT_Util.py	Tue Jul 11 12:40:45 2000
***************
*** 82,89 ****
  # attributions are listed in the accompanying credits file.
  # 
  ##############################################################################
! '''$Id: DT_Util.py,v 1.1.1.2 2000/06/22 11:04:53 tdickenson Exp $''' 
! __version__='$Revision: 1.1.1.2 $'[11:-2]
  
  import regex, string, math, os
  from string import strip, join, atoi, lower, split, find
--- 82,89 ----
  # attributions are listed in the accompanying credits file.
  # 
  ##############################################################################
! '''$Id: DT_Util.py,v 1.5 2000/07/11 11:40:45 tdickenson Exp $''' 
! __version__='$Revision: 1.5 $'[11:-2]
  
  import regex, string, math, os
  from string import strip, join, atoi, lower, split, find
***************
*** 94,110 ****
  ParseError='Document Template Parse Error'
  ValidationError='Unauthorized'
  
- 
  def html_quote(v, name='(Unknown name)', md={},
                 character_entities=(
                         (('&'),    '&'),
                         (('<'),    '&lt;' ),
                         (('>'),    '&gt;' ),
!                        (('\213'), '&lt;' ),
!                        (('\233'), '&gt;' ),
                         (('"'),    '&quot;'))): #"
!         text=str(v)
!         for re,name in character_entities:
              if find(text, re) >= 0: text=join(split(text,re),name)
          return text
  
--- 94,112 ----
  ParseError='Document Template Parse Error'
  ValidationError='Unauthorized'
  
  def html_quote(v, name='(Unknown name)', md={},
                 character_entities=(
                         (('&'),    '&amp;'),
                         (('<'),    '&lt;' ),
                         (('>'),    '&gt;' ),
! 
!                        # What are these two needed for?????
!                        #(('\213'), '&lt;' ),
!                        #(('\233'), '&gt;' ),
! 
                         (('"'),    '&quot;'))): #"
!         text=ustr(v)
!         for re,name in character_entities:  
              if find(text, re) >= 0: text=join(split(text,re),name)
          return text
  
***************
*** 211,220 ****
      from cDocumentTemplate import cDocument
  except: from pDocumentTemplate import InstanceDict, TemplateDict, render_blocks
  
- 
  d=TemplateDict.__dict__
  for name in ('None', 'abs', 'chr', 'divmod', 'float', 'hash', 'hex', 'int',
!              'len', 'max', 'min', 'oct', 'ord', 'round', 'str'):
      d[name]=__builtins__[name]
  d['string']=string
  d['math']=math
--- 213,222 ----
      from cDocumentTemplate import cDocument
  except: from pDocumentTemplate import InstanceDict, TemplateDict, render_blocks
  
  d=TemplateDict.__dict__
  for name in ('None', 'abs', 'chr', 'divmod', 'float', 'hash', 'hex', 'int',
!              'len', 'max', 'min', 'oct', 'ord', 'round', 'str',
!              'unicode', 'unichr', 'ustr',):
      d[name]=__builtins__[name]
  d['string']=string
  d['math']=math
***************
*** 535,537 ****
--- 532,553 ----
      text=strip(text[l:])
      if text: return apply(parse_params,(text,result),parms)
      else: return result
+ 
+ 
+ def join_unicode(sequence,sep=' '):
+     # Unicode aware join.
+     # This does roughly the same job as string.join, however this function will
+     # not report an error if there is a mix of unicode string and 8bit strings
+     # with the 7th bit set. Instead, the 8bit strings are interpreted as strings
+     # of latin-1 characters.
+     # This is the preferred way of combining content for Zope responses.
+     try:
+         return join(sequence, '')
+     except UnicodeError:
+         # A mix of unicode string and normal strings with the 7th bit.
+         # Python considers this mix more dangerous than I do.
+         for i in range(len(sequence)):
+             if type(sequence[i]) is type(''):
+                 sequence[i] = unicode(sequence[i],'latin-1')
+         return join(sequence, sep)
+     
\ No newline at end of file
Index: external/Zope2/lib/python/DocumentTemplate/DT_Var.py
diff -c external/Zope2/lib/python/DocumentTemplate/DT_Var.py:1.1.1.2 external/Zope2/lib/python/DocumentTemplate/DT_Var.py:1.2
*** external/Zope2/lib/python/DocumentTemplate/DT_Var.py:1.1.1.2	Thu Jun 22 12:04:53 2000
--- external/Zope2/lib/python/DocumentTemplate/DT_Var.py	Fri Jul  7 14:39:05 2000
***************
*** 217,224 ****
     
  
  ''' # '
! __rcs_id__='$Id: DT_Var.py,v 1.1.1.2 2000/06/22 11:04:53 tdickenson Exp $'
! __version__='$Revision: 1.1.1.2 $'[11:-2]
  
  from DT_Util import parse_params, name_param, html_quote, str
  import regex, string, sys, regex
--- 217,224 ----
     
  
  ''' # '
! __rcs_id__='$Id: DT_Var.py,v 1.2 2000/07/07 13:39:05 tdickenson Exp $'
! __version__='$Revision: 1.2 $'[11:-2]
  
  from DT_Util import parse_params, name_param, html_quote, str
  import regex, string, sys, regex
***************
*** 314,321 ****
  
          # finally, pump it through the actual string format...
          fmt=self.fmt
!         if fmt=='s': val=str(val)
!         else: val = ('%'+self.fmt) % (val,)
  
          # next, look for upper, lower, etc
          for f in self.modifiers: val=f(val)
--- 314,323 ----
  
          # finally, pump it through the actual string format...
          fmt=self.fmt
!         if fmt=='s':
!             val=ustr(val)
!         else:
!             val = ('%'+self.fmt) % (val,)
  
          # next, look for upper, lower, etc
          for f in self.modifiers: val=f(val)
Index: external/Zope2/lib/python/DocumentTemplate/__init__.py
diff -c external/Zope2/lib/python/DocumentTemplate/__init__.py:1.1.1.1 external/Zope2/lib/python/DocumentTemplate/__init__.py:1.3
*** external/Zope2/lib/python/DocumentTemplate/__init__.py:1.1.1.1	Fri Apr 28 21:06:55 2000
--- external/Zope2/lib/python/DocumentTemplate/__init__.py	Tue Jul 11 12:15:16 2000
***************
*** 87,95 ****
  This wrapper allows the (now many) document template modules to be
  segregated in a separate package.
  
! $Id: __init__.py,v 1.1.1.1 2000/04/28 20:06:55 tdickenson Exp $'''
! __version__='$Revision: 1.1.1.1 $'[11:-2]
  
  import ExtensionClass # work-around for import bug.
  from DocumentTemplate import String, File, HTML, HTMLDefault, HTMLFile
  from DocumentTemplate import html_quote
--- 87,111 ----
  This wrapper allows the (now many) document template modules to be
  segregated in a separate package.
  
! $Id: __init__.py,v 1.3 2000/07/11 11:15:16 tdickenson Exp $'''
! __version__='$Revision: 1.3 $'[11:-2]
  
+ try:
+     ustr
+ except:
+     # This ustr function was not provided in early python 1.6 alpha releases
+     import __builtin__
+     def ustr(v):
+         if type(v)==type(u''):
+             return v
+         else:
+             return str(v)
+     __builtin__.ustr = ustr
+     del __builtin__
+     del ustr
+ 
+ 
  import ExtensionClass # work-around for import bug.
  from DocumentTemplate import String, File, HTML, HTMLDefault, HTMLFile
  from DocumentTemplate import html_quote
+ 
Index: external/Zope2/lib/python/DocumentTemplate/cDocumentTemplate.c
diff -c external/Zope2/lib/python/DocumentTemplate/cDocumentTemplate.c:1.1.1.2 external/Zope2/lib/python/DocumentTemplate/cDocumentTemplate.c:1.3
*** external/Zope2/lib/python/DocumentTemplate/cDocumentTemplate.c:1.1.1.2	Thu Jun 22 12:04:53 2000
--- external/Zope2/lib/python/DocumentTemplate/cDocumentTemplate.c	Tue Jul 11 12:15:16 2000
***************
*** 84,95 ****
   ****************************************************************************/
  static char cDocumentTemplate_module_documentation[] = 
  ""
! "\n$Id: cDocumentTemplate.c,v 1.1.1.2 2000/06/22 11:04:53 tdickenson Exp $"
  ;
  
  #include "ExtensionClass.h"
  
! static PyObject *py_isDocTemp=0, *py_blocks=0, *py_=0, *join=0, *py_acquire;
  static PyObject *py___call__, *py___roles__, *py_AUTHENTICATED_USER;
  static PyObject *py_hasRole, *py__proxy_roles, *py_Unauthorized;
  static PyObject *py_Unauthorized_fmt, *py_validate;
--- 84,95 ----
   ****************************************************************************/
  static char cDocumentTemplate_module_documentation[] = 
  ""
! "\n$Id: cDocumentTemplate.c,v 1.3 2000/07/11 11:15:16 tdickenson Exp $"
  ;
  
  #include "ExtensionClass.h"
  
! static PyObject *py_isDocTemp=0, *py_blocks=0, *py_=0, *join=0, *py_acquire, *ustr=0;
  static PyObject *py___call__, *py___roles__, *py_AUTHENTICATED_USER;
  static PyObject *py_hasRole, *py__proxy_roles, *py_Unauthorized;
  static PyObject *py_Unauthorized_fmt, *py_validate;
***************
*** 736,742 ****
  	      block=PyTuple_GET_ITEM(block,0);
  	      if (PyString_Check(block)) block=PyObject_GetItem(md,block);
  	      else block=PyObject_CallObject(block,mda);
! 	      if (block) ASSIGN(block, PyObject_Str(block));
  	      UNLESS(block) return -1;
  	    }
  	  else
--- 736,742 ----
  	      block=PyTuple_GET_ITEM(block,0);
  	      if (PyString_Check(block)) block=PyObject_GetItem(md,block);
  	      else block=PyObject_CallObject(block,mda);
! 	      if (block) ASSIGN(block,PyObject_CallFunction(ustr,"O",block));
  	      UNLESS(block) return -1;
  	    }
  	  else
***************
*** 813,819 ****
  		if (if_finally(md,0) == -2) return -1;
  	    }
  	}
!       else if (PyString_Check(block))
  	{
  	  Py_INCREF(block);
  	}
--- 813,819 ----
  		if (if_finally(md,0) == -2) return -1;
  	    }
  	}
!       else if (PyString_Check(block) || PyUnicode_Check(block))
  	{
  	  Py_INCREF(block);
  	}
***************
*** 833,842 ****
    return 0;
  }
  
  static PyObject *
  render_blocks(PyObject *self, PyObject *args)
  {
!   PyObject *md, *blocks, *mda=0, *rendered=0;
    int l;
  
    UNLESS(PyArg_ParseTuple(args,"OO", &blocks, &md)) return NULL;
--- 833,871 ----
    return 0;
  }
  
+ 
+ static int
+ render_unicode_fixup(PyObject *list)
+ {
+     PyObject *string,*unicode;
+     int i,l;
+ 
+     l = PyList_Size(list);
+ 
+     for(i=0;i<l;++i)
+     {
+         if(PyString_Check(PyList_GetItem(list,i)))
+         {
+             string = PyList_GetItem(list,i);
+             unicode = PyUnicode_DecodeLatin1(PyString_AsString(string),PyString_Size(string),NULL);
+             if(unicode)
+             {
+                 PyList_SetItem(list,i,unicode);
+             }
+             else
+             {
+                 return -1;
+             }
+         }
+     }
+     return 0;
+ }
+ 
+ 
  static PyObject *
  render_blocks(PyObject *self, PyObject *args)
  {
!   PyObject *md, *blocks, *mda=0, *rendered=0, *prejoined=0;
    int l;
  
    UNLESS(PyArg_ParseTuple(args,"OO", &blocks, &md)) return NULL;
***************
*** 845,851 ****
    
    if (render_blocks_(blocks, rendered, md, mda) < 0) goto err;
  
!   Py_DECREF(mda);
  
    l=PyList_Size(rendered);
    if (l==0)
--- 874,880 ----
    
    if (render_blocks_(blocks, rendered, md, mda) < 0) goto err;
  
!   ASSIGN(mda,NULL);
  
    l=PyList_Size(rendered);
    if (l==0)
***************
*** 856,862 ****
    else if (l==1)
      ASSIGN(rendered, PySequence_GetItem(rendered,0));
    else
!     ASSIGN(rendered, PyObject_CallFunction(join,"OO",rendered,py_));
  
    return rendered;
  
--- 885,907 ----
    else if (l==1)
      ASSIGN(rendered, PySequence_GetItem(rendered,0));
    else
!     {
!       prejoined = rendered;
!       rendered = PyObject_CallFunction(join,"OO",prejoined,py_);
!       if(!rendered && PyErr_ExceptionMatches(PyExc_UnicodeError))
!       {
!           PyErr_Clear();
!           if(render_unicode_fixup(prejoined)<0)
!           {
!               Py_DECREF(prejoined);
!               goto err;
!           }
!           else
!           {
!               rendered = PyObject_CallFunction(join,"OO",prejoined,py_);
!           }
!       }
!     }
  
    return rendered;
  
***************
*** 1063,1069 ****
  initcDocumentTemplate()
  {
    PyObject *m, *d;
!   char *rev="$Revision: 1.1.1.2 $";
    PURE_MIXIN_CLASS(cDocument,
  	"Base class for documents that adds fast validation method",
  	Document_methods);
--- 1108,1114 ----
  initcDocumentTemplate()
  {
    PyObject *m, *d;
!   char *rev="$Revision: 1.3 $";
    PURE_MIXIN_CLASS(cDocument,
  	"Base class for documents that adds fast validation method",
  	Document_methods);
***************
*** 1089,1097 ****
--- 1134,1149 ----
      return;
  
    UNLESS(py_=PyString_FromString("")) return;
+ 
    UNLESS(join=PyImport_ImportModule("string")) return;
    ASSIGN(join,PyObject_GetAttrString(join,"join"));
    UNLESS(join) return;
+ 
+   /* does this have a C interface ? */
+   UNLESS(ustr=PyImport_ImportModule("__builtin__")) return;
+   ASSIGN(ustr,PyObject_GetAttrString(ustr,"ustr"));
+   UNLESS(ustr) return;
+ 
    UNLESS(ExtensionClassImported) return;
  
    m = Py_InitModule4("cDocumentTemplate", Module_Level__methods,
Index: external/Zope2/lib/python/DocumentTemplate/pDocumentTemplate.py
diff -c external/Zope2/lib/python/DocumentTemplate/pDocumentTemplate.py:1.1.1.1 external/Zope2/lib/python/DocumentTemplate/pDocumentTemplate.py:1.3
*** external/Zope2/lib/python/DocumentTemplate/pDocumentTemplate.py:1.1.1.1	Fri Apr 28 21:06:55 2000
--- external/Zope2/lib/python/DocumentTemplate/pDocumentTemplate.py	Tue Jul 11 12:15:16 2000
***************
*** 85,97 ****
  __doc__='''Python implementations of document template some features
  
  
! $Id: pDocumentTemplate.py,v 1.1.1.1 2000/04/28 20:06:55 tdickenson Exp $'''
! __version__='$Revision: 1.1.1.1 $'[11:-2]
  
  import string, sys, types
  from string import join
  
  StringType=type('')
  TupleType=type(())
  isFunctionType={}
  for name in ['BuiltinFunctionType', 'BuiltinMethodType', 'ClassType',
--- 85,98 ----
  __doc__='''Python implementations of document template some features
  
  
! $Id: pDocumentTemplate.py,v 1.3 2000/07/11 11:15:16 tdickenson Exp $'''
! __version__='$Revision: 1.3 $'[11:-2]
  
  import string, sys, types
  from string import join
  
  StringType=type('')
+ UnicodeType=type(u'')
  TupleType=type(())
  isFunctionType={}
  for name in ['BuiltinFunctionType', 'BuiltinMethodType', 'ClassType',
***************
*** 231,237 ****
                  section=section[0]
                  if type(section) is StringType: section=md[section]
                  else: section=section(md)
!                 section=str(section)
              else:
                  # if
                  cache={}
--- 232,238 ----
                  section=section[0]
                  if type(section) is StringType: section=md[section]
                  else: section=section(md)
!                 section=ustr(section)
              else:
                  # if
                  cache={}
***************
*** 264,270 ****
  
                  finally: md._pop()
  
!         elif type(section) is not StringType:
              section=section(md)
  
          if section: rendered.append(section)
--- 265,271 ----
  
                  finally: md._pop()
  
!         elif type(section) is not StringType and type(section) is not UnicodeType:
              section=section(md)
  
          if section: rendered.append(section)
***************
*** 272,276 ****
      l=len(rendered)
      if l==0: return ''
      elif l==1: return rendered[0]
!     return join(rendered, '')
!     return rendered
--- 273,288 ----
      l=len(rendered)
      if l==0: return ''
      elif l==1: return rendered[0]
!     try:
!         return join(rendered, '')
!     except UnicodeError:
!         # A mix of unicode string and normal strings with the 7th bit.
!         # Python considers this mix more dangerous than I do. Fix up the
!         # list, treating normal strings as latin-1
!         for i in range(len(rendered)):
!             if type(rendered[i]) is StringType:
!                 rendered[i] = unicode(rendered[i],'latin-1')
!         return join(rendered, u'')
! 
! 
! 
Index: external/Zope2/lib/python/OFS/SimpleItem.py
diff -c external/Zope2/lib/python/OFS/SimpleItem.py:1.1.1.2 external/Zope2/lib/python/OFS/SimpleItem.py:1.2
*** external/Zope2/lib/python/OFS/SimpleItem.py:1.1.1.2	Thu Jun 22 12:04:54 2000
--- external/Zope2/lib/python/OFS/SimpleItem.py	Tue Jul 11 12:15:16 2000
***************
*** 89,96 ****
  This module can also be used as a simple template for implementing new
  item types. 
  
! $Id: SimpleItem.py,v 1.1.1.2 2000/06/22 11:04:54 tdickenson Exp $'''
! __version__='$Revision: 1.1.1.2 $'[11:-2]
  
  import regex, sys, Globals, App.Management, Acquisition, App.Undo
  import AccessControl.Role, AccessControl.Owned, App.Common
--- 89,96 ----
  This module can also be used as a simple template for implementing new
  item types. 
  
! $Id: SimpleItem.py,v 1.2 2000/07/11 11:15:16 tdickenson Exp $'''
! __version__='$Revision: 1.2 $'[11:-2]
  
  import regex, sys, Globals, App.Management, Acquisition, App.Undo
  import AccessControl.Role, AccessControl.Owned, App.Common
***************
*** 232,240 ****
  
              if not error_message:
                  if type(error_value) is InstanceType:
!                     s=str(error_value)
!                     if tagSearch(s) >= 0:
!                         error_message=error_value
                  elif (type(error_value) is StringType
                        and tagSearch(error_value) >= 0):
                      error_message=error_value
--- 232,244 ----
  
              if not error_message:
                  if type(error_value) is InstanceType:
!                     try:
!                         s=str(error_value)
!                     except:
!                         pass
!                     else:
!                         if tagSearch(s) >= 0:
!                             error_message=error_value
                  elif (type(error_value) is StringType
                        and tagSearch(error_value) >= 0):
                      error_message=error_value
Index: external/Zope2/lib/python/OFS/properties.dtml
diff -c external/Zope2/lib/python/OFS/properties.dtml:1.1.1.2 external/Zope2/lib/python/OFS/properties.dtml:1.7
*** external/Zope2/lib/python/OFS/properties.dtml:1.1.1.2	Thu Jun 22 12:04:54 2000
--- external/Zope2/lib/python/OFS/properties.dtml	Tue Jul 11 15:00:30 2000
***************
*** 1,7 ****
--- 1,23 ----
  <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+ 
+ <dtml-comment>
+     Does this 'Properties' tab need to support any browsers that do not understand utf8?
+     I cant say 'No' for certain :-(
+ </dtml-comment>
+ 
+ <dtml-call "RESPONSE.setHeader('content-type','text/html; charset=ISO-8859-1')">
+ <dtml-in propertyMap mapping>
+   <dtml-if "type in ['ustring','utokens', 'ulines', 'utext']">
+     <dtml-call "RESPONSE.setHeader('content-type','text/html; charset=UTF-8')">
+   </dtml-if>
+ </dtml-in>
+ 
+ <dtml-let encoding="['latin1','utf8'][RESPONSE.getHeader('content-type')[-1]=='8']">
+ 
  <html lang="en">
  <head>
  <title>Properties</title>
+ <meta http-equiv="Content-Type" content="<dtml-var "RESPONSE.getHeader('content-type')">">
  </head>
  <body bgcolor="#FFFFFF" link="#000099" vlink="#555555">
  <dtml-var manage_tabs>
***************
*** 30,36 ****
  <tr>
    <td align="left" valign="top" width="16">
    <dtml-if "'d' in _['sequence-item'].get('mode', 'awd')">
!   <input type="checkbox" name="ids:list" value="<dtml-var id html_quote>" ID="cb-<dtml-var id>">
    <dtml-else>
    </dtml-if>
    </td>
--- 46,52 ----
  <tr>
    <td align="left" valign="top" width="16">
    <dtml-if "'d' in _['sequence-item'].get('mode', 'awd')">
!   <input type="checkbox" name="ids:<dtml-var encoding>:list" value="<dtml-var id html_quote>" ID="cb-<dtml-var id>">
    <dtml-else>
    </dtml-if>
    </td>
***************
*** 50,74 ****
    <dtml-elif "type in ('float', 'date')">
    <input type="text" name="<dtml-var id>:<dtml-var type>" size="35"
     value="<dtml-var "getProperty(id)" html_quote>">
!   <dtml-elif "type=='string'">
!   <input type="text" name="<dtml-var id>:string" size="35"
     value="<dtml-var "getProperty(id)" html_quote>">
    <dtml-elif "type=='boolean'">
    <input type="checkbox" name="<dtml-var id>:boolean" size="35"
     <dtml-if "getProperty(id)">CHECKED</dtml-if>>
!   <dtml-elif "type=='tokens'">
!   <input type="text" name="<dtml-var id>:tokens" size="35"
     value="<dtml-in "getProperty(id)"><dtml-var sequence-item html_quote> </dtml-in>">
!   <dtml-elif "type=='text'">
!   <textarea name="<dtml-var id>:text" rows="6" cols="35"><dtml-var 
     "getProperty(id)" html_quote></textarea>
!   <dtml-elif "type=='lines'">
!   <textarea name="<dtml-var id>:lines" rows="6" cols="35"><dtml-in
    "getProperty(id)"><dtml-var sequence-item html_quote><dtml-if
     sequence-end><dtml-else><dtml-var "'\n'"></dtml-if></dtml-in></textarea>
    <dtml-elif "type=='selection'">
      <dtml-if "_.has_key(select_variable)">
!     <select name="<dtml-var id>">
        <dtml-in "_[select_variable]">
           <option
            <dtml-if "_['sequence-item']==getProperty(id)">SELECTED</dtml-if>
--- 66,90 ----
    <dtml-elif "type in ('float', 'date')">
    <input type="text" name="<dtml-var id>:<dtml-var type>" size="35"
     value="<dtml-var "getProperty(id)" html_quote>">
!   <dtml-elif "type in ['string','ustring']">
!   <input type="text" name="<dtml-var id>:<dtml-var encoding>:<dtml-var type>" size="35"
     value="<dtml-var "getProperty(id)" html_quote>">
    <dtml-elif "type=='boolean'">
    <input type="checkbox" name="<dtml-var id>:boolean" size="35"
     <dtml-if "getProperty(id)">CHECKED</dtml-if>>
!   <dtml-elif "type in ['tokens','utokens']">
!   <input type="text" name="<dtml-var id>:<dtml-var encoding>:<dtml-var type>" size="35"
     value="<dtml-in "getProperty(id)"><dtml-var sequence-item html_quote> </dtml-in>">
!   <dtml-elif "type in ['text','utext']">
!   <textarea name="<dtml-var id>:<dtml-var encoding>:<dtml-var type>" rows="6" cols="35"><dtml-var 
     "getProperty(id)" html_quote></textarea>
!   <dtml-elif "type in ['lines','ulines']">
!   <textarea name="<dtml-var id>:<dtml-var encoding>:<dtml-var type>" rows="6" cols="35"><dtml-in
    "getProperty(id)"><dtml-var sequence-item html_quote><dtml-if
     sequence-end><dtml-else><dtml-var "'\n'"></dtml-if></dtml-in></textarea>
    <dtml-elif "type=='selection'">
      <dtml-if "_.has_key(select_variable)">
!     <select name="<dtml-var id>:<dtml-var encoding>:text">
        <dtml-in "_[select_variable]">
           <option
            <dtml-if "_['sequence-item']==getProperty(id)">SELECTED</dtml-if>
***************
*** 80,86 ****
      </dtml-if>
    <dtml-elif "type=='multiple selection'">
      <dtml-if "_.has_key(select_variable)">
!     <select name="<dtml-var id>:list" multiple
        size="<dtml-var "_.min(7, _.len(_[select_variable]))">">
        <dtml-in "_[select_variable]">
           <option<dtml-if 
--- 96,102 ----
      </dtml-if>
    <dtml-elif "type=='multiple selection'">
      <dtml-if "_.has_key(select_variable)">
!     <select name="<dtml-var id>:<dtml-var encoding>:list" multiple
        size="<dtml-var "_.min(7, _.len(_[select_variable]))">">
        <dtml-in "_[select_variable]">
           <option<dtml-if 
***************
*** 152,161 ****
  <table>
  <tr>
    <th align="left" valign="top">Name</th>
!   <td align="left" valign="top"><input type="text" name="id" size="20"></td>
    <th align="left" valign="top">Type</th>
    <td align="left" valign="top">
!     <select name="type">
        <option>boolean</option>
        <option>date</option>
        <option>float</option>
--- 168,177 ----
  <table>
  <tr>
    <th align="left" valign="top">Name</th>
!   <td align="left" valign="top"><input type="text" name="id:<dtml-var encoding>:string" size="20"></td>
    <th align="left" valign="top">Type</th>
    <td align="left" valign="top">
!     <select name="type:<dtml-var encoding>:string">
        <option>boolean</option>
        <option>date</option>
        <option>float</option>
***************
*** 163,170 ****
--- 179,190 ----
        <option>lines</option>
        <option>long</option>
        <option SELECTED>string</option>
+       <option>ustring</option>
        <option>text</option>
        <option>tokens</option>
+       <option>utext</option>
+       <option>utokens</option>
+       <option>ulines</option>
        <option>selection</option>
        <option>multiple selection</option>
      </select>
***************
*** 173,179 ****
  <tr>
    <th align="left" valign="top">Value</th>
    <td colspan=2 align="left" valign="top">
!     <input type="text" name="value" size="30">
    </td>
    <td align="right" valign="top">
    <input type="submit" value=" Add ">
--- 193,199 ----
  <tr>
    <th align="left" valign="top">Value</th>
    <td colspan=2 align="left" valign="top">
!     <input type="text" name="value:<dtml-var encoding>:string" size="30">
    </td>
    <td align="right" valign="top">
    <input type="submit" value=" Add ">
***************
*** 186,188 ****
--- 206,209 ----
  </body>
  </html>
  
+ </dtml-let>
Index: external/Zope2/lib/python/Products/ZCatalog/Catalog.py
diff -c external/Zope2/lib/python/Products/ZCatalog/Catalog.py:1.1.1.2 external/Zope2/lib/python/Products/ZCatalog/Catalog.py:1.4
*** external/Zope2/lib/python/Products/ZCatalog/Catalog.py:1.1.1.2	Thu Jun 22 12:04:55 2000
--- external/Zope2/lib/python/Products/ZCatalog/Catalog.py	Fri Jun 30 08:38:21 2000
***************
*** 533,540 ****
                              append((k,LazyMap(self.__getitem__, intset)))
                  else:
                      for r in rs:
!                         append(sort_index._unindex[r],
!                                LazyMap(self.__getitem__,[r]))
  
          return used
  
--- 533,540 ----
                              append((k,LazyMap(self.__getitem__, intset)))
                  else:
                      for r in rs:
!                         append((sort_index._unindex[r],
!                                LazyMap(self.__getitem__,[r])))
  
          return used
  
Index: external/Zope2/lib/python/ZLogger/ZLogger.py
diff -c external/Zope2/lib/python/ZLogger/ZLogger.py:1.1.1.1 external/Zope2/lib/python/ZLogger/ZLogger.py:1.2
*** external/Zope2/lib/python/ZPublisher/Converters.py:1.1.1.1	Fri Apr 28 21:07:08 2000
--- external/Zope2/lib/python/ZPublisher/Converters.py	Tue Jul 11 15:00:30 2000
***************
*** 82,88 ****
  # attributions are listed in the accompanying credits file.
  # 
  ##############################################################################
! __version__='$Revision: 1.1.1.1 $'[11:-2]
  
  import regex
  from string import atoi, atol, atof, join, split, strip
--- 82,88 ----
  # attributions are listed in the accompanying credits file.
  # 
  ##############################################################################
! __version__='$Revision: 1.4 $'[11:-2]
  
  import regex
  from string import atoi, atol, atof, join, split, strip
***************
*** 159,164 ****
--- 159,202 ----
  def field2boolean(v):
      return v
  
+ 
+ class _unicode_converter:
+     def __call__(self,v):
+         # Convert a regular python string. This probably doesnt do what you want,
+         # whatever that might be. If you are getting exceptions below, you
+         # probably missed the encoding tag from a form field name. Use:
+         #       <input name="description:utf8:ustring" .....
+         # rather than
+         #       <input name="description:ustring" .....
+         if hasattr(v,'read'): v=v.read()
+         v = unicode(v)
+         return self.convert_unicode(v)
+ 
+     def convert_unicode(self,v):
+         raise NotImplementedError('convert_unicode')
+ 
+ 
+ class field2ustring(_unicode_converter):
+     def convert_unicode(self,v):
+         return v
+ field2ustring = field2ustring()
+ 
+ class field2utokens(_unicode_converter):
+     def convert_unicode(self,v):
+         return split(v)
+ field2utokens = field2utokens()
+ 
+ class field2utext(_unicode_converter):
+     def convert_unicode(self,v):
+         return unicode(field2text(v.encode('utf8')),'utf8')
+ field2utext = field2utext()
+ 
+ class field2ulines(_unicode_converter):
+     def convert_unicode(self,v):
+         return split(field2utext.convert_unicode(v),'\n')
+ field2ulines = field2ulines()
+ 
+ 
  type_converters = {
      'float':    field2float,
      'int':      field2int,
***************
*** 169,175 ****
      'tokens':   field2tokens,
      'lines':    field2lines,
      'text':     field2text,
!     'boolean':     field2boolean,
      }
  
  get_converter=type_converters.get
--- 207,219 ----
      'tokens':   field2tokens,
      'lines':    field2lines,
      'text':     field2text,
!     'boolean':    field2boolean,
!     
!     # u-prefixed converters are Unicode variants. utokens, ulines, and utext will be added soon
!     'ustring':  field2ustring,
!     'utokens':  field2utokens,
!     'ulines':   field2ulines,
!     'utext':    field2utext,
      }
  
  get_converter=type_converters.get
Index: external/Zope2/lib/python/ZPublisher/HTTPRequest.py
diff -c external/Zope2/lib/python/ZPublisher/HTTPRequest.py:1.1.1.2 external/Zope2/lib/python/ZPublisher/HTTPRequest.py:1.4
*** external/Zope2/lib/python/ZPublisher/HTTPRequest.py:1.1.1.2	Thu Jun 22 12:05:03 2000
--- external/Zope2/lib/python/ZPublisher/HTTPRequest.py	Fri Jul  7 13:06:46 2000
***************
*** 83,91 ****
  # 
  ##############################################################################
  
! __version__='$Revision: 1.1.1.2 $'[11:-2]
  
! import regex, sys, os, string, urllib
  from string import lower, atoi, rfind, split, strip, join, upper, find
  from BaseRequest import BaseRequest
  from HTTPResponse import HTTPResponse
--- 83,91 ----
  # 
  ##############################################################################
  
! __version__='$Revision: 1.4 $'[11:-2]
  
! import regex, sys, os, string, urllib, codecs
  from string import lower, atoi, rfind, split, strip, join, upper, find
  from BaseRequest import BaseRequest
  from HTTPResponse import HTTPResponse
***************
*** 379,384 ****
--- 379,385 ----
                          item=item.value
  
                  flags=0
+                 character_encoding = ''
  
                  # Loop through the different types and set
                  # the appropriate flags
***************
*** 420,425 ****
--- 421,428 ----
                              flags=flags|RECORDS
                          elif type_name == 'ignore_empty':
                              if not item: flags=flags|EMPTY
+                         elif has_codec(type_name):
+                             character_encoding = type_name
      
                          l=rfind(key,':')
                          if l < 0: break
***************
*** 441,447 ****
                      # defer conversion
                      if flags&CONVERTED:
                          try:
!                             item=converter(item)
                          except:
                              if (not item and not (flags&DEFAULT) and
                                  defaults.has_key(key)):
--- 444,460 ----
                      # defer conversion
                      if flags&CONVERTED:
                          try:
!                             if character_encoding:
!                                 # We have a string with a specified character encoding.
!                                 # This gets passed to the converter either as unicode, if it can
!                                 # handle it, or crunched back down to latin-1 if it can not.
!                                 item = unicode(item,character_encoding)
!                                 if hasattr(converter,'convert_unicode'):
!                                     item = converter.convert_unicode(item)
!                                 else:
!                                     item = converter(item.encode('latin1'))
!                             else:
!                                 item=converter(item)
                          except:
                              if (not item and not (flags&DEFAULT) and
                                  defaults.has_key(key)):
***************
*** 885,890 ****
--- 898,912 ----
                  [name,password] = split(
                      base64.decodestring(split(auth)[-1]), ':')
                  return name, password
+ 
+ 
+ def has_codec(x):
+     try:
+         codecs.lookup(x)
+     except LookupError:
+         return 0
+     else:
+         return 1
  
  
  
Index: external/Zope2/lib/python/ZPublisher/HTTPResponse.py
diff -c external/Zope2/lib/python/ZPublisher/HTTPResponse.py:1.1.1.2 external/Zope2/lib/python/ZPublisher/HTTPResponse.py:1.3
*** external/Zope2/lib/python/ZPublisher/HTTPResponse.py:1.1.1.2	Thu Jun 22 12:05:03 2000
--- external/Zope2/lib/python/ZPublisher/HTTPResponse.py	Tue Jul 11 12:15:16 2000
***************
*** 84,97 ****
  ##############################################################################
  '''CGI Response Output formatter
  
! $Id: HTTPResponse.py,v 1.1.1.2 2000/06/22 11:05:03 tdickenson Exp $'''
! __version__='$Revision: 1.1.1.2 $'[11:-2]
  
! import string, types, sys, regex
  from string import find, rfind, lower, upper, strip, split, join, translate
! from types import StringType, InstanceType
  from BaseResponse import BaseResponse
  
  nl2sp=string.maketrans('\n',' ')
  
  status_reasons={
--- 84,98 ----
  ##############################################################################
  '''CGI Response Output formatter
  
! $Id: HTTPResponse.py,v 1.3 2000/07/11 11:15:16 tdickenson Exp $'''
! __version__='$Revision: 1.3 $'[11:-2]
  
! import string, types, sys, regex, re
  from string import find, rfind, lower, upper, strip, split, join, translate
! from types import StringType, InstanceType, UnicodeType
  from BaseResponse import BaseResponse
  
+ 
  nl2sp=string.maketrans('\n',' ')
  
  status_reasons={
***************
*** 289,295 ****
          message instead of a generic HTML page.
          '''
          if not body: return self
!         
          if type(body) is types.TupleType and len(body) == 2:
              title,body=body
  
--- 290,296 ----
          message instead of a generic HTML page.
          '''
          if not body: return self
! 
          if type(body) is types.TupleType and len(body) == 2:
              title,body=body
  
***************
*** 297,303 ****
              if hasattr(body,'asHTML'):
                  body=body.asHTML()
  
!         body=str(body)
          l=len(body)
              
          if (find(body,'>')==l-1 and body[:1]=='<' and l < 200 and
--- 298,308 ----
              if hasattr(body,'asHTML'):
                  body=body.asHTML()
  
!         body=ustr(body)
! 
!         if type(body) is UnicodeType:
!             body = self._encode_unicode(body)
! 
          l=len(body)
              
          if (find(body,'>')==l-1 and body[:1]=='<' and l < 200 and
***************
*** 316,321 ****
--- 321,336 ----
          self.insertBase()
          return self
  
+     def _encode_unicode(self,body,charset_re=re.compile(r'text/[0-9a-z]+\s*;\s*charset=([-_0-9a-z]+)(?:(?:\s*;)|\Z)',re.IGNORECASE)):
+         # Try to encode the Unicode data as requested.
+         if self.headers.has_key('content-type'):
+             match = charset_re.match(self.headers['content-type'])
+             if match:
+                 encoding = match.group(1)
+                 return body.encode(encoding)
+         # Choose a default character encoding.
+         return body.encode('latin1','replace')
+ 
      def setBase(self,base):
          'Set the base URL for the returned document.'
          if base[-1:] != '/': base=base+'/'
***************
*** 327,333 ****
                                                  regex.casefold).search
                     ):
          if (self.headers.has_key('content-type') and
!             self.headers['content-type'] != 'text/html'): return
  
          if self.base:
              body=self.body
--- 342,348 ----
                                                  regex.casefold).search
                     ):
          if (self.headers.has_key('content-type') and
!              self.headers['content-type'][:9] != 'text/html'): return
  
          if self.base:
              body=self.body
***************
*** 616,622 ****
                  except: pass
  
          b=v
!         if isinstance(b,Exception): b=str(b)
          
          if fatal and t is SystemExit and v.code==0:
                  tb=self.setBody(
--- 631,641 ----
                  except: pass
  
          b=v
!         if isinstance(b,Exception):
!             try:
!                 b=str(b)
!             except:
!                 b = "object at %x" % id(b)
          
          if fatal and t is SystemExit and v.code==0:
                  tb=self.setBody(
***************
*** 626,632 ****
                       is_error=1)
          #elif 1: self.setBody(v)
  
!         elif type(b) is not types.StringType or tag_search(b) < 0:
              tb=self.setBody(
                  (str(t),
                  'Sorry, a Zope error occurred.<p>'+
--- 645,651 ----
                       is_error=1)
          #elif 1: self.setBody(v)
  
!         elif type(b) is not types.StringType and type(b) is not types.UnicodeType or tag_search(b) < 0:
              tb=self.setBody(
                  (str(t),
                  'Sorry, a Zope error occurred.<p>'+
***************
*** 685,691 ****
                      c='text/plain'
                  self.setHeader('content-type',c)
              else:
!                 isHTML = headers['content-type']=='text/html'
              if isHTML and end_of_header_search(self.body) < 0:
                  lhtml=html_search(body)
                  if lhtml >= 0:
--- 704,710 ----
                      c='text/plain'
                  self.setHeader('content-type',c)
              else:
!                 isHTML = headers['content-type'][:9]=='text/html'
              if isHTML and end_of_header_search(self.body) < 0:
                  lhtml=html_search(body)
                  if lhtml >= 0: