[OpenTAL-checkins] opental OpenPT/OpenPTBase.py OpenPT/ZOPT.py Ope...

[Fernando Lalo Martins]

CVSROOT:        /cvsroot/opental
Module name:    opental
Changes by:     Fernando Lalo Martins <address@hidden>  03/01/29 17:40:24

Modified files:
        OpenPT         : OpenPTBase.py ZOPT.py 
        OpenPT/www     : ptEdit.pt 
        PlacelessTranslationService: GettextMessageCatalog.py 
                                     PlacelessTranslationService.py 

Log message:
        OpenPT can now negotiate output encoding base on an user-supplied list 
and additional application-specific constraints.
        PlacelessTranslationService uses this hook to constrain to the set of 
encodings declared as compatible in the potfile header.

CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/opental/opental/OpenPT/OpenPTBase.py.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/opental/opental/OpenPT/ZOPT.py.diff?tr1=1.41&tr2=1.42&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/opental/opental/OpenPT/www/ptEdit.pt.diff?tr1=1.1&tr2=1.2&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/opental/opental/PlacelessTranslationService/GettextMessageCatalog.py.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/opental/opental/PlacelessTranslationService/PlacelessTranslationService.py.diff?tr1=1.8&tr2=1.9&r1=text&r2=text

Patches:
Index: opental/OpenPT/OpenPTBase.py
diff -u opental/OpenPT/OpenPTBase.py:1.3 opental/OpenPT/OpenPTBase.py:1.4
--- opental/OpenPT/OpenPTBase.py:1.3    Mon Jan 27 14:59:12 2003
+++ opental/OpenPT/OpenPTBase.py        Wed Jan 29 17:40:24 2003
@@ -15,18 +15,18 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307, USA
-__version__='$Revision: 1.3 $'[11:-2]
+__version__='$Revision: 1.4 $'[11:-2]
 
+# ugh, this import list needs to be cleaned up
 import OpenTAL, pax, PTmetal_handler, pax.paxtransform
 from pax.backwards_compatibility import *
 from ZPythonExpr import PythonExpr, SecureModuleImporter
 import cPickle as pickle
 #import pickle
-
 import AccessControl, Acquisition, sys
 from Acquisition import aq_base
 from Globals import DTMLFile, ImageFile, MessageDialog, package_home
-from zLOG import LOG, ERROR, INFO
+from zLOG import LOG, PROBLEM, ERROR, INFO
 from OFS.SimpleItem import SimpleItem
 from DateTime.DateTime import DateTime
 from string import join, strip, rstrip, split, replace, lower
@@ -47,6 +47,7 @@
 from ZODB.PersistentMapping import PersistentMapping
 from OFS.SimpleItem import SimpleItem
 import Globals
+from encodings.aliases import aliases as encoding_aliases
 from DocumentTemplate.DT_Util import TemplateDict, InstanceDict
 from AccessControl.DTML import RestrictedDTML
 class Rtd(RestrictedDTML, TemplateDict):
@@ -143,6 +144,69 @@
         if not self._securityManager.validate(accessed, container, name, 
value, *a):
             raise Unauthorized, name
 
+class EncodingNegotiator(Acquisition.Explicit):
+    security = AccessControl.ClassSecurityInfo()
+    security.declareObjectPublic()
+    fallback = 'utf_8' # always acceptable
+    forced = None
+
+    def log(self, message, severity=INFO):
+        LOG('EncodingNegotiator', severity, message)
+    
+    def __init__(self, preferred=None):
+        if preferred is None:
+            self.preferred = ('utf_8', 'utf_16')
+        else:
+            self.preferred = self.purify(preferred)
+        self.log('Initializing with %s' % self.preferred)
+        self.acceptable = sets.Set(self.preferred)
+        self.clients = []
+
+    def purify_name(self, name):
+        "find if encoding is an alias and return the pythonic name"
+        name = name.replace('-', '_').replace(' ', '_').lower()
+        return encoding_aliases.get(name, name)
+
+    def purify(self, encodings):
+        "find all encodings that are aliases and replace them by the pythonic 
name"
+        return [self.purify_name(name) for name in encodings]
+
+    security.declarePublic('restrict')
+    def restrict(self, client, encodings):
+        "restrict the set of acceptable encodings by intersecting it with 
<encodings>"
+        client_id = id(client)
+        if client_id not in self.clients:
+            self.clients.append(client_id)
+            encodings = self.purify(encodings)
+            self.log('restricting with %s due to %s' % (encodings, client))
+            self.acceptable.intersection_update(sets.Set(encodings))
+            self.log('now: %s' % self.acceptable)
+        else:
+            self.log('client %s already cached' % client)
+
+    security.declarePublic('force')
+    def force(self, encoding):
+        "force one encoding.  If it's called multiple times, bad luck, last 
one wins."
+        self.log('forcing %s' % encoding)
+        self.forced = encoding
+
+    def encode(self, text):
+        if self.forced is not None:
+            try:
+                return text.encode(self.forced), self.forced
+            except UnicodeError:
+                self.log('Was told to force %s encoding but it failed' % 
self.forced, PROBLEM)
+        for enc in self.preferred:
+            if enc in self.acceptable:
+                try:
+                    return text.encode(enc), enc
+                except UnicodeError:
+                    pass
+        self.log('negotiation failed, falling back to %s' % self.fallback)
+        return text.encode(self.fallback), self.fallback # should never fail!
+Globals.InitializeClass(EncodingNegotiator)
+
+
 # paxtransform handlers
 handler_registry = [OpenTAL.metal_handler, OpenTAL.tal_handler]
 # paxtransform handlers
@@ -195,6 +259,7 @@
     source_type = 'HTML'
     handlers = ('tal', 'metal')
     encoding = 'utf-8'
+    acceptable_encodings = ('utf_8', 'utf_16')
     _errors = ()
     _warnings = ()
     _text = '<opental />'
@@ -431,23 +496,31 @@
             context = self._context_class(bound_names)
             try:
                 context.encoding = self.default_zope_data_encoding
-            except:
+            except AttributeError:
                 context.encoding = self.encoding
             context.root_xmlns_map = root_ns_map
             tr_engine.initialize(context)
+            encoding_neg = EncodingNegotiator(self.acceptable_encodings)
+            # we need the acquisition wrapper or AccessControl will squeal
+            self.REQUEST.pt_output_encoding = encoding_neg.__of__(self)
+            # now run the template
             #result = pax.XML(tr_engine.transform(self.paxtree(), context),
             #                     root_ns_map, self.encoding)
             result = tr_engine.transform(self.paxtree(), context).text
+            result, encoding = encoding_neg.encode(result)
+            # now this is hairy: how to get the official (IANA) http name
+            # of the encoding based on the python name?
+            # the hack below catches most usual cases, but far from all.
+            encoding = encoding.replace('_', '-')
+            if encoding.startswith('latin_'):
+                encoding = 'latin' + encoding[6:]
             if keyset is not None:
                 # Store the result in the cache.
                 self.ZCacheable_set(result, keywords=keyset)
-            try:
-                self.REQUEST.RESPONSE.setHeader('content-type',
-                                                '%s; charset=%s' % (
+            self.REQUEST.RESPONSE.setHeader('content-type', '%s; charset=%s' % 
(
                                                  self.content_type,
-                                                 context.encoding))
-            except AttributeError:
-                pass
+                                                 encoding))
+            print >>sys.stderr, self.REQUEST.RESPONSE.headers
             return result
         finally:
             security.removeContext(self)
Index: opental/OpenPT/ZOPT.py
diff -u opental/OpenPT/ZOPT.py:1.41 opental/OpenPT/ZOPT.py:1.42
--- opental/OpenPT/ZOPT.py:1.41 Mon Jan 27 00:11:10 2003
+++ opental/OpenPT/ZOPT.py      Wed Jan 29 17:40:24 2003
@@ -15,7 +15,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307, USA
-__version__='$Revision: 1.41 $'[11:-2]
+__version__='$Revision: 1.42 $'[11:-2]
 
 # FIXME: determine what exactly has to be imported.
 # some imports probably aren't needed by Base anymore.
@@ -40,6 +40,7 @@
     _properties=({'id':'title', 'type': 'string', 'mode': 'w'},
                  {'id':'content_type', 'type':'string', 'mode': 'w'},
                  {'id':'encoding', 'type':'string', 'mode': 'w'},
+                 {'id':'acceptable_encodings', 'type':'tokens', 'mode': 'w'},
                  {'id':'source_type', 'type':'selection', 'mode': 'w',
                   'select_variable': 'pt_parser_list'},
                  {'id':'handlers', 'type':'multiple selection', 'mode': 'w',
@@ -73,7 +74,7 @@
       'pt_editAction', 'pt_setTitle', 'pt_edit', 'pt_expand',
       'pt_upload', 'pt_changePrefs')
     def pt_editAction(self, REQUEST, title, text, content_type,
-                      source_type, encoding, handlers=()):
+                      source_type, encoding, handlers=(), 
acceptable_encodings=()):
         """Change the title and document."""
         if self.wl_isLocked():
             raise ResourceLockedError, "File is locked via WebDAV"
@@ -85,6 +86,8 @@
             self.encoding = encoding
         if self.handlers != handlers:
             self.handlers = handlers
+        if self.acceptable_encodings != acceptable_encodings:
+            self.acceptable_encodings = acceptable_encodings
         self.pt_setTitle(title)
         self.pt_edit(text)
         REQUEST.set('text', self.read()) # May not equal 'text'!
Index: opental/OpenPT/www/ptEdit.pt
diff -u opental/OpenPT/www/ptEdit.pt:1.1 opental/OpenPT/www/ptEdit.pt:1.2
--- opental/OpenPT/www/ptEdit.pt:1.1    Thu Jan 16 17:57:59 2003
+++ opental/OpenPT/www/ptEdit.pt        Wed Jan 29 17:40:24 2003
@@ -2,6 +2,8 @@
 <h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing"
     tal:replace="structure here/manage_tabs">Tabs</h2>
 
+<tal:block content="python:request.pt_output_encoding.force(here.encoding)" />
+
 <tal:block define="global body request/other/text | request/form/text
 | here/read" /> 
 <form action="" method="post" tal:attributes="action request/URL1">
@@ -19,7 +21,8 @@
     </td>
     <td align="right" valign="middle">
     <div class="form-optional">
-    Encoding
+    Source encoding
+    <div class="form-help">(be extremely careful when changing)</div>
     </div>
     </td>
     <td align="right" valign="middle">
@@ -49,6 +52,23 @@
        tal:attributes="selected
        python:test(parser==here.source_type,'selected',nothing)">XML</option>
     </select>
+    </td>
+  </tr>
+
+  <tr>
+    <td align="left" valign="middle">
+    <div class="form-optional">
+    Valid output encodings
+    </div>
+    <div class="form-help">
+    (in order of preference)
+    </div>
+    </td>
+    <td align="left" valign="middle" colspan="3">
+    <input type="text" name="acceptable_encodings:tokens" size="80" 
style="width: 100%"
+           tal:define="value request/acceptable_encodings | 
here/acceptable_encodings"
+           tal:attributes="value python:' '.join(value)" />
+    </td>
   </tr>
 
   <tr tal:define="errors here/pt_errors" tal:condition="errors">
Index: opental/PlacelessTranslationService/GettextMessageCatalog.py
diff -u opental/PlacelessTranslationService/GettextMessageCatalog.py:1.2 
opental/PlacelessTranslationService/GettextMessageCatalog.py:1.3
--- opental/PlacelessTranslationService/GettextMessageCatalog.py:1.2    Thu Jan 
16 18:15:43 2003
+++ opental/PlacelessTranslationService/GettextMessageCatalog.py        Wed Jan 
29 17:40:24 2003
@@ -17,7 +17,7 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307, USA
 """A simple implementation of a Message Catalog. 
 
-$Id: GettextMessageCatalog.py,v 1.2 2003/01/16 23:15:43 lalo Exp $
+$Id: GettextMessageCatalog.py,v 1.3 2003/01/29 22:40:24 lalo Exp $
 """
 
 from gettext import GNUTranslations
@@ -57,6 +57,7 @@
             self._domain = tro._info.get('domain', None)
             if self._language is None or self._domain is None:
                 raise ValueError, 'potfile has no metadata'
+            self.preferred_encodings = tro._info.get('preferred-encodings', 
'').split()
             self.__translation_object = tro
             missing = self._path_to_file[:-1] + 'issing'
             if os.access(missing, os.W_OK):
Index: opental/PlacelessTranslationService/PlacelessTranslationService.py
diff -u opental/PlacelessTranslationService/PlacelessTranslationService.py:1.8 
opental/PlacelessTranslationService/PlacelessTranslationService.py:1.9
--- opental/PlacelessTranslationService/PlacelessTranslationService.py:1.8      
Wed Jan 29 06:36:34 2003
+++ opental/PlacelessTranslationService/PlacelessTranslationService.py  Wed Jan 
29 17:40:24 2003
@@ -17,7 +17,7 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307, USA
 """Placeless Translation Service for providing I18n to file-based code.
 
-$Id: PlacelessTranslationService.py,v 1.8 2003/01/29 11:36:34 magnusheino Exp $
+$Id: PlacelessTranslationService.py,v 1.9 2003/01/29 22:40:24 lalo Exp $
 """
 
 import re, zLOG
@@ -133,10 +133,13 @@
             catalog = self._data[name]
             try:
                 text = catalog.getMessage(msgid, default)
-                break
             except KeyError:
                 # it's not in this catalog, try the next one
-                pass
+                continue
+            # found! negotiate output encodings now
+            if hasattr(context, 'pt_output_encoding') and 
catalog.preferred_encodings:
+                context.pt_output_encoding.restrict(catalog, 
catalog.preferred_encodings)
+            break
         else:
             # Did the fallback fail?  Sigh, use the default.
             # OpenTAL provides a default text.