ZpgCatalog-0.2/0040755000175000001640000000000007705454507013051 5ustar fafhrddatacomZpgCatalog-0.2/www/0040755000175000001640000000000007667307304013674 5ustar fafhrddatacomZpgCatalog-0.2/www/ZpgCatalog.gif0100644000175000001640000000026506733715167016421 0ustar fafhrddatacomGIF89aƄ!,@bPHBmFOE0E"h"b[JI,Ljq9ieA8XL!FB/TRMxz/R0›a:#JOP |zT6V1;ZpgCatalog-0.2/www/index.gif0100644000175000001640000000030707305214160015451 0ustar fafhrddatacomGIF89aG^jn 5q[! ,D'diK vn ɦX0‘ s̤h)>}"B2Iߧ;ZpgCatalog-0.2/dtml/0040755000175000001640000000000007704726673014016 5ustar fafhrddatacomZpgCatalog-0.2/dtml/addZpgCatalog.dtml0100755000175000001640000000300307667301406017367 0ustar fafhrddatacom
Id
Title
Connection Id

There are no SQL database connections.

ZpgCatalog-0.2/ZpgCatalog.py0100664000175000001640000001326307702472547015463 0ustar fafhrddatacom############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """ZCatalog product""" import Globals from cStringIO import StringIO import urllib, os, sys, time, types from Globals import DTMLFile, MessageDialog, InitializeClass from OFS.ObjectManager import ObjectManager from DateTime import DateTime from AccessControl import getSecurityManager from AccessControl.Permissions import \ manage_zcatalog_entries, manage_zcatalog_indexes, search_zcatalog from zLOG import LOG, ERROR from Shared.DC.ZRDB import RDB from Shared.DC.ZRDB.Results import Results from Products.ZCatalog.ZCatalog import ZCatalog from pgCatalog import pgCatalog from ZpgCatalogIndexes import ZpgCatalogIndexes from Indexes.PluggableIndex import DbPluggableIndexInterface manage_addZpgCatalogForm=DTMLFile('dtml/addZpgCatalog',globals()) def manage_addZpgCatalog(self, id, title, connection_id, REQUEST=None): """Add a ZpgCatalog object """ id=str(id) title=str(title) c=ZpgCatalog(id, connection_id, title, self) self._setObject(id, c) if REQUEST is not None: return self.manage_main(self, REQUEST, update_menu=1) class ZpgCatalog(ZCatalog): """ """ meta_type = "ZpgCatalog" Indexes = ZpgCatalogIndexes() def __init__(self, id, connection_id, title='', container=None): self.connection_id = connection_id if container is not None: self=self.__of__(container) self.id=id self.title=title self.threshold = 10000 self._v_total = 0 self._catalog = pgCatalog(db=self) def __len__(self): return len(self._catalog) def execute(self, query): try: dbc = getattr(self, self.connection_id) except AttributeError: raise AttributeError, ( "The database connection %s cannot be found."% ( self.connection_id)) try: DB__ = dbc() except: raise 'Database Error', ( '%s is not connected to a database' % self.id) res=DB__.query(query) if type(res) is type(''): f=StringIO() f.write(res) f.seek(0) return RDB.File(f) else: return Results(res) def refreshCatalog(self, clear=0): """ re-index everything we can find """ cat = self._catalog paths = cat.paths.values() if clear: paths = tuple(paths) cat.clear() for p in paths: obj = self.resolve_path(p) if not obj: obj = self.resolve_url(p, self.REQUEST) if obj is not None: self.catalog_object(obj, p) def reindexIndex(self, name, REQUEST): paths = self._catalog.uids.keys() for p in paths: obj = self.resolve_path(p) if not obj: obj = self.resolve_url(p, REQUEST) if obj is not None: self.catalog_object(obj, p, idxs=[name]) def getpath(self, rid): """Return the path to a cataloged object given a 'data_record_id_' """ return self._catalog.paths[rid] def getrid(self, path, default=None): """Return 'data_record_id_' the to a cataloged object given a 'path' """ return self._catalog.uids.get(path, default) def manage_convertBTrees(self, threshold=200): """Convert the catalog's data structures to use BTrees package""" return 'Finished conversion' def manage_convertIndex(self, ids, REQUEST=None, RESPONSE=None, URL1=None): """convert old-style indexes to new-style indexes""" RESPONSE.redirect( URL1 + '/manage_main?manage_tabs_message=Nothing%20to%20do') # # Indexing methods # def addIndex(self, name, type,extra=None): # Convert the type by finding an appropriate product which supports # this interface by that name. Bleah products = ObjectManager.all_meta_types(self, interfaces=( DbPluggableIndexInterface,)) p = None for prod in products: if prod['name'] == type: p = prod break if p is None: raise ValueError, "Index of type %s not found" % type base = p['instance'] if base is None: raise ValueError, "Index type %s does not support addIndex" % type # This code is *really* lame but every index type has its own # function signature *sigh* and there is no common way to pass # additional parameters to the constructor. The suggested way # for new index types is to use an "extra" record. if 'extra' in base.__init__.func_code.co_varnames: index = base(name, extra=extra, caller=self) elif 'caller' in base.__init__.func_code.co_varnames: index = base(name, caller=self) else: index = base(name) self._catalog.addIndex(name,index) def manage_beforeDelete(self, item, container): try: self._catalog._removeDbTable() except: pass ZCatalog.manage_beforeDelete(self, item, container) InitializeClass(ZpgCatalog) ZpgCatalog-0.2/Indexes/0040755000175000001640000000000007705454474014453 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/www/0040755000175000001640000000000007667306751015300 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/www/index.gif0100644000175000001640000000030707305214160017050 0ustar fafhrddatacomGIF89aG^jn 5q[! ,D'diK vn ɦX0‘ s̤h)>}"B2Iߧ;ZpgCatalog-0.2/Indexes/dtml/0040755000175000001640000000000007704724153015404 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/dtml/addPathIndex.dtml0100644000175000001640000000242207305214155020611 0ustar fafhrddatacom

A PathIndex indexes the physical path of all objects inside a catalog. It allows you to search for objects beginning or containing a special path component or a set of path component. A path component is defined as /<component1>/<component2>/..../<object_id> . Note: the object_id will not be indexed by the PathIndex.

Id
Type
PathIndex
ZpgCatalog-0.2/Indexes/dtml/addDateIndex.dtml0100644000175000001640000000172107477571643020615 0ustar fafhrddatacom

A DateIndex indexes DateTime attributes.

Id
Type
DateIndex
ZpgCatalog-0.2/Indexes/dtml/addTextIndex.dtml0100644000175000001640000000172707704725757020672 0ustar fafhrddatacom

Text Index full text search engine

Id
Type
Field Index
ZpgCatalog-0.2/Indexes/dtml/addStrFieldIndex.dtml0100644000175000001640000000215507674251070021442 0ustar fafhrddatacom

String Field Indexes treat the value of an objects attributes atomically, and can be used, for example, to track only a certain subset of object values, such as 'meta_type'.

Id
Type
Field Index
ZpgCatalog-0.2/Indexes/dtml/addFieldIndex.dtml0100644000175000001640000000214307674251050020744 0ustar fafhrddatacom

Field Indexes treat the value of an objects attributes atomically, and can be used, for example, to track only a certain subset of object values, such as 'meta_type'.

Id
Type
Field Index
ZpgCatalog-0.2/Indexes/dtml/manageRealDateIndex.dtml0100644000175000001640000000022207477571643022114 0ustar fafhrddatacom

Nothing to manage at this time.

ZpgCatalog-0.2/Indexes/dtml/addIntFieldIndex.dtml0100644000175000001640000000215607674251060021424 0ustar fafhrddatacom

Integer Field Indexes treat the value of an objects attributes atomically, and can be used, for example, to track only a certain subset of object values, such as 'meta_type'.

Id
Type
Field Index
ZpgCatalog-0.2/Indexes/dtml/manageFieldIndex.dtml0100644000175000001640000000022207305214154021433 0ustar fafhrddatacom

Nothing to manage at this time.

ZpgCatalog-0.2/Indexes/dtml/managePathIndex.dtml0100644000175000001640000000022207305214155021305 0ustar fafhrddatacom

Nothing to manage at this time.

ZpgCatalog-0.2/Indexes/dtml/manageTextIndexEx.dtml0100644000175000001640000000022207477571643021654 0ustar fafhrddatacom

Nothing to manage at this time.

ZpgCatalog-0.2/Indexes/dtml/manageDateIndex.dtml0100644000175000001640000000022207477571643021310 0ustar fafhrddatacom

Nothing to manage at this time.

ZpgCatalog-0.2/Indexes/dtml/manageTextIndex.dtml0100644000175000001640000000022207477571643021357 0ustar fafhrddatacom

Nothing to manage at this time.

ZpgCatalog-0.2/Indexes/dtml/addTextIndexEx.dtml0100644000175000001640000000173307704725776021165 0ustar fafhrddatacom

Text Index full text search engine

Id
Type
Field Index
ZpgCatalog-0.2/Indexes/dtml/addRealDateIndex.dtml0100644000175000001640000000173107702520362021401 0ustar fafhrddatacom

A DateIndex indexes DateTime attributes.

Id
Type
DateIndex
ZpgCatalog-0.2/Indexes/dtml/addKeywordIndex.dtml0100644000175000001640000000214407674050315021347 0ustar fafhrddatacom

Field Indexes treat the value of an objects attributes atomically, and can be used, for example, to track only a certain subset of object values, such as 'meta_type'.

Id
Type
Field Index
ZpgCatalog-0.2/Indexes/help/0040755000175000001640000000000007674246167015406 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/help/PathIndex_searchResults.stx0100644000175000001640000000167707305721562022735 0ustar fafhrddatacomZCatalog - searchResults: specifying parameters for a search query The searchResults() method of the ZCatalog accepts parameters that define a query to be made on that catalog. A query can either be passed as keyword argument to searchResults(), as a mapping, or as part of a Zope REQUEST object, typically from HTML forms. The index of the catalog to query is either the name of the keyword argument, a key in a mapping, or an attribute of a record object. Attributes of record objects 'query' -- either a sequence of objects or a single value to be passed as query to the index (mandatory) 'operator' -- specifies the combination of search results when query is a sequence of values. (optional, default: 'or'). Allowed values: 'and', 'or' 'level' -- only applies to Path Index. Specifies the directory level to start searching. (optional, default: 0) ZpgCatalog-0.2/Indexes/DateIndex.py0100644000175000001640000000507307705242465016667 0ustar fafhrddatacom############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """$Id: DateIndex.py,v 1.7 2002/08/14 22:19:27 mj Exp $ """ from DateTime.DateTime import DateTime from types import StringType, FloatType, IntType from Globals import DTMLFile import time _marker = [] import PluggableIndex from FieldIndex import FieldIndex class DateIndex(FieldIndex): """ Index for Dates """ __implements__ = (PluggableIndex.DbPluggableIndexInterface,) meta_type = 'DateIndex' query_options = ['query', 'range'] manage = manage_main = DTMLFile( 'dtml/manageDateIndex', globals() ) manage_main._setName( 'manage_main' ) manage_options = ( { 'label' : 'Settings', 'action' : 'manage_main' }, ) _db_type = 'integer' def _prepare_datum(self, value, default=_marker): """Convert Date/Time value to our internal representation""" if isinstance( value, DateTime ): t_tup = value.toZone('UTC').parts() elif type( value ) in (FloatType, IntType): t_tup = time.gmtime( value ) elif type( value ) is StringType: t_obj = DateTime( value ).toZone('UTC') t_tup = t_obj.parts() else: return default yr = t_tup[0] mo = t_tup[1] dy = t_tup[2] hr = t_tup[3] mn = t_tup[4] t_val = ( ( ( ( yr * 12 + mo ) * 31 + dy ) * 24 + hr ) * 60 + mn ) try: # t_val must be IntType, not LongType return int(t_val) except OverflowError: raise OverflowError, ( "%s is not within the range of indexable dates (index: %s)" % (value, self.id)) manage_addDateIndexForm = DTMLFile( 'dtml/addDateIndex', globals() ) def manage_addDateIndex( self, id, REQUEST=None, RESPONSE=None, URL3=None): """Add a Date index""" return self.manage_addIndex(id, 'DateIndex', extra=None, \ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3) ZpgCatalog-0.2/Indexes/TextIndexEx.py0100644000175000001640000001203307705237076017227 0ustar fafhrddatacom############################################################################# # # Copyright (c) 2003 Nikolay Kim # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## import traceback from types import InstanceType from converters.html import Converter from PyQueryParser import Parser from Globals import DTMLFile, get_request from Products.PluginIndexes.common.util import parseIndexRequest import PluggableIndex from FieldIndex import FieldIndex from Products.ZpgCatalog.utils import sql_quote__ _marker = None langs = {'ru':'russian', 'en': 'english', 'tr':'english', 'kk':'russian'} parser = Parser() converter = Converter() class TextIndexEx(FieldIndex): """Field Indexes""" __implements__ = (PluggableIndex.DbPluggableIndexInterface,) meta_type="TextIndexEx" manage_options= ( {'label': 'Settings', 'action': 'manage_main'}, ) query_options = ["query"] index_html = DTMLFile('dtml/index', globals()) manage_workspace = DTMLFile('dtml/manageTextIndexEx', globals()) _db_type = 'tsvector' operators = [] useOperator = '' keyForDocument = 1 dict = 'default_unicode' language = 'russian' def _apply_index(self, request, cid='', type=type, None=None): """ """ record = parseIndexRequest(request, self.id, self.query_options) if record.keys==None: return '' id = self.getId() name = self.db_id() query = '' opr = None # experimental code for specifing the operator operator = record.get('operator',self.useOperator) if not operator in self.operators : raise RuntimeError,"operator not valid: %s" % escape(operator) if record.get('usage',None): # see if any usage params are sent to field opr = record.usage.lower().split(':') opr, opr_args=opr[0], opr[1:] keys = record.keys def cr_str(node): tp = node.getType() val = node.getValue() if tp == 'WordNode': return val elif tp == 'AndNode': return '(%s&%s)'%(cr_str(val[0]), cr_str(val[1])) elif tp == 'OrNode': return '(%s|%s)'%(cr_str(val[0]), cr_str(val[1])) elif tp == 'NotNode': return '(!%s)'%cr_str(val[0]) else: return str(val) ptree = parser(keys[0]) s = sql_quote__(str(cr_str(ptree))) request = get_request() try: # iternal use (just for me) lang = request.user_lang except: lang = request.PARENTS[0].Localizer.get_selected_language() ts = "to_tsquery_ex('%s', %s, '%s')"%(self.dict, s, langs[lang]) request.__dict__['_text_idx_%s'%id] = ts query = "%s @@ %s "%(name, ts) return query def index_object(self, obj, threshold=None): """ index and object 'obj' with integer id 'documentId'""" # First we need to see if there's anything interesting to look at datum = self._get_object_datum(obj) if datum is None: datum = 'null' else: try: request = get_request() try: # iternal use (just for me) lang = request.user_lang except: lang = request.PARENTS[0].Localizer.get_selected_language() try: datum = converter(datum) datum = unicode(datum, 'utf-8', 'ignore').encode('utf-8') except: print 'converting error' datum = '' try: datum = self._prepare_datum(datum, langs[lang]) except KeyError, e: datum = 'null' except: traceback.print_exc() raise return self.db_id(), datum def _prepare_datum(self, datum, lang='russian'): return "to_tsvector_ex('%s', %s, '%s')" % (self.dict, sql_quote__(str(datum)), lang) def db_index(self): return 'USING gist(%s)'%self.db_id() def db_sort(self): id = self.db_id() ts = get_request().__dict__['_text_idx_%s'%self.getId()] return "rank(%s, %s)"%(id, ts) manage_addTextIndexExForm = DTMLFile('dtml/addTextIndexEx', globals()) def manage_addTextIndexEx(self, id, REQUEST=None, RESPONSE=None, URL3=None): """Add a text index""" return self.manage_addIndex(id, 'TextIndexEx', extra=None, \ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3) ZpgCatalog-0.2/Indexes/PluggableIndex.py0100644000175000001640000000466507667316404017724 0ustar fafhrddatacom############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """Pluggable Index Base Class """ __version__='$Revision: 1.7 $'[11:-2] import Interface class DbPluggableIndexInterface(Interface.Base): def getId(): """Return Id of index.""" def getEntryForObject(documentId, default=None): """Get all information contained for 'documentId'.""" def index_object(obj, threshold=None): """Index an object. 'obj' is the object to be indexed. 'threshold' is the number of words to process between committing subtransactions. If None, subtransactions are disabled. """ # XXX TextIndex does not implement uniqueValues(). def uniqueValues(name=None, withLengths=0): """Returns the unique values for name. If 'withLengths' is true, returns a sequence of tuples of (value, length). """ def _apply_index(request, cid=''): """Apply the index to query parameters given in 'request'. The argument should be a mapping object. If the request does not contain the needed parametrs, then None is returned. If the request contains a parameter with the name of the column + "_usage", it is sniffed for information on how to handle applying the index. (Note: this style or parameters is deprecated) If the request contains a parameter with the name of the column and this parameter is either a Record or a class instance then it is assumed that the parameters of this index are passed as attribute (Note: this is the recommended way to pass parameters since Zope 2.4) Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ ZpgCatalog-0.2/Indexes/FieldIndex.py0100644000175000001640000001201707705236225017026 0ustar fafhrddatacom############################################################################# # # Copyright (c) 2003 Nikolay Kim # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## import traceback from Globals import Persistent, DTMLFile from Acquisition import Implicit from zLOG import LOG, ERROR from types import StringType, ListType, IntType, TupleType from OFS.SimpleItem import SimpleItem from Products.PluginIndexes.common.util import parseIndexRequest from Products.PluginIndexes.common import safe_callable import PluggableIndex _marker = None class FieldIndex(Persistent, Implicit, SimpleItem): """Field Indexes""" __implements__ = (PluggableIndex.DbPluggableIndexInterface,) meta_type="FieldIndex" manage_options= ( {'label': 'Settings', 'action': 'manage_main', 'help': ('FieldIndex','FieldIndex_Settings.stx')}, ) query_options = ["query","range"] index_html = DTMLFile('dtml/index', globals()) manage_workspace = DTMLFile('dtml/manageFieldIndex', globals()) _db_type = 'int4' operators = ['or','and'] useOperator = 'or' keyForDocument = 1 def __init__(self, id, ignore_ex=None, call_methods=None): """Create an unindex""" self.id = id self._db_id = str(hash(id)).replace('-', '_') self.ignore_ex=ignore_ex # currently unimplimented self.call_methods=call_methods self.operators = ['or','and'] self.useOperator = 'or' def index_object(self, obj, threshold=None): """ index and object 'obj' with integer id 'documentId'""" # First we need to see if there's anything interesting to look at datum = self._get_object_datum(obj) if datum is None: datum = 'null' else: try: datum = self._prepare_datum(datum) except: traceback.print_exc() raise return self.db_id(), datum def _apply_index(self, request, cid='', type=type, None=None): """ """ record = parseIndexRequest(request, self.id, self.query_options) if record.keys==None: return '' name = self.db_id() query = '' opr = None # experimental code for specifing the operator operator = record.get('operator',self.useOperator) if not operator in self.operators : raise RuntimeError,"operator not valid: %s" % escape(operator) # Range parameter range_parm = record.get('range',None) if range_parm: opr = "range" opr_args = [] #print range_parm if range_parm.find("min")>-1: opr_args.append("min") if range_parm.find("max")>-1: opr_args.append("max") if record.get('usage',None): # see if any usage params are sent to field opr = record.usage.lower().split(':') opr, opr_args=opr[0], opr[1:] #print opr, record.keys , record.__dict__ if opr=="range": # range search if 'min' in opr_args: lo = min(record.keys) else: lo = None if 'max' in opr_args: hi = max(record.keys) else: hi = None #print lo,hi if lo is not None: query += '%s > %s and '%(name, self._prepare_datum(lo)) if hi is not None: query += '%s < %s and '%(name, self._prepare_datum(hi)) return query[:-4] else: # not a range search keys = record.keys if len(keys) > 1: s = '' for key in keys: s += '%s,'%(self._prepare_datum(key)) query += '%s in (%s) '%(name, s[:-1]) else: query += '%s=%s '%(name, self._prepare_datum(keys[0])) return query def _prepare_datum(self, datum): return hash(datum) def _get_object_datum(self,obj): try: datum = getattr(obj, self.id) if safe_callable(datum): datum = datum() except AttributeError: datum = _marker return datum def db_type(self): return self._db_type def db_index(self): return '(idx_%s)'%self._db_id def db_id(self): return 'idx_%s'%self._db_id db_sort = db_id manage_addFieldIndexForm = DTMLFile('dtml/addFieldIndex', globals()) def manage_addFieldIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None): """Add a field index""" return self.manage_addIndex(id, 'FieldIndex', extra=None, \ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3) ZpgCatalog-0.2/Indexes/RealDateIndex.py0100644000175000001640000000336507702521542017466 0ustar fafhrddatacom############################################################################# # # Copyright (c) 2003 Nikolay Kim # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## from Globals import DTMLFile from DateTime.DateTime import DateTime import PluggableIndex from FieldIndex import FieldIndex from Products.ZpgCatalog.utils import sql_quote__ class RealDateIndex(FieldIndex): """Real Date Indexes""" __implements__ = (PluggableIndex.DbPluggableIndexInterface,) meta_type="RealDateIndex" manage_options= ( {'label': 'Settings', 'action': 'manage_main', 'help': ('RealDateIndex','RealDateIndex_Settings.stx')}, ) query_options = ["query","range"] index_html = DTMLFile('dtml/index', globals()) manage_workspace = DTMLFile('dtml/manageRealDateIndex', globals()) _db_type = 'timestamp' def _prepare_datum(self, value, default='null'): """ """ if not isinstance( value, DateTime ): value = DateTime(str(value)) return sql_quote__(str(value)) manage_addRealDateIndexForm = DTMLFile('dtml/addRealDateIndex', globals()) def manage_addRealDateIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None): """Add a field index""" return self.manage_addIndex(id, 'RealDateIndex', extra=None, \ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3) ZpgCatalog-0.2/Indexes/PyQueryParser/0040755000175000001640000000000007705454470017242 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/PyQueryParser/yacc.py0100644000175000001640000017612507504072601020531 0ustar fafhrddatacom#----------------------------------------------------------------------------- # ply: yacc.py # # Author: David M. Beazley (beazley@cs.uchicago.edu) # Department of Computer Science # University of Chicago # Chicago, IL 60637 # # Copyright (C) 2001, David M. Beazley # # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # See the file COPYING for a complete copy of the LGPL. # # # This implements an LR parser that is constructed from grammar rules defined # as Python functions. Roughly speaking, this module is a cross between # John Aycock's Spark system and the GNU bison utility. # # Disclaimer: This is a work in progress. SLR parsing seems to work fairly # well and there is extensive error checking. LALR(1) is in progress. The # rest of this file is a bit of a mess. Please pardon the dust. # # The current implementation is only somewhat object-oriented. The # LR parser itself is defined in terms of an object (which allows multiple # parsers to co-exist). However, most of the variables used during table # construction are defined in terms of global variables. Users shouldn't # notice unless they are trying to define multiple parsers at the same # time using threads (in which case they should have their head examined). #----------------------------------------------------------------------------- __version__ = "1.1" #----------------------------------------------------------------------------- # === User configurable parameters === # # Change these to modify the default behavior of yacc (if you wish) #----------------------------------------------------------------------------- yaccdebug = 1 # Debugging mode. If set, yacc generates a # a 'parser.out' file in the current directory debug_file = 'parser.out' # Default name of the debugging file tab_module = 'parsetab' # Default name of the table module default_lr = 'SLR' # Default LR table generation method error_count = 3 # Number of symbols that must be shifted to leave recovery mode import re, types, sys, cStringIO, md5, os.path # Exception raised for yacc-related errors class YaccError(Exception): pass #----------------------------------------------------------------------------- # === LR Parsing Engine === # # The following classes are used for the LR parser itself. These are not # used during table construction and are independent of the actual LR # table generation algorithm #----------------------------------------------------------------------------- # This class is used to hold non-terminal grammar symbols during parsing. # It normally has the following attributes set: # .type = Grammar symbol type # .value = Symbol value # .lineno = Starting line number # .endlineno = Ending line number (optional, set automatically) class YaccSymbol: def __str__(self): return self.type def __repr__(self): return str(self) # This class is a wrapper around the objects actually passed to each # grammar rule. Index lookup and assignment actually assign the # .value attribute of the underlying YaccSymbol object. # The lineno() method returns the line number of a given # item (or 0 if not defined). The linespan() method returns # a tuple of (startline,endline) representing the range of lines # for a symbol. class YaccSlice: def __init__(self,s): self.slice = s self.pbstack = [] def __getitem__(self,n): return self.slice[n].value def __setitem__(self,n,v): self.slice[n].value = v def lineno(self,n): return getattr(self.slice[n],"lineno",0) def linespan(self,n): startline = getattr(self.slice[n],"lineno",0) endline = getattr(self.slice[n],"endlineno",startline) return startline,endline def pushback(self,n): if n <= 0: raise ValueError, "Expected a positive value" if n > (len(self.slice)-1): raise ValueError, "Can't push %d tokens. Only %d are available." % (n,len(self.slice)-1) for i in range(0,n): self.pbstack.append(self.slice[-i-1]) # The LR Parsing engine. This is defined as a class so that multiple parsers # can exist in the same process. A user never instantiates this directly. # Instead, the global yacc() function should be used to create a suitable Parser # object. class Parser: def __init__(self,magic=None): # This is a hack to keep users from trying to instantiate a Parser # object directly. if magic != "xyzzy": raise YaccError, "Can't instantiate Parser. Use yacc() instead." # Reset internal state self.productions = None # List of productions self.errorfunc = None # Error handling function self.action = { } # LR Action table self.goto = { } # LR goto table self.require = { } # Attribute require table self.method = "Unknown LR" # Table construction method used def errok(self): self.errorcount = 0 def restart(self): del self.statestack[:] del self.symstack[:] sym = YaccSymbol() sym.type = '$' self.symstack.append(sym) self.statestack.append(0) self.recover() def parse(self,input=None,lexer=None,debug=0): lookahead = None # Current lookahead symbol lookaheadstack = [ ] # Stack of lookahead symbols actions = self.action # Local reference to action table goto = self.goto # Local reference to goto table prod = self.productions # Local reference to production list pslice = YaccSlice(None) # Slice object passed to grammar rules self.errorcount = 0 # Used during error recovery # If no lexer was given, we will try to use the lex module if not lexer: import lex as lexer # If input was supplied, pass to lexer if input: lexer.input(input) # Tokenize function get_token = lexer.token statestack = [ ] # Stack of parsing states self.statestack = statestack symstack = [ ] # Stack of grammar symbols self.symstack = symstack errtoken = None # Err token # The start state is assumed to be (0,$) statestack.append(0) sym = YaccSymbol() sym.type = '$' symstack.append(sym) while 1: # Get the next symbol on the input. If a lookahead symbol # is already set, we just use that. Otherwise, we'll pull # the next token off of the lookaheadstack or from the lexer if not lookahead: if not lookaheadstack: lookahead = get_token() # Get the next token else: lookahead = lookaheadstack.pop() if not lookahead: lookahead = YaccSymbol() lookahead.type = '$' if debug: print "%-20s : %s" % (lookahead, [xx.type for xx in symstack]) # Check the action table s = statestack[-1] ltype = lookahead.type t = actions.get((s,ltype),None) if t is not None: if t > 0: # shift a symbol on the stack if ltype == '$': # Error, end of input print "yacc: Parse error. EOF" return statestack.append(t) symstack.append(lookahead) lookahead = None # Decrease error count on successful shift if self.errorcount > 0: self.errorcount -= 1 continue if t < 0: # reduce a symbol on the stack, emit a production p = prod[-t] pname = p.name plen = p.len # Get production function sym = YaccSymbol() sym.type = pname # Production name sym.value = None if plen: targ = symstack[-plen-1:] targ[0] = sym try: sym.lineno = targ[1].lineno sym.endlineno = getattr(targ[-1],"endlineno",targ[-1].lineno) except AttributeError: sym.lineno = 0 del symstack[-plen:] del statestack[-plen:] else: sym.lineno = 0 targ = [ sym ] pslice.slice = targ pslice.pbstack = [] # Call the grammar rule with our special slice object p.func(pslice) # Validate attributes of the resulting value attribute # if require: # try: # t0 = targ[0] # r = Requires.get(t0.type,None) # t0d = t0.__dict__ # if r: # for field in r: # tn = t0 # for fname in field: # try: # tf = tn.__dict__ # tn = tf.get(fname) # except StandardError: # tn = None # if not tn: # print "%s:%d. Rule %s doesn't set required attribute '%s'" % \ # (p.file,p.line,p.name,".".join(field)) # except TypeError,LookupError: # print "Bad requires directive " % r # pass # If there was a pushback, put that on the stack if pslice.pbstack: lookaheadstack.append(lookahead) for _t in pslice.pbstack: lookaheadstack.append(_t) lookahead = None symstack.append(sym) statestack.append(goto[statestack[-1],pname]) continue if t == 0: n = symstack[-1] return getattr(n,"value",None) if t == None: # We have some kind of parsing error here. To handle this, # we are going to push the current token onto the tokenstack # and replace it with an 'error' token. If there are any synchronization # rules, they may catch it. # # In addition to pushing the error token, we call call the user defined p_error() # function if this is the first syntax error. This function is only called # if errorcount == 0. if not self.errorcount: self.errorcount = error_count errtoken = lookahead if errtoken.type == '$': errtoken = None # End of file! if self.errorfunc: global errok,token,restart errok = self.errok # Set some special functions available in error recovery token = get_token restart = self.restart tok = self.errorfunc(errtoken) del errok, token, restart # Delete special functions if not self.errorcount: # User must have done some kind of panic mode recovery on their own. The returned token # is the next lookahead lookahead = tok errtoken = None continue else: if errtoken: if hasattr(errtoken,"lineno"): lineno = lookahead.lineno else: lineno = 0 if lineno: print "yacc: Syntax error at line %d, token=%s" % (lineno, errtoken.type) else: print "yacc: Syntax error, token=%s" % errtoken.type else: print "yacc: Parse error in input. EOF" return else: self.errorcount = error_count # case 1: the statestack only has 1 entry on it. If we're in this state, the # entire parse has been rolled back and we're completely hosed. The token is # discarded and we just keep going. if len(statestack) <= 1 and lookahead.type != '$': lookahead = None errtoken = None # Nuke the pushback stack del lookaheadstack[:] continue # case 2: the statestack has a couple of entries on it, but we're # at the end of the file. nuke the top entry and generate an error token # Start nuking entries on the stack if lookahead.type == '$': # Whoa. We're really hosed here. Bail out return if lookahead.type != 'error': sym = symstack[-1] if sym.type == 'error': # Hmmm. Error is on top of stack, we'll just nuke input # symbol and continue lookahead = None continue t = YaccSymbol() t.type = 'error' if hasattr(lookahead,"lineno"): t.lineno = lookahead.lineno t.value = lookahead lookaheadstack.append(lookahead) lookahead = t else: symstack.pop() statestack.pop() continue # Call an error function here raise RuntimeError, "yacc: internal parser error!!!\n" # ----------------------------------------------------------------------------- # === Parser Construction === # # The following functions and variables are used to implement the yacc() function # itself. This is pretty hairy stuff involving lots of error checking, # construction of LR items, kernels, and so forth. Although a lot of # this work is done using global variables, the resulting Parser object # is completely self contained--meaning that it is safe to repeatedly # call yacc() with different grammars in the same application. # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # validate_file() # # This function checks to see if there are duplicated p_rulename() functions # in the parser module file. Without this function, it is really easy for # users to make mistakes by cutting and pasting code fragments (and it's a real # bugger to try and figure out why the resulting parser doesn't work). Therefore, # we just do a little regular expression pattern matching of def statements # to try and detect duplicates. # ----------------------------------------------------------------------------- def validate_file(filename): base,ext = os.path.splitext(filename) if ext != '.py': return 1 # No idea. Assume it's okay. try: f = open(filename) lines = f.readlines() f.close() except IOError: return 1 # Oh well # Match def p_funcname( fre = re.compile(r'\s*def\s+(p_[a-zA-Z_0-9]*)\(') counthash = { } linen = 1 noerror = 1 for l in lines: m = fre.match(l) if m: name = m.group(1) prev = counthash.get(name) if not prev: counthash[name] = linen else: print "%s:%d: Function %s redefined. Previously defined on line %d" % (filename,linen,name,prev) noerror = 0 linen += 1 return noerror # This function looks for functions that might be grammar rules, but which don't have the proper p_suffix. def validate_dict(d): for n,v in d.items(): if n[0:2] == 'p_' and isinstance(v,types.FunctionType): continue if n[0:2] == 't_': continue if n[0:2] == 'p_': print "yacc: Warning. '%s' not defined as a function" % n if isinstance(v,types.FunctionType) and v.func_code.co_argcount == 1: try: doc = v.__doc__.split(" ") if doc[1] == ':': print "%s:%d. Warning. Possible grammar rule '%s' defined without p_ prefix." % (v.func_code.co_filename, v.func_code.co_firstlineno,n) except StandardError: pass # ----------------------------------------------------------------------------- # === GRAMMAR FUNCTIONS === # # The following global variables and functions are used to store, manipulate, # and verify the grammar rules specified by the user. # ----------------------------------------------------------------------------- # Initialize all of the global variables used during grammar construction def initialize_vars(): global Productions, Prodnames, Prodmap, Prodempty, Terminals global Nonterminals, First, Follow, Precedence, LRitems global Errorfunc, Signature, Requires Productions = [None] # A list of all of the productions. The first # entry is always reserved for the purpose of # building an augmented grammar Prodnames = { } # A dictionary mapping the names of nonterminals to a list of all # productions of that nonterminal. Prodmap = { } # A dictionary that is only used to detect duplicate # productions. Prodempty = { } # A dictionary of all productions that have an empty rule # of the form P : Terminals = { } # A dictionary mapping the names of terminal symbols to a # list of the rules where they are used. Nonterminals = { } # A dictionary mapping names of nonterminals to a list # of rule numbers where they are used. First = { } # A dictionary of precomputed FIRST(x) symbols Follow = { } # A dictionary of precomputed FOLLOW(x) symbols Precedence = { } # Precedence rules for each terminal. Contains tuples of the # form ('right',level) or ('left',level) LRitems = [ ] # A list of all LR items for the grammar. These are the # productions with the "dot" like E -> E . PLUS E Errorfunc = None # User defined error handler Signature = md5.new() # Digital signature of the grammar rules, precedence # and other information. Used to determined when a # parsing table needs to be regenerated. Requires = { } # Requires list # File objects used when creating the parser.out debugging file global _vf, _vfc _vf = cStringIO.StringIO() _vfc = cStringIO.StringIO() # ----------------------------------------------------------------------------- # class Production: # # This class stores the raw information about a single production or grammar rule. # It has a few required attributes: # # name - Name of the production (nonterminal) # prod - A list of symbols making up its production # number - Production number. # # In addition, a few additional attributes are used to help with debugging or # optimization of table generation. # # file - File where production action is defined. # lineno - Line number where action is defined # func - Action function # prec - Precedence level # lr_next - Next LR item. Example, if we are ' E -> E . PLUS E' # then lr_next refers to 'E -> E PLUS . E' # lr_index - LR item index (location of the ".") in the prod list. # len - Length of the production (number of symbols on right hand side) # ----------------------------------------------------------------------------- class Production: def __init__(self,**kw): for k,v in kw.items(): setattr(self,k,v) self.lr_index = -1 self.lr0_added = 0 # Flag indicating whether or not added to LR0 closure self.usyms = [ ] def __str__(self): if self.prod: s = "%s -> %s" % (self.name," ".join(self.prod)) else: s = "%s -> " % self.name return s def __repr__(self): return str(self) # Compute lr_items from the production def lr_item(self,n): if n > len(self.prod): return None p = Production() p.name = self.name p.prod = list(self.prod) p.number = self.number p.lr_index = n p.prod.insert(n,".") p.prod = tuple(p.prod) p.len = len(p.prod) p.usyms = self.usyms # Precompute list of productions immediately following try: p.lrafter = Prodnames[p.prod[n+1]] except (IndexError,KeyError),e: p.lrafter = [] try: p.lrbefore = p.prod[n-1] except IndexError: p.lrbefore = None return p # Utility function def is_identifier(s): for c in s: if not (c.isalnum() or c == '_'): return 0 return 1 # ----------------------------------------------------------------------------- # add_production() # # Given an action function, this function assembles a production rule. # The production rule is assumed to be found in the function's docstring. # This rule has the general syntax: # # name1 ::= production1 # | production2 # | production3 # ... # | productionn # name2 ::= production1 # | production2 # ... # ----------------------------------------------------------------------------- def add_production(f,file,line,prodname,syms): if Terminals.has_key(prodname): print "%s:%d. Illegal rule name '%s'. Already defined as a token." % (file,line,prodname) return -1 if prodname == 'error': print "%s:%d. Illegal rule name '%s'. error is a reserved word." % (file,line,prodname) return -1 if not is_identifier(prodname): print "%s:%d. Illegal rule name '%s'" % (file,line,prodname) return -1 for s in syms: if not is_identifier(s) and s != '%prec': print "%s:%d. Illegal name '%s' in rule '%s'" % (file,line,s, prodname) return -1 # See if the rule is already in the rulemap map = "%s -> %s" % (prodname,syms) if Prodmap.has_key(map): m = Prodmap[map] print "%s:%d. Duplicate rule %s." % (file,line, m) print "%s:%d. Previous definition at %s:%d" % (file,line, m.file, m.line) return -1 p = Production() p.name = prodname p.prod = syms p.file = file p.line = line p.func = f p.number = len(Productions) Productions.append(p) Prodmap[map] = p if not Nonterminals.has_key(prodname): Nonterminals[prodname] = [ ] # Add all terminals to Terminals i = 0 while i < len(p.prod): t = p.prod[i] if t == '%prec': try: precname = p.prod[i+1] except IndexError: print "%s:%d. Syntax error. Nothing follows %%prec." % (p.file,p.line) return -1 prec = Precedence.get(precname,None) if not prec: print "%s:%d. Nothing known about the precedence of '%s'" % (p.file,p.line,precname) return -1 else: p.prec = prec del p.prod[i] del p.prod[i] continue if Terminals.has_key(t): Terminals[t].append(p.number) # Is a terminal. We'll assign a precedence to p based on this if not hasattr(p,"prec"): p.prec = Precedence.get(t,('right',0)) else: if not Nonterminals.has_key(t): Nonterminals[t] = [ ] Nonterminals[t].append(p.number) i += 1 if not hasattr(p,"prec"): p.prec = ('right',0) # Set final length of productions p.len = len(p.prod) p.prod = tuple(p.prod) # Calculate unique syms in the production p.usyms = [ ] for s in p.prod: if s not in p.usyms: p.usyms.append(s) # Add to the global productions list try: Prodnames[p.name].append(p) except KeyError: Prodnames[p.name] = [ p ] return 0 # Given a raw rule function, this function rips out its doc string # and adds rules to the grammar def add_function(f): line = f.func_code.co_firstlineno file = f.func_code.co_filename error = 0 if f.func_code.co_argcount > 1: print "%s:%d. Rule '%s' has too many arguments." % (file,line,f.__name__) return -1 if f.func_code.co_argcount < 1: print "%s:%d. Rule '%s' requires an argument." % (file,line,f.__name__) return -1 if f.__doc__: # Split the doc string into lines pstrings = f.__doc__.splitlines() lastp = None dline = line for ps in pstrings: dline += 1 p = ps.split() if not p: continue try: if p[0] == '|': # This is a continuation of a previous rule if not lastp: print "%s:%d. Misplaced '|'." % (file,dline) return -1 prodname = lastp if len(p) > 1: syms = p[1:] else: syms = [ ] else: prodname = p[0] lastp = prodname assign = p[1] if len(p) > 2: syms = p[2:] else: syms = [ ] if assign != ':' and assign != '::=': print "%s:%d. Syntax error. Expected ':'" % (file,dline) return -1 e = add_production(f,file,dline,prodname,syms) error += e except StandardError: print "%s:%d. Syntax error in rule '%s'" % (file,dline,ps) error -= 1 else: print "%s:%d. No documentation string specified in function '%s'" % (file,line,f.__name__) return error # ----------------------------------------------------------------------------- # check_cycles() # # This function looks at the various parsing rules and tries to detect # infinite recursion cycles (grammar rules where there is no possible way # to derive a string of terminals). # ----------------------------------------------------------------------------- def check_cycles(p=None,val=1,start=None): if not p: # Walk through list of productions and zero out cycle counts for i in Productions: i.cychk = 0 # Already encountered this rule i.cyvalue = -1 error = 0 # First run a cycle check on the top rule. This will find # everything that's used. p = Productions[1] p = Productions[0] c = check_cycles(p,1) val = 2 for n,pl in Prodnames.items(): term = 0 used = 0 for p in pl: if not p.cychk: continue used +=1 term += check_cycles(p,val) if not used: print "yacc: Rule '%s' never used." % n if used and not term and Nonterminals[n]: print "yacc: Infinite recursion detected in rule '%s'." % n error = 1 val += 1 return error # We are checking this rule for a cycle if p.cychk == val: # Hmmm. We already saw this rule before. if p.cyvalue >= 0: return p.cyvalue else: return 0 p.cyvalue = -1 # Looking for cycles p.cychk = val # Look at all of the symbols to the right of this production term = 0 for s in p.prod: # Get list of productions for 's' pl = Prodnames.get(s,None) if not pl: term += 1 continue # Must be a terminal. Continue # Check all of the rules for each non-terminal for cycles pterm = 0 for x in pl: c = check_cycles(x,val) if c: pterm = 1 if val > 1: break term += pterm # All right hand side symbols terminate # print p.name, term, len(p.prod), val if term == len(p.prod): p.cyvalue = 1 else: p.cyvalue = 0 return p.cyvalue # ----------------------------------------------------------------------------- # verify_productions() # # This function examines all of the supplied rules to see if they seem valid. # ----------------------------------------------------------------------------- def verify_productions(cycle_check=1): error = 0 for p in Productions: if not p: continue # Look for a purely empty productions and record in special dictionary if not len(p.prod): Prodempty[p.name] = p continue for s in p.prod: if not Prodnames.has_key(s) and not Terminals.has_key(s) and s != 'error': print "%s:%d. Symbol '%s' used, but not defined as a token or a rule." % (p.file,p.line,s) error = 1 continue unused_tok = 0 # Now verify all of the tokens if yaccdebug: _vf.write("Unused terminals:\n\n") for s,v in Terminals.items(): if s != 'error' and not v: print "yacc: Warning. Token '%s' defined, but not used." % s if yaccdebug: _vf.write(" %s\n"% s) unused_tok += 1 # Print out all of the productions if yaccdebug: _vf.write("\nGrammar\n\n") for i in range(1,len(Productions)): _vf.write("Rule %-5d %s\n" % (i, Productions[i])) unused_prod = 0 # Verify the use of all productions for s,v in Nonterminals.items(): if not v: p = Prodnames[s][0] print "%s:%d: Warning. Rule '%s' defined, but not used." % (p.file,p.line, s) unused_prod += 1 if unused_tok == 1: print "yacc: Warning. There is 1 unused token." if unused_tok > 1: print "yacc: Warning. There are %d unused tokens." % unused_tok if unused_prod == 1: print "yacc: Warning. There is 1 unused rule." if unused_prod > 1: print "yacc: Warning. There are %d unused rules." % unused_prod if yaccdebug: _vf.write("\nTerminals, with rules where they appear\n\n") for k in Terminals.keys(): _vf.write("%-20s : %s\n" % (k, " ".join([str(s) for s in Terminals[k]]))) _vf.write("\nNonterminals, with rules where they appear\n\n") for k in Nonterminals.keys(): _vf.write("%-20s : %s\n" % (k, " ".join([str(s) for s in Nonterminals[k]]))) if (cycle_check): error += check_cycles() return error # ----------------------------------------------------------------------------- # build_lritems() # # This function walks the list of productions and builds a complete set of the # LR items. The LR items are stored in two ways: First, they are uniquely # numbered and placed in the list _lritems. Second, a linked list of LR items # is built for each production. For example: # # E -> E PLUS E # # Creates the list # # [E -> . E PLUS E, E -> E . PLUS E, E -> E PLUS . E, E -> E PLUS E . ] # ----------------------------------------------------------------------------- def build_lritems(): for p in Productions: lastlri = p lri = p.lr_item(0) i = 0 while 1: lri = p.lr_item(i) lastlri.lr_next = lri if not lri: break lri.lr_num = len(LRitems) LRitems.append(lri) lastlri = lri i += 1 # In order for the rest of the parser generator to work, we need to # guarantee that no more lritems are generated. Therefore, we nuke # the p.lr_item method. (Only used in debugging) # Production.lr_item = None # ----------------------------------------------------------------------------- # add_precedence() # # Given a list of precedence rules, add to the precedence table. # ----------------------------------------------------------------------------- def add_precedence(plist): plevel = 0 error = 0 for p in plist: plevel += 1 try: prec = p[0] terms = p[1:] if prec != 'left' and prec != 'right': print "yacc: Invalid precedence '%s'" % prec return -1 for t in terms: if Precedence.has_key(t): print "yacc: Precedence already specified for terminal '%s'" % t error += 1 continue Precedence[t] = (prec,plevel) except: print "yacc: Invalid precedence table." error += 1 return error # ----------------------------------------------------------------------------- # augment_grammar() # # Compute the augmented grammar. This is just a rule S' -> start where start # is the starting symbol. # ----------------------------------------------------------------------------- def augment_grammar(start=None): if not start: start = Productions[1].name Productions[0] = Production(name="S'",prod=[start],number=0,len=1,prec=('right',0),func=None) Productions[0].usyms = [ start ] Nonterminals[start].append(0) # ----------------------------------------------------------------------------- # first() # # Compute the value of FIRST1(X) where x is a terminal, nonterminal, or tuple # of grammar symbols. # ----------------------------------------------------------------------------- def first(x): # Check if already computed fst = First.get(x,None) if fst: return fst if isinstance(x,types.TupleType): # We are computing First(x1,x2,x3,x4,...xn) fst = [ ] numempty = 0 for s in x: f = first(s) for i in f: if i == '': numempty +=1 continue if i not in fst: fst.append(i) if not numempty: break if numempty == len(x): fst.append('') elif Terminals.has_key(x): fst = [x] elif Prodnames.has_key(x): fst = [ ] prodlist = Prodnames[x] for p in prodlist: # Check if we're already computing first on this production if p.cfirst: continue # Check for empty, we just add empty to first in this case if not len(p.prod): fst.append('') p.cfirst = 1 continue # Go down the list of production symbols looking for empty # productions p.cfirst = 1 for ps in p.prod: if ps == '': continue # Add everything in first(ps) to our set f = first(ps) for i in f: if i not in fst: fst.append(i) # If this rule doesn't produce empty, we're done if not Prodempty.has_key(ps): break else: raise YaccError, "first: %s not a terminal or nonterminal" % x First[x] = fst return fst # FOLLOW(x) # Given a non-terminal. This function computes the set of all symbols # that might follow it. Dragon book, p. 189. def compute_follow(start=None): # Add '$' to the follow list of the start symbol for k in Nonterminals.keys(): Follow[k] = [ ] if not start: start = Productions[1].name Follow[start] = [ '$' ] while 1: didadd = 0 for p in Productions[1:]: # Here is the production set for i in range(len(p.prod)): B = p.prod[i] if Nonterminals.has_key(B): # Okay. We got a non-terminal in a production fst = first(p.prod[i+1:]) hasempty = 0 for f in fst: if f != '' and f not in Follow[B]: Follow[B].append(f) didadd = 1 if f == '': hasempty = 1 if hasempty or i == (len(p.prod)-1): # Add elements of follow(a) to follow(b) for f in Follow[p.name]: if f not in Follow[B]: Follow[B].append(f) didadd = 1 if not didadd: break # Compute first for all symbols def compute_first1(): # Result first flag for p in Productions: p.cfirst = 0 # Compute all terminals for t in Terminals.keys(): first(t) # Compute for all nonterminals for n in Nonterminals.keys(): prodlist = Prodnames[n] first(n) # First for the end symbol marker First['$'] = ['$'] First['#'] = ['#'] # ----------------------------------------------------------------------------- # === SLR Generation === # # The following functions are used to construct SLR (Simple LR) parsing tables # as described on p.221-229 of the dragon book. # ----------------------------------------------------------------------------- # Global variables for the LR parsing engine def lr_init_vars(): global _lr_action, _lr_goto, _lr_method global _lr_goto_cache _lr_action = { } # Action table _lr_goto = { } # Goto table _lr_method = "Unknown" # LR method used _lr_goto_cache = { } # Compute the LR(0) closure operation on I, where I is a set of LR(0) items. # prodlist is a list of productions. _add_count = 0 # Counter used to detect cycles def lr0_closure(I): global _add_count _add_count += 1 prodlist = Productions # Add everything in I to J J = I[:] didadd = 1 while didadd: didadd = 0 for j in J: for x in j.lrafter: if x.lr0_added == _add_count: continue # Add B --> .G to J J.append(x.lr_next) x.lr0_added = _add_count didadd = 1 return J # Compute the LR(0) goto function goto(I,X) where I is a set # of LR(0) items and X is a grammar symbol. This function is written # in a way that guarantees uniqueness of the generated goto sets # (i.e. the same goto set will never be returned as two different Python # objects). With uniqueness, we can later do fast set comparisons using # id(obj) instead of element-wise comparison. def lr0_goto(I,x): # First we look for a previously cached entry g = _lr_goto_cache.get((id(I),x),None) if g: return g # Now we generate the goto set in a way that guarantees uniqueness # of the result s = _lr_goto_cache.get(x,None) if not s: s = { } _lr_goto_cache[x] = s gs = [ ] for p in I: n = p.lr_next if n and n.lrbefore == x: s1 = s.get(id(n),None) if not s1: s1 = { } s[id(n)] = s1 gs.append(n) s = s1 g = s.get('$',None) if not g: if gs: g = lr0_closure(gs) s['$'] = g else: s['$'] = gs _lr_goto_cache[(id(I),x)] = g return g # Compute the kernel of a set of LR(0) items def lr0_kernel(I): KI = [ ] for p in I: if p.name == "S'" or p.lr_index > 0 or p.len == 0: KI.append(p) return KI _lr0_cidhash = { } # Compute the LR(0) sets of item function def lr0_items(): C = [ lr0_closure([Productions[0].lr_next]) ] i = 0 for I in C: _lr0_cidhash[id(I)] = i i += 1 # Loop over the items in C and each grammar symbols i = 0 while i < len(C): I = C[i] i += 1 # Collect all of the symbols that could possibly be in the goto(I,X) sets asyms = { } for ii in I: for s in ii.usyms: asyms[s] = None for x in asyms.keys(): g = lr0_goto(I,x) if not g: continue if _lr0_cidhash.has_key(id(g)): continue _lr0_cidhash[id(g)] = len(C) C.append(g) return C # ----------------------------------------------------------------------------- # slr_parse_table() # # This function constructs an SLR table. # ----------------------------------------------------------------------------- def slr_parse_table(): global _lr_method goto = _lr_goto # Goto array action = _lr_action # Action array actionp = { } # Action production array (temporary) _lr_method = "SLR" n_srconflict = 0 n_rrconflict = 0 if yaccdebug: print "yacc: Generating SLR parsing table..." _vf.write("\n\nParsing method: SLR\n\n") # Step 1: Construct C = { I0, I1, ... IN}, collection of LR(0) items # This determines the number of states C = lr0_items() # Build the parser table, state by state st = 0 for I in C: # Loop over each production in I actlist = [ ] # List of actions if yaccdebug: _vf.write("\nstate %d\n\n" % st) for p in I: _vf.write(" %s\n" % str(p)) _vf.write("\n") for p in I: try: if p.prod[-1] == ".": if p.name == "S'": # Start symbol. Accept! action[st,"$"] = 0 actionp[st,"$"] = p else: # We are at the end of a production. Reduce! for a in Follow[p.name]: actlist.append((a,p,"reduce using rule %d" % p.number)) r = action.get((st,a),None) if r is not None: # Whoa. Have a shift/reduce or reduce/reduce conflict if r > 0: # Need to decide on shift or reduce here # By default we favor shifting. Need to add # some precedence rules here. sprec,slevel = Productions[actionp[st,a].number].prec rprec,rlevel = Precedence.get(a,('right',0)) if (slevel < rlevel) or ((slevel == rlevel) and (rprec == 'left')): # We really need to reduce here. action[st,a] = -p.number actionp[st,a] = p if not slevel and not rlevel: _vfc.write("shift/reduce conflict in state %d resolved as reduce.\n" % st) _vf.write(" ! shift/reduce conflict for %s resolved as reduce.\n" % a) n_srconflict += 1 else: # Hmmm. Guess we'll keep the shift if not slevel and not rlevel: _vfc.write("shift/reduce conflict in state %d resolved as shift.\n" % st) _vf.write(" ! shift/reduce conflict for %s resolved as shift.\n" % a) n_srconflict +=1 elif r < 0: # Reduce/reduce conflict. In this case, we favor the rule # that was defined first in the grammar file oldp = Productions[-r] pp = Productions[p.number] if oldp.line > pp.line: action[st,a] = -p.number actionp[st,a] = p # print "Reduce/reduce conflict in state %d" % st n_rrconflict += 1 _vfc.write("reduce/reduce conflict in state %d resolved using rule %d.\n" % (st, actionp[st,a].number)) _vf.write(" ! reduce/reduce conflict for %s resolved using rule %d.\n" % (a,actionp[st,a].number)) else: print "Unknown conflict in state %d" % st else: action[st,a] = -p.number actionp[st,a] = p else: i = p.lr_index a = p.prod[i+1] # Get symbol right after the "." if Terminals.has_key(a): g = lr0_goto(I,a) j = _lr0_cidhash.get(id(g),-1) if j >= 0: # We are in a shift state actlist.append((a,p,"shift and go to state %d" % j)) r = action.get((st,a),None) if r is not None: # Whoa have a shift/reduce or shift/shift conflict if r > 0: if r != j: print "Shift/shift conflict in state %d" % st elif r < 0: # Do a precedence check. # - if precedence of reduce rule is higher, we reduce. # - if precedence of reduce is same and left assoc, we reduce. # - otherwise we shift rprec,rlevel = Productions[actionp[st,a].number].prec sprec,slevel = Precedence.get(a,('right',0)) if (slevel > rlevel) or ((slevel == rlevel) and (rprec != 'left')): # We decide to shift here... highest precedence to shift action[st,a] = j actionp[st,a] = p if not slevel and not rlevel: n_srconflict += 1 _vfc.write("shift/reduce conflict in state %d resolved as shift.\n" % st) _vf.write(" ! shift/reduce conflict for %s resolved as shift.\n" % a) else: # Hmmm. Guess we'll keep the reduce if not slevel and not rlevel: n_srconflict +=1 _vfc.write("shift/reduce conflict in state %d resolved as reduce.\n" % st) _vf.write(" ! shift/reduce conflict for %s resolved as reduce.\n" % a) else: print "Unknown conflict in state %d" % st else: action[st,a] = j actionp[st,a] = p except StandardError,e: raise YaccError, "Hosed in slr_parse_table", e # Print the actions associated with each terminal if yaccdebug: for a,p,m in actlist: if action.has_key((st,a)): if p is actionp[st,a]: _vf.write(" %-15s %s\n" % (a,m)) _vf.write("\n") for a,p,m in actlist: if action.has_key((st,a)): if p is not actionp[st,a]: _vf.write(" ! %-15s [ %s ]\n" % (a,m)) # Construct the goto table for this state nkeys = { } for ii in I: for s in ii.usyms: if Nonterminals.has_key(s): nkeys[s] = None # Construct the goto table for this state for n in nkeys.keys(): g = lr0_goto(I,n) j = _lr0_cidhash.get(id(g),-1) if j >= 0: goto[st,n] = j st += 1 if n_srconflict == 1 and yaccdebug: print "yacc: %d shift/reduce conflict" % n_srconflict if n_srconflict > 1 and yaccdebug: print "yacc: %d shift/reduce conflicts" % n_srconflict if n_rrconflict == 1 and yaccdebug: print "yacc: %d reduce/reduce conflict" % n_rrconflict if n_rrconflict > 1 and yaccdebug: print "yacc: %d reduce/reduce conflicts" % n_rrconflict # ----------------------------------------------------------------------------- # ==== LALR(1) Parsing ==== # **** UNFINISHED! 6/16/01 # ----------------------------------------------------------------------------- # Compute the lr1_closure of a set I. I is a list of tuples (p,a) where # p is a LR0 item and a is a terminal _lr1_add_count = 0 def lr1_closure(I): global _lr1_add_count _lr1_add_count += 1 J = I[:] # Loop over items (p,a) in I. ji = 0 while ji < len(J): p,a = J[ji] # p = [ A -> alpha . B beta] # For each production B -> gamma for B in p.lr1_after: f = tuple(p.lr1_beta + (a,)) # For each terminal b in first(Beta a) for b in first(f): # Check if (B -> . gamma, b) is in J # Only way this can happen is if the add count mismatches pn = B.lr_next if pn.lr_added.get(b,0) == _lr1_add_count: continue pn.lr_added[b] = _lr1_add_count J.append((pn,b)) ji += 1 return J def lalr_parse_table(): # Compute some lr1 information about all of the productions for p in LRitems: try: after = p.prod[p.lr_index + 1] p.lr1_after = Prodnames[after] p.lr1_beta = p.prod[p.lr_index + 2:] except LookupError: p.lr1_after = [ ] p.lr1_beta = [ ] p.lr_added = { } # Compute the LR(0) items C = lr0_items() CK = [] for I in C: CK.append(lr0_kernel(I)) print CK # ----------------------------------------------------------------------------- # ==== LR Utility functions ==== # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # _lr_write_tables() # # This function writes the LR parsing tables to a file # ----------------------------------------------------------------------------- def lr_write_tables(modulename=tab_module): filename = modulename + ".py" try: f = open(filename,"w") f.write(""" # %s # This file is automatically generated. Do not edit. _lr_method = %s _lr_signature = %s """ % (filename, repr(_lr_method), repr(Signature.digest()))) # Change smaller to 0 to go back to original tables smaller = 1 # Factor out names to try and make smaller if smaller: items = { } for k,v in _lr_action.items(): i = items.get(k[1]) if not i: i = ([],[]) items[k[1]] = i i[0].append(k[0]) i[1].append(v) f.write("\n_lr_action_items = {") for k,v in items.items(): f.write("%r:([" % k) for i in v[0]: f.write("%r," % i) f.write("],[") for i in v[1]: f.write("%r," % i) f.write("]),") f.write("}\n") f.write(""" _lr_action = { } for _k, _v in _lr_action_items.items(): for _x,_y in zip(_v[0],_v[1]): _lr_action[(_x,_k)] = _y del _lr_action_items """) else: f.write("\n_lr_action = { "); for k,v in _lr_action.items(): f.write("(%r,%r):%r," % (k[0],k[1],v)) f.write("}\n"); if smaller: # Factor out names to try and make smaller items = { } for k,v in _lr_goto.items(): i = items.get(k[1]) if not i: i = ([],[]) items[k[1]] = i i[0].append(k[0]) i[1].append(v) f.write("\n_lr_goto_items = {") for k,v in items.items(): f.write("%r:([" % k) for i in v[0]: f.write("%r," % i) f.write("],[") for i in v[1]: f.write("%r," % i) f.write("]),") f.write("}\n") f.write(""" _lr_goto = { } for _k, _v in _lr_goto_items.items(): for _x,_y in zip(_v[0],_v[1]): _lr_goto[(_x,_k)] = _y del _lr_goto_items """) else: f.write("\n_lr_goto = { "); for k,v in _lr_goto.items(): f.write("(%r,%r):%r," % (k[0],k[1],v)) f.write("}\n"); f.close() except IOError,e: print "Unable to create '%s'" % filename print e return def lr_read_tables(module=tab_module): global _lr_action, _lr_goto try: exec "import %s as parsetab" % module if Signature.digest() == parsetab._lr_signature: _lr_action = parsetab._lr_action _lr_goto = parsetab._lr_goto return 1 else: return 0 except (ImportError,AttributeError): return 0 # ----------------------------------------------------------------------------- # yacc(module) # # Build the parser module # ----------------------------------------------------------------------------- def yacc(method=default_lr, debug=yaccdebug, module=None, tabmodule=tab_module, start=None, check_recursion=1): global yaccdebug yaccdebug = debug initialize_vars() files = { } error = 0 # Add starting symbol to signature if start: Signature.update(start) # Try to figure out what module we are working with if module: # User supplied a module object. if not isinstance(module, types.ModuleType): raise ValueError,"Expected a module" ldict = module.__dict__ else: # No module given. We might be able to get information from the caller. # Throw an exception and unwind the traceback to get the globals try: raise RuntimeError except RuntimeError: e,b,t = sys.exc_info() f = t.tb_frame f = f.f_back # Walk out to our calling function ldict = f.f_globals # Grab its globals dictionary # Get the tokens map tokens = ldict.get("tokens",None) if not tokens: raise YaccError,"module does not define a list 'tokens'" if not (isinstance(tokens,types.ListType) or isinstance(tokens,types.TupleType)): raise YaccError,"tokens must be a list or tuple." # Check to see if a requires dictionary is defined. requires = ldict.get("require",None) if requires: if not (isinstance(requires,types.DictType)): raise YaccError,"require must be a dictionary." for r,v in requires.items(): try: if not (isinstance(v,types.ListType)): raise TypeError v1 = [x.split(".") for x in v] Requires[r] = v1 except StandardError: print "Invalid specification for rule '%s' in require. Expected a list of strings" % r # Build the dictionary of terminals. We a record a 0 in the # dictionary to track whether or not a terminal is actually # used in the grammar if 'error' in tokens: print "yacc: Illegal token 'error'. Is a reserved word." raise YaccError,"Illegal token name" for n in tokens: if Terminals.has_key(n): print "yacc: Warning. Token '%s' multiply defined." % n Terminals[n] = [ ] Terminals['error'] = [ ] # Get the precedence map (if any) prec = ldict.get("precedence",None) if prec: if not (isinstance(prec,types.ListType) or isinstance(prec,types.TupleType)): raise YaccError,"precedence must be a list or tuple." add_precedence(prec) Signature.update(repr(prec)) for n in tokens: if not Precedence.has_key(n): Precedence[n] = ('right',0) # Default, right associative, 0 precedence # Look for error handler ef = ldict.get('p_error',None) if ef: if not isinstance(ef,types.FunctionType): raise YaccError,"'p_error' defined, but is not a function." eline = ef.func_code.co_firstlineno efile = ef.func_code.co_filename files[efile] = None if (ef.func_code.co_argcount != 1): raise YaccError,"%s:%d. p_error() requires 1 argument." % (efile,eline) global Errorfunc Errorfunc = ef else: print "yacc: Warning. no p_error() function is defined." # Get the list of built-in functions with p_ prefix symbols = [ldict[f] for f in ldict.keys() if (isinstance(ldict[f],types.FunctionType) and ldict[f].__name__[:2] == 'p_' and ldict[f].__name__ != 'p_error')] # Check for non-empty symbols if len(symbols) == 0: raise YaccError,"no rules of the form p_rulename are defined." # Sort the symbols by line number symbols.sort(lambda x,y: cmp(x.func_code.co_firstlineno,y.func_code.co_firstlineno)) # Add all of the symbols to the grammar for f in symbols: if (add_function(f)) < 0: error += 1 else: files[f.func_code.co_filename] = None # Make a signature of the docstrings for f in symbols: if f.__doc__: Signature.update(f.__doc__) lr_init_vars() if error: raise YaccError,"Unable to construct parser." if not lr_read_tables(tabmodule): # Validate files for filename in files.keys(): if not validate_file(filename): error = 1 # Validate dictionary validate_dict(ldict) if start and not Prodnames.has_key(start): raise YaccError,"Bad starting symbol '%s'" % start augment_grammar(start) error = verify_productions(cycle_check=check_recursion) otherfunc = [ldict[f] for f in ldict.keys() if (isinstance(ldict[f],types.FunctionType) and ldict[f].__name__[:2] != 'p_')] if error: raise YaccError,"Unable to construct parser." build_lritems() compute_first1() compute_follow(start) if method == 'SLR': slr_parse_table() elif method == 'LALR1': lalr_parse_table() return else: raise YaccError, "Unknown parsing method '%s'" % method lr_write_tables(tabmodule) if yaccdebug: try: f = open(debug_file,"w") f.write(_vfc.getvalue()) f.write("\n\n") f.write(_vf.getvalue()) f.close() except IOError,e: print "yacc: can't create '%s'" % debug_file,e # Made it here. Create a parser object and set up its internal state. # Set global parse() method to bound method of parser object. p = Parser("xyzzy") p.productions = Productions p.errorfunc = Errorfunc p.action = _lr_action p.goto = _lr_goto p.method = _lr_method p.require = Requires global parse parse = p.parse # Clean up all of the globals we created yacc_cleanup() return p # yacc_cleanup function. Delete all of the global variables # used during table construction def yacc_cleanup(): global _lr_action, _lr_goto, _lr_method, _lr_goto_cache del _lr_action, _lr_goto, _lr_method, _lr_goto_cache global Productions, Prodnames, Prodmap, Prodempty, Terminals global Nonterminals, First, Follow, Precedence, LRitems global Errorfunc, Signature, Requires del Productions, Prodnames, Prodmap, Prodempty, Terminals del Nonterminals, First, Follow, Precedence, LRitems del Errorfunc, Signature, Requires global _vf, _vfc del _vf, _vfc # Stub that raises an error if parsing is attempted without first calling yacc() def parse(*args,**kwargs): raise YaccError, "yacc: No parser built with yacc()" ZpgCatalog-0.2/Indexes/PyQueryParser/parsetab.py0100644000175000001640000000345407702453323021411 0ustar fafhrddatacom # parsetab.py # This file is automatically generated. Do not edit. _lr_method = 'SLR' _lr_signature = '\xa4Io\xc3\xc0A\x83[\x9b\xbe4\x1d?O\x89\x8e' _lr_action_items = {'AND':([18,6,10,17,21,2,16,20,1,13,24,8,19,23,],[9,-6,9,-6,-1,9,9,9,-9,9,-10,-15,9,-14,]),'NEAR':([10,1,16,20,13,24,19,8,18,6,21,2,23,17,],[12,-9,12,12,12,-10,12,-15,12,-6,-1,12,-14,-6,]),'OPENP':([7,19,3,23,0,12,21,8,1,20,13,16,9,2,24,18,17,10,6,11,],[3,3,3,-14,3,3,-1,-15,-9,3,3,3,3,3,-10,3,-6,3,-6,3,]),'STRING':([6,18,11,19,5,10,17,14,2,4,24,9,16,20,1,3,7,21,8,12,13,23,0,],[-6,8,8,8,-13,8,-6,8,8,8,-10,8,8,8,-9,8,8,-1,-15,8,8,-14,8,]),'CLOSEP':([18,10,17,21,16,13,20,1,24,8,19,23,6,],[-2,-5,-6,-1,-7,21,-4,-9,-10,-15,-3,-14,-6,]),'NOT':([18,10,6,21,17,9,11,13,1,2,20,16,24,23,0,19,12,8,7,3,],[7,7,-6,-1,-6,7,7,7,-9,7,7,7,-10,-14,7,7,7,-15,7,7,]),'QUOTE':([17,0,7,3,12,22,20,8,6,9,24,10,23,11,16,2,14,21,19,18,15,1,13,],[-6,5,5,5,5,-11,5,-15,-6,5,-10,5,-14,5,5,5,-12,-1,5,5,23,-9,5,]),'OR':([1,13,16,24,20,23,19,8,18,6,2,21,17,10,],[-9,11,11,-10,11,-14,11,-15,11,-6,11,-1,-6,11,]),'$':([8,19,23,16,20,10,1,17,21,2,24,18,6,],[-15,-3,-14,-7,-4,-5,-9,-6,-1,0,-10,-2,-6,]),} _lr_action = { } for _k, _v in _lr_action_items.items(): for _x,_y in zip(_v[0],_v[1]): _lr_action[(_x,_k)] = _y del _lr_action_items _lr_goto_items = {'quotestart':([3,2,7,19,10,12,20,0,9,16,11,18,13,],[4,4,4,4,4,4,4,4,4,4,4,4,4,]),'string':([4,9,13,11,16,20,3,7,2,10,12,14,19,18,0,],[14,1,1,1,1,1,1,1,1,1,1,14,1,1,1,]),'factor':([20,2,0,10,12,3,7,9,13,18,11,16,19,],[6,6,6,6,6,6,17,6,6,6,6,6,6,]),'expr':([11,13,18,9,20,16,12,10,0,2,19,7,3,],[19,10,10,18,10,10,20,10,2,10,10,16,13,]),'quoteend':([15,],[24,]),'qterm':([14,4,],[22,15,]),} _lr_goto = { } for _k, _v in _lr_goto_items.items(): for _x,_y in zip(_v[0],_v[1]): _lr_goto[(_x,_k)] = _y del _lr_goto_items ZpgCatalog-0.2/Indexes/PyQueryParser/lex.py0100644000175000001640000005231107472761025020401 0ustar fafhrddatacom#----------------------------------------------------------------------------- # ply: lex.py # # Author: David M. Beazley (beazley@cs.uchicago.edu) # Department of Computer Science # University of Chicago # Chicago, IL 60637 # # Copyright (C) 2001, David M. Beazley # # $Header: /cvsroot/textindexng/TextIndexNG/parsers/PyQueryParser/lex.py,v 1.3 2002/05/22 18:23:17 ajung Exp $ # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # See the file COPYING for a complete copy of the LGPL. # # # This module automatically constructs a lexical analysis module from regular # expression rules defined in a user-defined module. The idea is essentially the same # as that used in John Aycock's Spark framework, but the implementation works # at the module level rather than requiring the use of classes. # # This module tries to provide an interface that is closely modeled after # the traditional lex interface in Unix. It also differs from Spark # in that: # # - It provides more extensive error checking and reporting if # the user supplies a set of regular expressions that can't # be compiled or if there is any other kind of a problem in # the specification. # # - The interface is geared towards LALR(1) and LR(1) parser # generators. That is tokens are generated one at a time # rather than being generated in advanced all in one step. # # There are a few limitations of this module # # - The module interface makes it somewhat awkward to support more # than one lexer at a time. Although somewhat inelegant from a # design perspective, this is rarely a practical concern for # most compiler projects. # # - The lexer requires that the entire input text be read into # a string before scanning. I suppose that most machines have # enough memory to make this a minor issues, but it makes # the lexer somewhat difficult to use in interactive sessions # or with streaming data. # #----------------------------------------------------------------------------- r""" lex.py This module builds lex-like scanners based on regular expression rules. To use the module, simply write a collection of regular expression rules and actions like this: # lexer.py import lex # Define a list of valid tokens tokens = ( 'IDENTIFIER', 'NUMBER', 'PLUS', 'MINUS' ) # Define tokens as functions def t_IDENTIFIER(t): r' ([a-zA-Z_](\w|_)* ' return t def t_NUMBER(t): r' \d+ ' return t # Some simple tokens with no actions t_PLUS = r'\+' t_MINUS = r'-' # Initialize the lexer lex.lex() The tokens list is required and contains a complete list of all valid token types that the lexer is allowed to produce. Token types are restricted to be valid identifiers. This means that 'MINUS' is a valid token type whereas '-' is not. Rules are defined by writing a function with a name of the form t_rulename. Each rule must accept a single argument which is a token object generated by the lexer. This token has the following attributes: t.type = type string of the token. This is initially set to the name of the rule without the leading t_ t.value = The value of the lexeme. t.lineno = The value of the line number where the token was encountered For example, the t_NUMBER() rule above might be called with the following: t.type = 'NUMBER' t.value = '42' t.lineno = 3 Each rule returns the token object it would like to supply to the parser. In most cases, the token t is returned with few, if any modifications. To discard a token for things like whitespace or comments, simply return nothing. For instance: def t_whitespace(t): r' \s+ ' pass For faster lexing, you can also define this in terms of the ignore set like this: t_ignore = ' \t' The characters in this string are ignored by the lexer. Use of this feature can speed up parsing significantly since scanning will immediately proceed to the next token. lex requires that the token returned by each rule has an attribute t.type. Other than this, rules are free to return any kind of token object that they wish and may construct a new type of token object from the attributes of t (provided the new object has the required type attribute). If illegal characters are encountered, the scanner executes the function t_error(t) where t is a token representing the rest of the string that hasn't been matched. If this function isn't defined, a LexError exception is raised. The .text attribute of this exception object contains the part of the string that wasn't matched. The t.skip(n) method can be used to skip ahead n characters in the input stream. This is usually only used in the error handling rule. For instance, the following rule would print an error message and continue: def t_error(t): print "Illegal character in input %s" % t.value[0] t.skip(1) Of course, a nice scanner might wish to skip more than one character if the input looks very corrupted. The lex module defines a t.lineno attribute on each token that can be used to track the current line number in the input. The value of this variable is not modified by lex so it is up to your lexer module to correctly update its value depending on the lexical properties of the input language. To do this, you might write rules such as the following: def t_newline(t): r' \n+ ' t.lineno += t.value.count("\n") To initialize your lexer so that it can be used, simply call the lex.lex() function in your rule file. If there are any errors in your specification, warning messages or an exception will be generated to alert you to the problem. (dave: this needs to be rewritten) To use the newly constructed lexer from another module, simply do this: import lex import lexer plex.input("position = initial + rate*60") while 1: token = plex.token() # Get a token if not token: break # No more tokens ... do whatever ... Assuming that the module 'lexer' has initialized plex as shown above, parsing modules can safely import 'plex' without having to import the rule file or any additional imformation about the scanner you have defined. """ # ----------------------------------------------------------------------------- __version__ = "1.0" import re, types, sys # Exception thrown when invalid token encountered and no default class LexError(Exception): def __init__(self,message,s): self.args = (message,) self.text = s # Token class class LexToken: def __str__(self): return "LexToken(%s,%r,%d)" % (self.type,self.value,self.lineno) def __repr__(self): return str(self) def skip(self,n): try: self._skipn += n except AttributeError: self._skipn = n # ----------------------------------------------------------------------------- # Lexer class # # input() - Store a new string in the lexer # token() - Get the next token # ----------------------------------------------------------------------------- class Lexer: def __init__(self): self.lexre = None # Master regular expression self.lexdata = None # Actual input data (as a string) self.lexpos = 0 # Current position in input text self.lexlen = 0 # Length of the input text self.lexindexfunc = [ ] # Reverse mapping of groups to functions and types self.lexerrorf = None # Error rule (if any) self.lextokens = None # List of valid tokens self.lexignore = None # Ignored characters self.lineno = 1 # Current line number self.debug = 0 # Debugging mode self.optimize = 0 # Optimized mode self.token = self.errtoken self.hack = LexToken() # ------------------------------------------------------------ # input() - Push a new string into the lexer # ------------------------------------------------------------ def input(self,s): if not type(s) in (types.StringType,types.UnicodeType): raise ValueError, "Expected a string or unicode" self.lexdata = s self.lexpos = 0 self.lexlen = len(s) self.token = self.realtoken # Change the token routine to point to realtoken() global token if token == self.errtoken: token = self.token # ------------------------------------------------------------ # errtoken() - Return error if token is called with no data # ------------------------------------------------------------ def errtoken(self): raise RuntimeError, "No input string given with input()" # ------------------------------------------------------------ # token() - Return the next token from the Lexer # # Note: This function has been carefully implemented to be as fast # as possible. Don't make changes unless you really know what # you are doing # ------------------------------------------------------------ def realtoken(self): # Make local copies of frequently referenced attributes lexpos = self.lexpos lexlen = self.lexlen lexignore = self.lexignore lexdata = self.lexdata while lexpos < lexlen: # This code provides some short-circuit code for whitespace, tabs, and other ignored characters if lexdata[lexpos] in lexignore: lexpos += 1 continue # Look for a regular expression match m = self.lexre.match(lexdata,lexpos) if m: i = m.lastindex lexpos = m.end() tok = LexToken() tok.value = m.group() tok.lineno = self.lineno func,tok.type = self.lexindexfunc[i] if not func: self.lexpos = lexpos return tok # If token is processed by a function, call it self.lexpos = lexpos tok.lexer = self newtok = func(tok) self.lineno = tok.lineno # Update line number # Every function must return a token, if nothing, we just move to next token if not newtok: continue # Verify type of the token. If not in the token map, raise an error if not self.optimize: if not self.lextokens.has_key(newtok.type): raise LexError, ("%s:%d. Rule '%s' returned an unknown token type '%s'" % ( func.func_code.co_filename, func.func_code.co_firstlineno, func.__name__, newtok.type),lexdata[lexpos:]) return newtok # No match. Call t_error() if defined. if self.lexerrorf: tok = LexToken() tok.value = self.lexdata[lexpos:] tok.lineno = self.lineno tok.type = "error" tok.lexer = self oldpos = lexpos newtok = self.lexerrorf(tok) lexpos += getattr(tok,"_skipn",0) if oldpos == lexpos: # Error method didn't change text position at all. This is an error. self.lexpos = lexpos raise LexError, ("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:]) if not newtok: continue self.lexpos = lexpos return newtok self.lexpos = lexpos raise LexError, ("No match found", lexdata[lexpos:]) # No more input data self.lexpos = lexpos + 1 return None # ----------------------------------------------------------------------------- # validate_file() # # This checks to see if there are duplicated t_rulename() functions or strings # in the parser input file. This is done using a simple regular expression # match on each line in the filename. # ----------------------------------------------------------------------------- def validate_file(filename): import os.path base,ext = os.path.splitext(filename) if ext != '.py': return 1 # No idea what the file is. Return OK try: f = open(filename) lines = f.readlines() f.close() except IOError: return 1 # Oh well fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(') sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=') counthash = { } linen = 1 noerror = 1 for l in lines: m = fre.match(l) if not m: m = sre.match(l) if m: name = m.group(1) prev = counthash.get(name) if not prev: counthash[name] = linen else: print "%s:%d: Rule %s redefined. Previously defined on line %d" % (filename,linen,name,prev) noerror = 0 linen += 1 return noerror # ----------------------------------------------------------------------------- # lex(module) # # Build all of the regular expression rules from definitions in the supplied module # ----------------------------------------------------------------------------- def lex(module=None,debug=0,optimize=0): ldict = None regex = "" error = 0 files = { } lexer = Lexer() lexer.debug = debug lexer.optimize = optimize if module: if not isinstance(module, types.ModuleType): raise ValueError,"Expected a module" ldict = module.__dict__ else: # No module given. We might be able to get information from the caller. try: raise RuntimeError except RuntimeError: e,b,t = sys.exc_info() f = t.tb_frame f = f.f_back # Walk out to our calling function ldict = f.f_globals # Grab its globals dictionary # Get the tokens map tokens = ldict.get("tokens",None) if not tokens: raise SyntaxError,"lex: module does not define 'tokens'" if not (isinstance(tokens,types.ListType) or isinstance(tokens,types.TupleType)): raise SyntaxError,"lex: tokens must be a list or tuple." # Build a dictionary of valid token names lexer.lextokens = { } if not optimize: # Utility function for verifying tokens def is_identifier(s): for c in s: if not (c.isalnum() or c == '_'): return 0 return 1 for n in tokens: if not is_identifier(n): print "lex: Bad token name '%s'" % n error = 1 if lexer.lextokens.has_key(n): print "lex: Warning. Token '%s' multiply defined." % n lexer.lextokens[n] = None else: for n in tokens: lexer.lextokens[n] = None if debug: print "lex: tokens = '%s'" % lexer.lextokens.keys() # Get a list of symbols with the t_ prefix tsymbols = [f for f in ldict.keys() if f[:2] == 't_'] # Now build up a list of functions and a list of strings fsymbols = [ ] ssymbols = [ ] for f in tsymbols: if isinstance(ldict[f],types.FunctionType): fsymbols.append(ldict[f]) elif isinstance(ldict[f],types.StringType): ssymbols.append((f,ldict[f])) else: print "lex: %s not defined as a function or string" % f error = 1 # Sort the functions by line number fsymbols.sort(lambda x,y: cmp(x.func_code.co_firstlineno,y.func_code.co_firstlineno)) # Sort the strings by regular expression length ssymbols.sort(lambda x,y: (len(x[1]) < len(y[1])) - (len(x[1]) > len(y[1]))) # Check for non-empty symbols if len(fsymbols) == 0 and len(ssymbols) == 0: raise SyntaxError,"lex: no rules of the form t_rulename are defined." # Add all of the rules defined with actions first for f in fsymbols: line = f.func_code.co_firstlineno file = f.func_code.co_filename files[file] = None if not optimize: if f.func_code.co_argcount > 1: print "%s:%d. Rule '%s' has too many arguments." % (file,line,f.__name__) error = 1 continue if f.func_code.co_argcount < 1: print "%s:%d. Rule '%s' requires an argument." % (file,line,f.__name__) error = 1 continue if f.__name__ == 't_ignore': print "%s:%d. Rule '%s' must be defined as a string." % (file,line,f.__name__) error = 1 continue if f.__name__ == 't_error': lexer.lexerrorf = f continue if f.__doc__: if not optimize: try: c = re.compile(f.__doc__, re.VERBOSE|re.UNICODE|re.LOCALE) except re.error,e: print "%s:%d. Invalid regular expression for rule '%s'. %s" % (file,line,f.__name__,e) error = 1 continue if debug: print "lex: Adding rule %s -> '%s'" % (f.__name__,f.__doc__) # Okay. The regular expression seemed okay. Let's append it to the master regular # expression we're building if (regex): regex += "|" regex += "(?P<%s>%s)" % (f.__name__,f.__doc__) else: print "%s:%d. No regular expression defined for rule '%s'" % (file,line,f.__name__) # Now add all of the simple rules for name,r in ssymbols: if name == 't_ignore': lexer.lexignore = r continue if not optimize: if name == 't_error': raise SyntaxError,"lex: Rule 't_error' must be defined as a function" error = 1 continue if not lexer.lextokens.has_key(name[2:]): print "lex: Rule '%s' defined for an unspecified token %s." % (name,name[2:]) error = 1 continue try: c = re.compile(r,re.VERBOSE|re.UNICODE|re.LOCALE) except re.error,e: print "lex: Invalid regular expression for rule '%s'. %s" % (name,e) error = 1 continue if debug: print "lex: Adding rule %s -> '%s'" % (name,r) if regex: regex += "|" regex += "(?P<%s>%s)" % (name,r) if not optimize: for f in files.keys(): if not validate_file(f): error = 1 try: if debug: print "lex: regex = '%s'" % regex lexer.lexre = re.compile(regex, re.VERBOSE|re.UNICODE|re.LOCALE) # Build the index to function map for the matching engine lexer.lexindexfunc = [ None ] * (max(lexer.lexre.groupindex.values())+1) for f,i in lexer.lexre.groupindex.items(): handle = ldict[f] if isinstance(handle,types.FunctionType): lexer.lexindexfunc[i] = (handle,handle.__name__[2:]) else: # If rule was specified as a string, we build an anonymous # callback function to carry out the action lexer.lexindexfunc[i] = (None,f[2:]) except re.error,e: print "lex: Fatal error. Unable to compile regular expression rules. %s" % e error = 1 if error: raise SyntaxError,"lex: Unable to build lexer." if not lexer.lexerrorf: print "lex: Warning. no t_error rule is defined." if not lexer.lexignore: lexer.lexignore = "" # Create global versions of the token() and input() functions global token, input token = lexer.token input = lexer.input return lexer # ----------------------------------------------------------------------------- # run() # # This runs the lexer as a main program # ----------------------------------------------------------------------------- def runmain(lexer=None,data=None): if not data: try: filename = sys.argv[1] f = open(filename) data = f.read() f.close() except IndexError: print "Reading from standard input (type EOF to end):" data = sys.stdin.read() if lexer: _input = lexer.input else: _input = input _input(data) if lexer: _token = lexer.token else: _token = token while 1: tok = _token() if not tok: break print "(%s,'%s',%d)" % (tok.type, tok.value, tok.lineno) ZpgCatalog-0.2/Indexes/PyQueryParser/ParseTree.py0100644000175000001640000000652007702456533021505 0ustar fafhrddatacom########################################################################### # # TextIndexNG The next generation TextIndex for Zope # # This software is governed by a license. See # LICENSE.txt for the terms of this license. # ########################################################################### class BaseNode: """ base class for all nodes """ # __implements__ = IParseTreeNode def __init__(self, v): if isinstance(self.__class__, BaseNode): raise ImplementationError, "don't instantiate BaseNode" self._value = v def getType(self): return self.__class__.__name__ def getValue(self): return self._value def __cmp__(self, node): if self.getType()==node.getType() and \ self.getValue()==node.getValue(): return 0 else: return -1 def __repr__(self): return "%s(%r)" % (self.getType(), self.getValue()) class WordNode(BaseNode): """ normal word """ class GlobNode(BaseNode): """ globbing """ class TruncNode(BaseNode): """ right truncation """ class SubstringNode(BaseNode): """ substring """ class LTruncNode(BaseNode): """ left truncation """ class SimNode(BaseNode): """ similarity """ class NotNode(BaseNode): """ NOT node """ class AndNode(BaseNode): """ AND node """ class OrNode(BaseNode): """ OR node """ class NearNode(BaseNode): """ NEAR node """ class PhraseNode(BaseNode): """ Phrase node """ class RangeNode(BaseNode): """ Range node """ class Evaluator: """ evaluator for a ParseTree instance """ def __init__(self, index): self._index = index def __call__(self, node): if isinstance(node, WordNode): res = self._index.lookupWord( node.getValue() ) return res elif isinstance(node, AndNode): sets = [ self(n) for n in node.getValue() ] return intersectResultSets(sets) elif isinstance(node, OrNode): sets = [ self(n) for n in node.getValue() ] return unionResultSets(sets) elif isinstance(node, GlobNode): return self._index.lookupWordsByPattern( node.getValue() ) elif isinstance(node, TruncNode): return self._index.lookupWordsByTruncation(node.getValue(), right=1) elif isinstance(node, LTruncNode): return self._index.lookupWordsByTruncation(node.getValue(), left=1) elif isinstance(node, SubstringNode): return self._index.lookupWordsBySubstring(node.getValue()) elif isinstance(node, RangeNode): return self._index.lookupRange(node.getValue()[0], node.getValue()[1]) elif isinstance(node, SimNode): return self._index.lookupWordsBySimilarity( node.getValue() ) elif isinstance(node, NotNode): rset = self(node.getValue()) return inverseResultSet(rset, self._index) elif isinstance(node, NearNode): sets = [ self(n) for n in node.getValue() ] return nearResultSets(sets, self._index, distance=self.near_distance) elif isinstance(node, PhraseNode): sets = [ self(n) for n in node.getValue() ] return phraseResultSets(sets, self._index) else: raise ValueError, node ZpgCatalog-0.2/Indexes/PyQueryParser/BaseParser.py0100644000175000001640000000134407702456522021637 0ustar fafhrddatacom########################################################################### # # TextIndexNG The next generation TextIndex for Zope # # This software is governed by a license. See # LICENSE.txt for the terms of this license. # ########################################################################### """base parser class""" class QueryParserError(Exception): """ """ class BaseParser: """ Base class for all parsers """ # __implements__ = ParserInterface id = None parser_description = None def getDescription(self): return self.parser_description def getId(self): return self.id def __call__(self, query, operator=''): return self.parse(query, operator) ZpgCatalog-0.2/Indexes/PyQueryParser/__init__.py0100644000175000001640000000004107467316511021341 0ustar fafhrddatacomfrom PyQueryParser import Parser ZpgCatalog-0.2/Indexes/PyQueryParser/PyQueryParser.py0100644000175000001640000001213707702453766022414 0ustar fafhrddatacom########################################################################### # # TextIndexNG The next generation TextIndex for Zope # # This software is governed by a license. See # LICENSE.txt for the terms of this license. # ########################################################################### ####################################################### # a new native Python QueryParser for TextIndexNG # based on the QueryParser by Sidnei da Silva ####################################################### import sys, re from BaseParser import BaseParser,QueryParserError from ParseTree import * class PhraseElements: """ wrapper class around a list to keep the nodes for a phrase search """ def __init__(self): self.clear() def addElement(self, item): self._list.insert(0, item) def getElements(self): return tuple(self._list) def clear(self): self._list = [] tokens = ( 'STRING' , 'NOT', 'OR', 'AND', 'NEAR', 'QUOTE', 'OPENP', 'CLOSEP', ) t_QUOTE = r'\"' t_OPENP = r'\(' t_CLOSEP = r'\)' t_ignore = '\t' def t_NOT(t): 'NOT\s+|not\s+|\-' return t def t_AND(t): '\s+AND\s+|\s+and\s+' return t def t_OR(t): '\s+OR\s+|\s+or\s+' return t def t_NEAR(t): '\s+NEAR\s+|\s+near\s+' return t def t_STRING(t): r'[^()\s"]+' return t def t_newline(t): r'\n+' t.lineno += t.value.count("\n") def t_error(t): if t.value[0] in [' ']: t.skip(1) else: raise QueryParserError,"Illegal character '%s'" % t.value[0] # Build the lexer import lex lex.lex(debug=0) def p_expr_parens(t): """expr : OPENP expr CLOSEP """ t[0] = t[2] def p_expr_and(t): """expr : expr AND expr """ t[0] = AndNode( (t[1], t[3]) ) def p_expr_or(t): """expr : expr OR expr """ t[0] = OrNode( (t[1], t[3]) ) def p_expr_near(t): """expr : expr NEAR expr """ t[0] = NearNode( (t[1], t[3]) ) def p_expr_noop(t): """expr : expr expr""" t[0] = AndNode( (t[1], t[2])) def p_expr_expr_factor(t): """expr : factor """ t[0] = t[1] def p_expr_expr_factor3(t): """expr : NOT expr""" t[0] = NotNode( t[2] ) def p_expr_expr_factor2(t): """expr : NOT factor """ t[0] = NotNode( t[2] ) def p_factor_string(t): """factor : string""" t[0] = t[1] def p_factor_quote(t): """factor : quotestart qterm quoteend """ t[0] = PhraseNode( phrase_elements.getElements() ) phrase_elements = PhraseElements() def p_qterm_1(t): """ qterm : string qterm""" phrase_elements.addElement( t[1] ) t[0] = [t[1], t[2]] def p_qterm_2(t): """ qterm : string""" phrase_elements.addElement( t[1] ) t[0] = t[1] def p_quotestart_1(t): """ quotestart : QUOTE """ phrase_elements.clear() def p_quoteend_1(t): """ quoteend : QUOTE """ pass # some regexes to distinguish between normal strings # truncated strings and patterns str_regex = re.compile(r'[^%*?()\s"]+$',re.LOCALE|re.UNICODE) range_regex = re.compile(r'[^%*?()\s"]+\.\.[^%*?()\s"]+$',re.LOCALE|re.UNICODE) sim_regex = re.compile(r'[%][^%*?()\s"]+$',re.LOCALE|re.UNICODE) sub_regex = re.compile(r'[*][^%*?()\s"]+[*]$',re.LOCALE|re.UNICODE) rt_regex = re.compile(r'[^%*?()\s"]+[*]$',re.LOCALE|re.UNICODE) lt_regex = re.compile(r'[*][^%*?()\s"]+$',re.LOCALE|re.UNICODE) def p_string(t): """string : STRING""" if range_regex.match(t[1]): t[0] = RangeNode( tuple(t[1].split('..')) ) elif str_regex.match(t[1]): t[0] = WordNode( t[1] ) elif sim_regex.match(t[1]): t[0] = SimNode( t[1][1:] ) elif sub_regex.match(t[1]): t[0] = SubstringNode( t[1][1:-1] ) elif rt_regex.match(t[1]): t[0] = TruncNode( t[1][:-1] ) elif lt_regex.match(t[1]): t[0] = LTruncNode( t[1][1:] ) else: t[0] = GlobNode( t[1] ) def p_error(t): raise QueryParserError,"Syntax error at '%s'" % t.value import yacc try: yacc.yacc(debug=0) except: from zLOG import LOG, ERROR LOG("TextIndexNG", ERROR, "parsetab creation failed") class Parser(BaseParser): id = 'NewQueryParser' parser_description = 'A TextIndex compatible parser (native Python version)' def parse(self, query, operator): try: return yacc.parse( query ) except QueryParserError: raise except: import traceback traceback.print_exc() raise QueryParserError, 'parser failed for query: %s' % query def test(): import os, sys, re,traceback, atexit histfile = os.path.expanduser('~/.pyhist') try: import readline readline.read_history_file(histfile) atexit.register(readline.write_history_file,histfile) except: pass print "entering interactive query mode:" while 1: s = raw_input('> ') print s try: P = Parser() res = P(s) print 'res:',res, type(res) except: traceback.print_exc() if __name__ == '__main__': test() ZpgCatalog-0.2/Indexes/KeywordIndex.py0100644000175000001640000000543607705142707017437 0ustar fafhrddatacom############################################################################# # # Copyright (c) 2003 Nikolay Kim # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## from Globals import DTMLFile from Products.PluginIndexes.common.util import parseIndexRequest import PluggableIndex from FieldIndex import FieldIndex class KeywordIndex(FieldIndex): """Keyword Indexes""" __implements__ = (PluggableIndex.DbPluggableIndexInterface,) meta_type="KeywordIndex" manage_options= ( {'label': 'Settings', 'action': 'manage_main',}, ) query_options = ["query"] index_html = DTMLFile('dtml/index', globals()) manage_workspace = DTMLFile('dtml/manageFieldIndex', globals()) _db_type = 'int[]' def _prepare_datum(self, datum): res = '' try: for el in datum: res += '%d,'%hash(el) except: res += '%d,'%hash(datum) return "'{%s}'" % res[:-1] def _apply_index(self, request, cid='', type=type, None=None): """ """ record = parseIndexRequest(request, self.id, self.query_options) if record.keys==None: return '' name = self.db_id() query = '' opr = None # experimental code for specifing the operator operator = record.get('operator',self.useOperator) if not operator in self.operators : raise RuntimeError,"operator not valid: %s" % escape(operator) if record.get('usage',None): # see if any usage params are sent to field opr = record.usage.lower().split(':') opr, opr_args=opr[0], opr[1:] #print opr, record.keys , record.__dict__ keys = record.keys if len(keys) > 1: s = '' for key in keys: s += '%s|'%(hash(key)) query += "%s @@ '%s'::query_int "%(name, s[:-1]) else: query += "%s @ '{%s}' "%(name, hash(keys[0])) return query def db_index(self): return 'USING gist(%s gist__intbig_ops)'%self.db_id() manage_addKeywordIndexForm = DTMLFile('dtml/addKeywordIndex', globals()) def manage_addKeywordIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None): """Add a Keyword index""" return self.manage_addIndex(id, 'KeywordIndex', extra=None, REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3) ZpgCatalog-0.2/Indexes/PathIndex.py0100644000175000001640000000455607703210735016704 0ustar fafhrddatacom############################################################################## # # Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## from types import StringType, ListType, TupleType import time from Globals import DTMLFile import PluggableIndex from FieldIndex import FieldIndex from Products.ZpgCatalog.utils import sql_quote__ class PathIndex(FieldIndex): """ """ __implements__ = (PluggableIndex.DbPluggableIndexInterface,) meta_type="PathIndex" manage_options= ( {'label': 'Settings', 'action': 'manage_main', 'help': ('PathIndex','PathIndex_Settings.stx')}, ) query_options = ["query","operator"] _db_type = 'varchar(1000)' def index_object(self, obj, threshold=None): """ """ if hasattr(obj,self.id): f = getattr(obj,self.id) try: if callable(f): path = f() else: path = f except: return 0 if not (isinstance(path,StringType) or isinstance(path,TupleType)): raise TypeError, "attribute/method must be/return string or tuple" else: try: path = obj.getPhysicalPath() except: return 0 if type(path) in (ListType,TupleType): path = '/'+ '/'.join(path[1:]) return self.db_id(), "'%s'"%path index_html = DTMLFile('dtml/index', globals()) manage_workspace = DTMLFile('dtml/managePathIndex', globals()) def _prepare_datum(self, datum): return sql_quote__(str(datum)) manage_addPathIndexForm = DTMLFile('dtml/addPathIndex', globals()) def manage_addPathIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None): """Add a path index""" return self.manage_addIndex(id, 'PathIndex', extra=None, \ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3) ZpgCatalog-0.2/Indexes/converters/0040755000175000001640000000000007705454470016641 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/converters/ps.py0100666000175000001640000000140707671055130017630 0ustar fafhrddatacom########################################################################### # # TextIndexNG The next generation TextIndex for Zope # # This software is governed by a license. See # LICENSE.txt for the terms of this license. # ########################################################################### # pdf converter # from Products.TextIndexNG2.BaseConverter import BaseConverter class Converter(BaseConverter): content_type = ('application/postscript',) content_description = "Adobe Postscript Document" depends_on = 'ps2ascii' def convert(self, doc): """Convert postscript data to raw text""" tmp_name = self.saveFile(doc) text = self.execute('ps2ascii %s -' % tmp_name) return text ZpgCatalog-0.2/Indexes/converters/StripTagParser.py0100666000175000001640000000145607654013436022130 0ustar fafhrddatacom########################################################################### # # TextIndexNG The next generation TextIndex for Zope # # This software is governed by a license. See # LICENSE.txt for the terms of this license. # ########################################################################### from sgmllib import SGMLParser # Code taken from Dieter Maurer's CatalogSupport Module # http://www.handshake.de/~dieter/pyprojects/zope # Thank you! class StripTagParser(SGMLParser): '''SGML Parser removing any tags and translating HTML entities.''' from htmlentitydefs import entitydefs data= None def handle_data(self,data): if self.data is None: self.data=[] self.data.append(data) def __str__(self): if self.data is None: return '' return ''.join(self.data) ZpgCatalog-0.2/Indexes/converters/sgml.py0100666000175000001640000000140607671055130020147 0ustar fafhrddatacom########################################################################### # # TextIndexNG The next generation TextIndex for Zope # # This software is governed by a license. See # LICENSE.txt for the terms of this license. # ########################################################################### # a stupid HTML to Ascii converter # from types import StringType from Products.TextIndexNG2.BaseConverter import BaseConverter from StripTagParser import StripTagParser class Converter(BaseConverter): content_type = ('text/sgml',) content_description = "Converter SGML to ASCII" def convert(self, doc): """Convert html data to raw text""" p = StripTagParser() p.feed(doc) p.close() return str(p) ZpgCatalog-0.2/Indexes/converters/stripogram/0040755000175000001640000000000007705454470021030 5ustar fafhrddatacomZpgCatalog-0.2/Indexes/converters/stripogram/html2text.py0100666000175000001640000001045707651536301023337 0ustar fafhrddatacom# Copyright (c) 2001 Chris Withers # # This Software is released under the MIT License: # http://www.opensource.org/licenses/mit-license.html # See license.txt for more details. # # $Id: html2text.py,v 1.1 2003/04/23 16:18:09 ajung Exp $ import sgmllib from string import lower, replace, split, join class HTML2Text(sgmllib.SGMLParser): from htmlentitydefs import entitydefs # replace entitydefs from sgmllib def __init__(self, ignore_tags=(), indent_width=4, page_width=80): sgmllib.SGMLParser.__init__(self) self.result = "" self.indent = 0 self.ol_number = 0 self.page_width=page_width self.inde_width=indent_width self.lines=[] self.line=[] self.ignore_tags = ignore_tags def add_text(self,text): # convert text into words words = split(replace(text,'\n',' ')) self.line.extend(words) def add_break(self): self.lines.append((self.indent,self.line)) self.line=[] def generate(self): # join lines with indents indent_width = self.inde_width page_width = self.page_width out_paras=[] for indent,line in self.lines+[(self.indent,self.line)]: i=indent*indent_width indent_string = i*' ' line_width = page_width-i out_para='' out_line=[] len_out_line=0 for word in line: len_word = len(word) if len_out_line+len_word' def handle_endtag(self, tag): try: while tag != self.openTags[-1] and self.openTags[-1] in self.can_close: self.openTags.pop() if tag==self.openTags[-1]: self.end_tag(self.openTags.pop()) except IndexError: pass def cleanup(self): """ Append missing closing tags """ while self.openTags: tag = self.openTags.pop() if tag not in self.can_close: self.end_tag(tag) def parse_starttag(self,i): try: return HTMLParser.parse_starttag(self,i) except HTMLParseError: try: return piclose.search(self.rawdata,i).end() except AttributeError: return -1 def parse_endtag(self,i): try: return HTMLParser.parse_endtag(self,i) except HTMLParseError: try: return piclose.search(self.rawdata,i).end() except: return -1 def goahead(self,end): # fix incomplete entity and char refs rawdata = self.rawdata i = 0 n = len(rawdata) newdata='' while i < n: j = find(rawdata,'&',i) if j==-1: break newdata = newdata + rawdata[i:j] if charref.match(rawdata, j) or entityref.match(rawdata, j): newdata = newdata + '&' else: newdata = newdata + '&' i = j+1 self.rawdata = newdata + rawdata[i:] # do normal parsing try: return HTMLParser.goahead(self,end) except HTMLParseError: pass ZpgCatalog-0.2/Indexes/converters/stripogram/HTMLParser.py0100666000175000001640000003653407651536300023330 0ustar fafhrddatacom############################################################################## # # This software is released under the Zope Public License (ZPL) Version 1.0 # # Copyright (c) Zope Corportation. All rights reserved. # ############################################################################## """A parser for HTML and XHTML.""" # This file is based on sgmllib.py, but the API is slightly different. # XXX There should be a way to distinguish between PCDATA (parsed # character data -- the normal case), RCDATA (replaceable character # data -- only char and entity references and end tags are special) # and CDATA (character data -- only end tags are special). import re import string # Regular expressions used for parsing interesting_normal = re.compile('[&<]') interesting_cdata = re.compile(r'<(/|\Z)') incomplete = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*|#[0-9]*)?') entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*);') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+);') starttagopen = re.compile('<[a-zA-Z]') piopen = re.compile(r'<\?') piclose = re.compile('>') endtagopen = re.compile(']*>') commentopen = re.compile('