FileStorage123/0040755000175100001440000000000007740563644012553 5ustar rocheusersFileStorage123/ClientStorage.py0100644000175100001440000000477207732621724015672 0ustar rocheusersimport os import cPickle from ZEO.ClientStorage import ClientStorage as ZEOClientStorage from FileStorage123 import ServerStub MB = 1024**2 class ClientStorage(ZEOClientStorage): """ Extends ClientStorage to retrieve sequential ids from a ZEO server """ StorageServerStubClass = ServerStub.StorageServer def __init__(self, addr, storage='1', cache_size=20 * MB, name='', client=None, debug=0, var=None, min_disconnect_poll=5, max_disconnect_poll=300, wait_for_server_on_startup=None, # deprecated alias for wait wait=None, # defaults to 1 read_only=0, read_only_fallback=0): ZEOClientStorage.__init__(self, addr, storage, cache_size, name, client, debug, var, min_disconnect_poll, max_disconnect_poll, wait_for_server_on_startup, wait, read_only, read_only_fallback) # CLIENT_HOME and INSTANCE_HOME are builtins set by App.FindHomes if var is None: try: var = CLIENT_HOME except: try: var = os.path.join(INSTANCE_HOME, 'var') except: var = os.getcwd() self._ids_filename = os.path.join(var, '_ids') def get_id_block(self, key): if not hasattr(self, '_id_blocks'): try: f = open(self._ids_filename, 'r+') self._id_blocks = cPickle.Unpickler(f).load() except: f = open(self._ids_filename, 'w+') self._id_blocks = {} f.close() if not self._id_blocks.has_key(key): self._id_blocks[key] = [] return self._id_blocks[key] def set_id_block(self): f = open(self._ids_filename, 'r+') p = cPickle.Pickler(f) p.dump(self._id_blocks) f.close() def generate_id(self, key=""): """Storage API: return a new id for a given key.""" if self._is_read_only: raise POSException.ReadOnlyError() # avoid multiple id requests to server at the same time self._oid_lock.acquire() try: block = self.get_id_block(key) if not block: self._id_blocks[key] = self._server.generate_ids(key) self._id_blocks[key].reverse() new_id = self._id_blocks[key].pop() self.set_id_block() return new_id finally: self._oid_lock.release() FileStorage123/FileStorage.py0100644000175100001440000000247507732621724015331 0ustar rocheusersfrom ZODB.FileStorage import FileStorage as ZODBFileStorage from cPickle import Pickler, Unpickler class FileStorage(ZODBFileStorage): """ Extends FileStorage with a counters that return sequential numeric integers. """ def getCounter(self, key): if not hasattr(self, '_counters'): # initialise counter try: f = open(self._file_name + '.counter', 'r+') self._counters = Unpickler(f).load() except: from zLOG import LOG; LOG('INFO', 0, '', 'FileStorage.py> Creating counter') #DBG f = open(self._file_name + '.counter', 'w+') self._counters = {} f.close() if not self._counters.has_key(key): self._counters[key] = 0 return self._counters[key] def setCounter(self, key, count): self._counters[key] = count f = open(self._file_name + '.counter', 'r+') p = Pickler(f) p.dump(self._counters) f.close() def generate_id(self, key): if self._is_read_only: raise POSException.ReadOnlyError() self._lock_acquire() try: count = self.getCounter(key) + 1 self.setCounter(key, count) return count finally: self._lock_release() FileStorage123/ServerStub.py0100644000175100001440000000044307727065346015230 0ustar rocheusersfrom ZEO.ServerStub import StorageServer as ZEOStorageServer class StorageServer(ZEOStorageServer): def generate_ids(self, key, n=None): if n is None: return self.rpc.call('generate_ids', key) else: return self.rpc.call('generate_ids', key, n) FileStorage123/StorageServer.py0100644000175100001440000000063607727065346015723 0ustar rocheusersfrom ZEO.StorageServer import StorageServer as ZEOStorageServer class StorageServer(ZEOStorageServer): def generate_ids(self, key="", n=100): """ Return a sequence of n new ids for 'key', where n defaults to 100 """ if self.read_only: raise ReadOnlyError() if n <= 0: n = 1 return [self.storage.generate_id(key) for i in range(n)] FileStorage123/TmpStorePatch.py0100644000175100001440000000061007730575454015655 0ustar rocheusers'''Patches TmpStore to have generate_id method. ''' from ZODB.TmpStore import TmpStore try: def generate_id(self, key): return self._db._storage.generate_id(key) # Apply the patch. TmpStore.generate_id = generate_id from zLOG import LOG, INFO LOG('FileStorage123', INFO, 'Applied generate_id patch to TmpStore') except: import traceback traceback.print_exc() FileStorage123/__init__.py0100644000175100001440000000002507730575454014657 0ustar rocheusersimport TmpStorePatch FileStorage123/start.py0100644000175100001440000002354107730575454014265 0ustar rocheusers############################################################################## # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## """Start the ZEO storage server.""" from __future__ import nested_scopes import sys, os, getopt import types import errno import socket def directory(p, n=1): d = os.path.abspath(p) while n: d = os.path.split(d)[0] if not d or d == '.': d = os.getcwd() n -= 1 return d def get_storage(m, n, cache={}): p = sys.path d, m = os.path.split(m) if m.endswith('.py'): m = m[:-3] im = cache.get((d, m)) if im is None: if d: p = [d] + p import imp im = imp.find_module(m, p) im = imp.load_module(m, *im) cache[(d, m)] = im return getattr(im, n) def set_uid(arg): """Try to set uid and gid based on -u argument. This will only work if this script is run by root. """ try: import pwd except ImportError: LOG('ZEO/start.py', INFO, ("Can't set uid to %s." "pwd module is not available." % arg)) return try: gid = None try: arg = int(arg) except: # conversion could raise all sorts of errors uid = pwd.getpwnam(arg)[2] gid = pwd.getpwnam(arg)[3] else: uid = pwd.getpwuid(arg)[2] gid = pwd.getpwuid(arg)[3] if gid is not None: try: os.setgid(gid) except OSError: pass try: os.setuid(uid) except OSError: pass except KeyError: LOG('ZEO/start.py', ERROR, ("can't find uid %s" % arg)) def setup_signals(storages): try: import signal except ImportError: return if hasattr(signal, 'SIGXFSZ'): signal.signal(signal.SIGXFSZ, signal.SIG_IGN) if hasattr(signal, 'SIGTERM'): signal.signal(signal.SIGTERM, lambda sig, frame: shutdown(storages)) if hasattr(signal, 'SIGHUP'): signal.signal(signal.SIGHUP, lambda sig, frame: shutdown(storages, 0)) if hasattr(signal, 'SIGUSR2'): signal.signal(signal.SIGUSR2, rotate_logs_handler) def main(argv): me = argv[0] sys.path.insert(0, directory(me, 2)) global LOG, INFO, ERROR from zLOG import LOG, INFO, WARNING, ERROR, PANIC from ZEO.util import Environment env = Environment(me) # XXX hack for profiling support global unix, storages, asyncore args = [] last = '' for a in argv[1:]: if (a[:1] != '-' and a.find('=') > 0 and last != '-S'): # lame, sorry a = a.split("=") os.environ[a[0]] = "=".join(a[1:]) continue args.append(a) last = a usage="""%s [options] [filename] where options are: -D -- Run in debug mode -d -- Set STUPD_LOG_SEVERITY to -300 -U -- Unix-domain socket file to listen on -u username or uid number The username to run the ZEO server as. You may want to run the ZEO server as 'nobody' or some other user with limited resouces. The only works under Unix, and if ZServer is started by root. -p port -- port to listen on -h adddress -- host address to listen on -s -- Don't use zdeamon -S storage_name=module_path:attr_name -- A storage specification where: storage_name -- is the storage name used in the ZEO protocol. This is the name that you give as the optional 'storage' keyword argument to the ClientStorage constructor. module_path -- This is the path to a Python module that defines the storage object(s) to be served. The module path should omit the prefix (e.g. '.py'). attr_name -- This is the name to which the storage object is assigned in the module. -t timeout -- transaction timeout in seconds (default no timeout) -P file -- Run under profile and dump output to file. Implies the -s flag. if no file name is specified, then %s is used. """ % (me, env.fs) try: opts, args = getopt.getopt(args, 'p:Dh:U:sS:t:u:P:d') except getopt.error, msg: print usage print msg sys.exit(1) port = None debug = 0 host = '' unix = None Z = 1 UID = 'nobody' prof = None detailed = 0 fs = None transaction_timeout = None for o, v in opts: if o =='-p': port = int(v) elif o =='-h': host = v elif o =='-U': unix = v elif o =='-u': UID = v elif o =='-D': debug = 1 elif o =='-d': detailed = 1 elif o =='-s': Z = 0 elif o =='-t': transaction_timeout = float(v) elif o =='-P': prof = v if prof: Z = 0 if port is None and unix is None: print usage print 'No port specified.' sys.exit(1) if args: if len(args) > 1: print usage print 'Unrecognizd arguments: ', " ".join(args[1:]) sys.exit(1) fs = args[0] if debug: os.environ['Z_DEBUG_MODE'] = '1' if detailed: os.environ['STUPID_LOG_SEVERITY'] = '-300' set_uid(UID) if Z: try: import posix except: pass else: import zdaemon zdaemon.run(sys.argv, env.zeo_pid) try: if Z: # Change current directory (for core dumps etc.) try: os.chdir(env.var) except os.error: LOG('ZEO/start.py', WARNING, "Couldn't chdir to %s" % env.var) else: LOG('ZEO/start.py', INFO, "Changed directory to %s" % env.var) import FileStorage123.StorageServer, asyncore storages = {} for o, v in opts: if o == '-S': n, m = v.split("=", 1) if m.find(":") >= 0: # we got an attribute name m, a = m.split(':') else: # attribute name must be same as storage name a=n storages[n]=get_storage(m,a) if not storages: from FileStorage123.FileStorage import FileStorage storages['1'] = FileStorage(fs or env.fs) # Try to set up a signal handler setup_signals(storages) items = storages.items() items.sort() for kv in items: LOG('ZEO/start.py', INFO, 'Serving %s:\t%s' % kv) if not unix: unix = host, port FileStorage123.StorageServer.StorageServer( unix, storages, transaction_timeout=transaction_timeout) if not Z: try: pid = os.getpid() except: pass # getpid not supported else: f = open(env.zeo_pid, 'w') f.write("%s\n" % pid) f.close() except: # Log startup exception and tell zdaemon not to restart us. info = sys.exc_info() try: LOG("ZEO/start.py", PANIC, "Startup exception", error=info) except: pass import traceback traceback.print_exception(*info) sys.exit(0) try: try: while 1: try: asyncore.loop() except (socket.error, OSError), err: if err[0] != errno.EBADF: raise finally: if os.path.isfile(env.zeo_pid): os.unlink(env.zeo_pid) except SystemExit: raise except: info = sys.exc_info() try: LOG("ZEO/start.py", PANIC, "Unexpected error", error=info) except: pass import traceback traceback.print_exception(*info) sys.exit(1) def rotate_logs(): import zLOG # There hasn't been a clear way to reinitialize the MinimalLogger. # I'll checkin the public initialize() method soon, but also try some # other strategies for older Zope installs :-(. init = getattr(zLOG, 'initialize', None) if init is not None: init() else: # This will work if the minimal logger is in use, but not if some # other logger is active. import zLOG.MinimalLogger zLOG.MinimalLogger._log.initialize() zLOG.LOG("ZEO/start.py", zLOG.INFO, "Caught USR2, logs rotated") def rotate_logs_handler(signum, frame): rotate_logs() def shutdown(storages, die=1): LOG("ZEO/start.py", INFO, "Received signal") import asyncore # Do this twice, in case we got some more connections # while going through the loop. This is really sort of # unnecessary, since we now use so_reuseaddr. for ignored in 1,2: for socket in asyncore.socket_map.values(): try: socket.close() except: pass for storage in storages.values(): try: storage.close() finally: pass try: s = die and "shutdown" or "restart" LOG('ZEO/start.py', INFO, "Shutting down (%s)" % s) except: pass if die: sys.exit(0) else: sys.exit(1) if __name__=='__main__': main(sys.argv)