From 32a467cec5e616b70b744e55ed162a201be67bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Sun, 23 Feb 2020 22:48:56 -0500 Subject: [PATCH] WIP --- libs2/anydbm.py | 85 ---------------- libs2/dbhash.py | 18 ---- libs2/dumbdbm.py | 249 ----------------------------------------------- 3 files changed, 352 deletions(-) delete mode 100644 libs2/anydbm.py delete mode 100644 libs2/dbhash.py delete mode 100644 libs2/dumbdbm.py diff --git a/libs2/anydbm.py b/libs2/anydbm.py deleted file mode 100644 index ba7e90510..000000000 --- a/libs2/anydbm.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Generic interface to all dbm clones. - -Instead of - - import dbm - d = dbm.open(file, 'w', 0666) - -use - - import anydbm - d = anydbm.open(file, 'w') - -The returned object is a dbhash, gdbm, dbm or dumbdbm object, -dependent on the type of database being opened (determined by whichdb -module) in the case of an existing dbm. If the dbm does not exist and -the create or new flag ('c' or 'n') was specified, the dbm type will -be determined by the availability of the modules (tested in the above -order). - -It has the following interface (key and data are strings): - - d[key] = data # store data at key (may override data at - # existing key) - data = d[key] # retrieve data at key (raise KeyError if no - # such key) - del d[key] # delete data stored at key (raises KeyError - # if no such key) - flag = key in d # true if the key exists - list = d.keys() # return a list of all existing keys (slow!) - -Future versions may change the order in which implementations are -tested for existence, and add interfaces to other dbm-like -implementations. -""" - -class error(Exception): - pass - -_names = ['dbhash', 'gdbm', 'dbm', 'dumbdbm'] -_errors = [error] -_defaultmod = None - -for _name in _names: - try: - _mod = __import__(_name) - except ImportError: - continue - if not _defaultmod: - _defaultmod = _mod - _errors.append(_mod.error) - -if not _defaultmod: - raise ImportError, "no dbm clone found; tried %s" % _names - -error = tuple(_errors) - -def open(file, flag='r', mode=0666): - """Open or create database at path given by *file*. - - Optional argument *flag* can be 'r' (default) for read-only access, 'w' - for read-write access of an existing database, 'c' for read-write access - to a new or existing database, and 'n' for read-write access to a new - database. - - Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it - only if it doesn't exist; and 'n' always creates a new database. - """ - - # guess the type of an existing database - from whichdb import whichdb - result=whichdb(file) - if result is None: - # db doesn't exist - if 'c' in flag or 'n' in flag: - # file doesn't exist and the new - # flag was used so use default type - mod = _defaultmod - else: - raise error, "need 'c' or 'n' flag to open new db" - elif result == "": - # db type cannot be determined - raise error, "db type could not be determined" - else: - mod = __import__(result) - return mod.open(file, flag, mode) diff --git a/libs2/dbhash.py b/libs2/dbhash.py deleted file mode 100644 index a5d5375ba..000000000 --- a/libs2/dbhash.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Provide a (g)dbm-compatible interface to bsddb.hashopen.""" - -import sys -import warnings -warnings.warnpy3k("in 3.x, the dbhash module has been removed", stacklevel=2) -try: - import bsddb -except ImportError: - # prevent a second import of this module from spuriously succeeding - del sys.modules[__name__] - raise - -__all__ = ["error","open"] - -error = bsddb.error # Exported for anydbm - -def open(file, flag = 'r', mode=0666): - return bsddb.hashopen(file, flag, mode) diff --git a/libs2/dumbdbm.py b/libs2/dumbdbm.py deleted file mode 100644 index 8ef7e7958..000000000 --- a/libs2/dumbdbm.py +++ /dev/null @@ -1,249 +0,0 @@ -"""A dumb and slow but simple dbm clone. - -For database spam, spam.dir contains the index (a text file), -spam.bak *may* contain a backup of the index (also a text file), -while spam.dat contains the data (a binary file). - -XXX TO DO: - -- seems to contain a bug when updating... - -- reclaim free space (currently, space once occupied by deleted or expanded -items is never reused) - -- support concurrent access (currently, if two processes take turns making -updates, they can mess up the index) - -- support efficient access to large databases (currently, the whole index -is read when the database is opened, and some updates rewrite the whole index) - -- support opening for read-only (flag = 'm') - -""" - -import ast as _ast -import os as _os -import __builtin__ -import UserDict - -_open = __builtin__.open - -_BLOCKSIZE = 512 - -error = IOError # For anydbm - -class _Database(UserDict.DictMixin): - - # The on-disk directory and data files can remain in mutually - # inconsistent states for an arbitrarily long time (see comments - # at the end of __setitem__). This is only repaired when _commit() - # gets called. One place _commit() gets called is from __del__(), - # and if that occurs at program shutdown time, module globals may - # already have gotten rebound to None. Since it's crucial that - # _commit() finish successfully, we can't ignore shutdown races - # here, and _commit() must not reference any globals. - _os = _os # for _commit() - _open = _open # for _commit() - - def __init__(self, filebasename, mode): - self._mode = mode - - # The directory file is a text file. Each line looks like - # "%r, (%d, %d)\n" % (key, pos, siz) - # where key is the string key, pos is the offset into the dat - # file of the associated value's first byte, and siz is the number - # of bytes in the associated value. - self._dirfile = filebasename + _os.extsep + 'dir' - - # The data file is a binary file pointed into by the directory - # file, and holds the values associated with keys. Each value - # begins at a _BLOCKSIZE-aligned byte offset, and is a raw - # binary 8-bit string value. - self._datfile = filebasename + _os.extsep + 'dat' - self._bakfile = filebasename + _os.extsep + 'bak' - - # The index is an in-memory dict, mirroring the directory file. - self._index = None # maps keys to (pos, siz) pairs - - # Mod by Jack: create data file if needed - try: - f = _open(self._datfile, 'r') - except IOError: - with _open(self._datfile, 'w') as f: - self._chmod(self._datfile) - else: - f.close() - self._update() - - # Read directory file into the in-memory index dict. - def _update(self): - self._index = {} - try: - f = _open(self._dirfile) - except IOError: - pass - else: - with f: - for line in f: - line = line.rstrip() - key, pos_and_siz_pair = _ast.literal_eval(line) - self._index[key] = pos_and_siz_pair - - # Write the index dict to the directory file. The original directory - # file (if any) is renamed with a .bak extension first. If a .bak - # file currently exists, it's deleted. - def _commit(self): - # CAUTION: It's vital that _commit() succeed, and _commit() can - # be called from __del__(). Therefore we must never reference a - # global in this routine. - if self._index is None: - return # nothing to do - - try: - self._os.unlink(self._bakfile) - except self._os.error: - pass - - try: - self._os.rename(self._dirfile, self._bakfile) - except self._os.error: - pass - - with self._open(self._dirfile, 'w') as f: - self._chmod(self._dirfile) - for key, pos_and_siz_pair in self._index.iteritems(): - f.write("%r, %r\n" % (key, pos_and_siz_pair)) - - sync = _commit - - def __getitem__(self, key): - pos, siz = self._index[key] # may raise KeyError - with _open(self._datfile, 'rb') as f: - f.seek(pos) - dat = f.read(siz) - return dat - - # Append val to the data file, starting at a _BLOCKSIZE-aligned - # offset. The data file is first padded with NUL bytes (if needed) - # to get to an aligned offset. Return pair - # (starting offset of val, len(val)) - def _addval(self, val): - with _open(self._datfile, 'rb+') as f: - f.seek(0, 2) - pos = int(f.tell()) - npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE - f.write('\0'*(npos-pos)) - pos = npos - f.write(val) - return (pos, len(val)) - - # Write val to the data file, starting at offset pos. The caller - # is responsible for ensuring that there's enough room starting at - # pos to hold val, without overwriting some other value. Return - # pair (pos, len(val)). - def _setval(self, pos, val): - with _open(self._datfile, 'rb+') as f: - f.seek(pos) - f.write(val) - return (pos, len(val)) - - # key is a new key whose associated value starts in the data file - # at offset pos and with length siz. Add an index record to - # the in-memory index dict, and append one to the directory file. - def _addkey(self, key, pos_and_siz_pair): - self._index[key] = pos_and_siz_pair - with _open(self._dirfile, 'a') as f: - self._chmod(self._dirfile) - f.write("%r, %r\n" % (key, pos_and_siz_pair)) - - def __setitem__(self, key, val): - if not type(key) == type('') == type(val): - raise TypeError, "keys and values must be strings" - if key not in self._index: - self._addkey(key, self._addval(val)) - else: - # See whether the new value is small enough to fit in the - # (padded) space currently occupied by the old value. - pos, siz = self._index[key] - oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE - newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE - if newblocks <= oldblocks: - self._index[key] = self._setval(pos, val) - else: - # The new value doesn't fit in the (padded) space used - # by the old value. The blocks used by the old value are - # forever lost. - self._index[key] = self._addval(val) - - # Note that _index may be out of synch with the directory - # file now: _setval() and _addval() don't update the directory - # file. This also means that the on-disk directory and data - # files are in a mutually inconsistent state, and they'll - # remain that way until _commit() is called. Note that this - # is a disaster (for the database) if the program crashes - # (so that _commit() never gets called). - - def __delitem__(self, key): - # The blocks used by the associated value are lost. - del self._index[key] - # XXX It's unclear why we do a _commit() here (the code always - # XXX has, so I'm not changing it). _setitem__ doesn't try to - # XXX keep the directory file in synch. Why should we? Or - # XXX why shouldn't __setitem__? - self._commit() - - def keys(self): - return self._index.keys() - - def has_key(self, key): - return key in self._index - - def __contains__(self, key): - return key in self._index - - def iterkeys(self): - return self._index.iterkeys() - __iter__ = iterkeys - - def __len__(self): - return len(self._index) - - def close(self): - try: - self._commit() - finally: - self._index = self._datfile = self._dirfile = self._bakfile = None - - __del__ = close - - def _chmod (self, file): - if hasattr(self._os, 'chmod'): - self._os.chmod(file, self._mode) - - -def open(file, flag=None, mode=0666): - """Open the database file, filename, and return corresponding object. - - The flag argument, used to control how the database is opened in the - other DBM implementations, is ignored in the dumbdbm module; the - database is always opened for update, and will be created if it does - not exist. - - The optional mode argument is the UNIX mode of the file, used only when - the database has to be created. It defaults to octal code 0666 (and - will be modified by the prevailing umask). - - """ - # flag argument is currently ignored - - # Modify mode depending on the umask - try: - um = _os.umask(0) - _os.umask(um) - except AttributeError: - pass - else: - # Turn off any bits that are set in the umask - mode = mode & (~um) - - return _Database(file, mode)