diff options
Diffstat (limited to 'python/rpmdb')
| -rw-r--r-- | python/rpmdb/__init__.py | 142 | ||||
| -rw-r--r-- | python/rpmdb/db.py | 4 | ||||
| -rw-r--r-- | python/rpmdb/dbobj.py | 10 | ||||
| -rw-r--r-- | python/rpmdb/dbtables.py | 5 | ||||
| -rw-r--r-- | python/rpmdb/test/test_associate.py | 2 | ||||
| -rw-r--r-- | python/rpmdb/test/test_basics.py | 135 | ||||
| -rw-r--r-- | python/rpmdb/test/test_compat.py | 2 | ||||
| -rw-r--r-- | python/rpmdb/test/test_dbtables.py | 2 | ||||
| -rw-r--r-- | python/rpmdb/test/test_join.py | 106 | ||||
| -rw-r--r-- | python/rpmdb/test/test_thread.py | 4 |
10 files changed, 353 insertions, 59 deletions
diff --git a/python/rpmdb/__init__.py b/python/rpmdb/__init__.py index 64eb5a97c..860d0153b 100644 --- a/python/rpmdb/__init__.py +++ b/python/rpmdb/__init__.py @@ -33,7 +33,7 @@ #---------------------------------------------------------------------- -"""Support for BerkeleyDB 3.1 through 4.1. +"""Support for BerkeleyDB 3.2 through 4.2. """ try: @@ -52,23 +52,143 @@ error = db.DBError # So bsddb.error will mean something... #---------------------------------------------------------------------- +import sys + +# for backwards compatibility with python versions older than 2.3, the +# iterator interface is dynamically defined and added using a mixin +# class. old python can't tokenize it due to the yield keyword. +if sys.version >= '2.3': + exec """ +import UserDict +from weakref import ref +class _iter_mixin(UserDict.DictMixin): + def _make_iter_cursor(self): + cur = self.db.cursor() + key = id(cur) + self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key)) + return cur + + def _gen_cref_cleaner(self, key): + # use generate the function for the weakref callback here + # to ensure that we do not hold a strict reference to cur + # in the callback. + return lambda ref: self._cursor_refs.pop(key, None) + + def __iter__(self): + try: + cur = self._make_iter_cursor() + + # FIXME-20031102-greg: race condition. cursor could + # be closed by another thread before this call. + + # since we're only returning keys, we call the cursor + # methods with flags=0, dlen=0, dofs=0 + key = cur.first(0,0,0)[0] + yield key + + next = cur.next + while 1: + try: + key = next(0,0,0)[0] + yield key + except _bsddb.DBCursorClosedError: + cur = self._make_iter_cursor() + # FIXME-20031101-greg: race condition. cursor could + # be closed by another thread before this call. + cur.set(key,0,0,0) + next = cur.next + except _bsddb.DBNotFoundError: + return + except _bsddb.DBCursorClosedError: + # the database was modified during iteration. abort. + return + + def iteritems(self): + try: + cur = self._make_iter_cursor() + + # FIXME-20031102-greg: race condition. cursor could + # be closed by another thread before this call. + + kv = cur.first() + key = kv[0] + yield kv + + next = cur.next + while 1: + try: + kv = next() + key = kv[0] + yield kv + except _bsddb.DBCursorClosedError: + cur = self._make_iter_cursor() + # FIXME-20031101-greg: race condition. cursor could + # be closed by another thread before this call. + cur.set(key,0,0,0) + next = cur.next + except _bsddb.DBNotFoundError: + return + except _bsddb.DBCursorClosedError: + # the database was modified during iteration. abort. + return +""" +else: + class _iter_mixin: pass + -class _DBWithCursor: +class _DBWithCursor(_iter_mixin): """ A simple wrapper around DB that makes it look like the bsddbobject in the old module. It uses a cursor as needed to provide DB traversal. """ def __init__(self, db): self.db = db - self.dbc = None self.db.set_get_returns_none(0) + # FIXME-20031101-greg: I believe there is still the potential + # for deadlocks in a multithreaded environment if someone + # attempts to use the any of the cursor interfaces in one + # thread while doing a put or delete in another thread. The + # reason is that _checkCursor and _closeCursors are not atomic + # operations. Doing our own locking around self.dbc, + # self.saved_dbc_key and self._cursor_refs could prevent this. + # TODO: A test case demonstrating the problem needs to be written. + + # self.dbc is a DBCursor object used to implement the + # first/next/previous/last/set_location methods. + self.dbc = None + self.saved_dbc_key = None + + # a collection of all DBCursor objects currently allocated + # by the _iter_mixin interface. + self._cursor_refs = {} + def __del__(self): self.close() def _checkCursor(self): if self.dbc is None: self.dbc = self.db.cursor() + if self.saved_dbc_key is not None: + self.dbc.set(self.saved_dbc_key) + self.saved_dbc_key = None + + # This method is needed for all non-cursor DB calls to avoid + # BerkeleyDB deadlocks (due to being opened with DB_INIT_LOCK + # and DB_THREAD to be thread safe) when intermixing database + # operations that use the cursor internally with those that don't. + def _closeCursors(self, save=True): + if self.dbc: + c = self.dbc + self.dbc = None + if save: + self.saved_dbc_key = c.current(0,0,0)[0] + c.close() + del c + for cref in self._cursor_refs.values(): + c = cref() + if c is not None: + c.close() def _checkOpen(self): if self.db is None: @@ -87,13 +207,16 @@ class _DBWithCursor: def __setitem__(self, key, value): self._checkOpen() + self._closeCursors() self.db[key] = value def __delitem__(self, key): self._checkOpen() + self._closeCursors() del self.db[key] def close(self): + self._closeCursors(save=False) if self.dbc is not None: self.dbc.close() v = 0 @@ -152,7 +275,8 @@ def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None, cachesize=None, lorder=None, hflags=0): flags = _checkflag(flag) - d = db.DB() + e = _openDBEnv() + d = db.DB(e) d.set_flags(hflags) if cachesize is not None: d.set_cachesize(0, cachesize) if pgsize is not None: d.set_pagesize(pgsize) @@ -169,7 +293,8 @@ def btopen(file, flag='c', mode=0666, pgsize=None, lorder=None): flags = _checkflag(flag) - d = db.DB() + e = _openDBEnv() + d = db.DB(e) if cachesize is not None: d.set_cachesize(0, cachesize) if pgsize is not None: d.set_pagesize(pgsize) if lorder is not None: d.set_lorder(lorder) @@ -187,7 +312,8 @@ def rnopen(file, flag='c', mode=0666, rlen=None, delim=None, source=None, pad=None): flags = _checkflag(flag) - d = db.DB() + e = _openDBEnv() + d = db.DB(e) if cachesize is not None: d.set_cachesize(0, cachesize) if pgsize is not None: d.set_pagesize(pgsize) if lorder is not None: d.set_lorder(lorder) @@ -201,6 +327,10 @@ def rnopen(file, flag='c', mode=0666, #---------------------------------------------------------------------- +def _openDBEnv(): + e = db.DBEnv() + e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL) + return e def _checkflag(flag): if flag == 'r': diff --git a/python/rpmdb/db.py b/python/rpmdb/db.py index 3bf5f8e35..5ac0fa05f 100644 --- a/python/rpmdb/db.py +++ b/python/rpmdb/db.py @@ -40,5 +40,5 @@ from _rpmdb import * from _rpmdb import __version__ -if version() < (3, 1, 0): - raise ImportError, "BerkeleyDB 3.x symbols not found. Perhaps python was statically linked with an older version?" +if version() < (3, 2, 0): + raise ImportError, "correct BerkeleyDB symbols not found. Perhaps python was statically linked with an older version?" diff --git a/python/rpmdb/dbobj.py b/python/rpmdb/dbobj.py index b2632a1a0..abda657ba 100644 --- a/python/rpmdb/dbobj.py +++ b/python/rpmdb/dbobj.py @@ -15,6 +15,12 @@ # implied. # +# +# TODO it would be *really nice* to have an automatic shadow class populator +# so that new methods don't need to be added here manually after being +# added to _bsddb.c. +# + import db try: @@ -33,6 +39,8 @@ class DBEnv: return apply(self._cobj.open, args, kwargs) def remove(self, *args, **kwargs): return apply(self._cobj.remove, args, kwargs) + def set_shm_key(self, *args, **kwargs): + return apply(self._cobj.set_shm_key, args, kwargs) def set_cachesize(self, *args, **kwargs): return apply(self._cobj.set_cachesize, args, kwargs) def set_data_dir(self, *args, **kwargs): @@ -57,6 +65,8 @@ class DBEnv: return apply(self._cobj.set_lk_max_objects, args, kwargs) def set_mp_mmapsize(self, *args, **kwargs): return apply(self._cobj.set_mp_mmapsize, args, kwargs) + def set_timeout(self, *args, **kwargs): + return apply(self._cobj.set_timeout, args, kwargs) def set_tmp_dir(self, *args, **kwargs): return apply(self._cobj.set_tmp_dir, args, kwargs) def txn_begin(self, *args, **kwargs): diff --git a/python/rpmdb/dbtables.py b/python/rpmdb/dbtables.py index 8947c5434..e5be5f115 100644 --- a/python/rpmdb/dbtables.py +++ b/python/rpmdb/dbtables.py @@ -15,7 +15,7 @@ # This provides a simple database table interface built on top of # the Python BerkeleyDB 3 interface. # -_cvsid = 'Id: dbtables.py,v 1.7 2003/01/28 17:20:42 bwarsaw Exp ' +_cvsid = 'Id: dbtables.py,v 1.9 2003/09/21 00:08:14 greg Exp ' import re import sys @@ -150,6 +150,9 @@ class bsdTableDB : if truncate: myflags |= DB_TRUNCATE self.db = DB(self.env) + # this code relies on DBCursor.set* methods to raise exceptions + # rather than returning None + self.db.set_get_returns_none(1) # allow duplicate entries [warning: be careful w/ metadata] self.db.set_flags(DB_DUP) self.db.open(filename, DB_BTREE, dbflags | myflags, mode) diff --git a/python/rpmdb/test/test_associate.py b/python/rpmdb/test/test_associate.py index be6ef610d..1dbae37da 100644 --- a/python/rpmdb/test/test_associate.py +++ b/python/rpmdb/test/test_associate.py @@ -1,5 +1,5 @@ """ -TestCases for multi-threaded access to a DB. +TestCases for DB.associate. """ import sys, os, string diff --git a/python/rpmdb/test/test_basics.py b/python/rpmdb/test/test_basics.py index e35002b3f..ba629e517 100644 --- a/python/rpmdb/test/test_basics.py +++ b/python/rpmdb/test/test_basics.py @@ -44,6 +44,8 @@ class BasicTestCase(unittest.TestCase): envflags = 0 envsetflags = 0 + _numKeys = 1002 # PRIVATE. NOTE: must be an even value + def setUp(self): if self.useEnv: homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home') @@ -101,17 +103,23 @@ class BasicTestCase(unittest.TestCase): - def populateDB(self): + def populateDB(self, _txn=None): d = self.d - for x in range(500): - key = '%04d' % (1000 - x) # insert keys in reverse order + + for x in range(self._numKeys/2): + key = '%04d' % (self._numKeys - x) # insert keys in reverse order data = self.makeData(key) - d.put(key, data) + d.put(key, data, _txn) - for x in range(500): + d.put('empty value', '', _txn) + + for x in range(self._numKeys/2-1): key = '%04d' % x # and now some in forward order data = self.makeData(key) - d.put(key, data) + d.put(key, data, _txn) + + if _txn: + _txn.commit() num = len(d) if verbose: @@ -231,20 +239,20 @@ class BasicTestCase(unittest.TestCase): if verbose: print data - assert len(d) == 1000 + assert len(d) == self._numKeys keys = d.keys() - assert len(keys) == 1000 + assert len(keys) == self._numKeys assert type(keys) == type([]) d['new record'] = 'a new record' - assert len(d) == 1001 + assert len(d) == self._numKeys+1 keys = d.keys() - assert len(keys) == 1001 + assert len(keys) == self._numKeys+1 d['new record'] = 'a replacement record' - assert len(d) == 1001 + assert len(d) == self._numKeys+1 keys = d.keys() - assert len(keys) == 1001 + assert len(keys) == self._numKeys+1 if verbose: print "the first 10 keys are:" @@ -256,7 +264,7 @@ class BasicTestCase(unittest.TestCase): assert d.has_key('spam') == 0 items = d.items() - assert len(items) == 1001 + assert len(items) == self._numKeys+1 assert type(items) == type([]) assert type(items[0]) == type(()) assert len(items[0]) == 2 @@ -266,7 +274,7 @@ class BasicTestCase(unittest.TestCase): pprint(items[:10]) values = d.values() - assert len(values) == 1001 + assert len(values) == self._numKeys+1 assert type(values) == type([]) if verbose: @@ -277,27 +285,36 @@ class BasicTestCase(unittest.TestCase): #---------------------------------------- - def test03_SimpleCursorStuff(self): + def test03_SimpleCursorStuff(self, get_raises_error=0, set_raises_error=1): if verbose: print '\n', '-=' * 30 - print "Running %s.test03_SimpleCursorStuff..." % \ - self.__class__.__name__ + print "Running %s.test03_SimpleCursorStuff (get_error %s, set_error %s)..." % \ + (self.__class__.__name__, get_raises_error, set_raises_error) if self.env and self.dbopenflags & db.DB_AUTO_COMMIT: txn = self.env.txn_begin() else: txn = None c = self.d.cursor(txn=txn) - + rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec - rec = c.next() - - assert count == 1000 + try: + rec = c.next() + except db.DBNotFoundError, val: + if get_raises_error: + assert val[0] == db.DB_NOTFOUND + if verbose: print val + rec = None + else: + self.fail("unexpected DBNotFoundError") + assert c.get_current_size() == len(c.current()[1]), "%s != len(%r)" % (c.get_current_size(), c.current()[1]) + + assert count == self._numKeys rec = c.last() @@ -306,34 +323,54 @@ class BasicTestCase(unittest.TestCase): count = count + 1 if verbose and count % 100 == 0: print rec - rec = c.prev() + try: + rec = c.prev() + except db.DBNotFoundError, val: + if get_raises_error: + assert val[0] == db.DB_NOTFOUND + if verbose: print val + rec = None + else: + self.fail("unexpected DBNotFoundError") - assert count == 1000 + assert count == self._numKeys rec = c.set('0505') rec2 = c.current() assert rec == rec2 assert rec[0] == '0505' assert rec[1] == self.makeData('0505') + assert c.get_current_size() == len(rec[1]) + # make sure we get empty values properly + rec = c.set('empty value') + assert rec[1] == '' + assert c.get_current_size() == 0 + try: - c.set('bad key') + n = c.set('bad key') except db.DBNotFoundError, val: assert val[0] == db.DB_NOTFOUND if verbose: print val else: - self.fail("expected exception") + if set_raises_error: + self.fail("expected exception") + if n != None: + self.fail("expected None: "+`n`) rec = c.get_both('0404', self.makeData('0404')) assert rec == ('0404', self.makeData('0404')) try: - c.get_both('0404', 'bad data') + n = c.get_both('0404', 'bad data') except db.DBNotFoundError, val: assert val[0] == db.DB_NOTFOUND if verbose: print val else: - self.fail("expected exception") + if get_raises_error: + self.fail("expected exception") + if n != None: + self.fail("expected None: "+`n`) if self.d.get_type() == db.DB_BTREE: rec = c.set_range('011') @@ -409,6 +446,29 @@ class BasicTestCase(unittest.TestCase): # SF pybsddb bug id 667343 del oldcursor + def test03b_SimpleCursorWithoutGetReturnsNone0(self): + # same test but raise exceptions instead of returning None + if verbose: + print '\n', '-=' * 30 + print "Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone..." % \ + self.__class__.__name__ + + old = self.d.set_get_returns_none(0) + assert old == 1 + self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1) + + def test03c_SimpleCursorGetReturnsNone2(self): + # same test but raise exceptions instead of returning None + if verbose: + print '\n', '-=' * 30 + print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \ + self.__class__.__name__ + + old = self.d.set_get_returns_none(2) + assert old == 1 + old = self.d.set_get_returns_none(2) + assert old == 2 + self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0) #---------------------------------------- @@ -525,23 +585,8 @@ class BasicTransactionTestCase(BasicTestCase): def populateDB(self): - d = self.d txn = self.env.txn_begin() - for x in range(500): - key = '%04d' % (1000 - x) # insert keys in reverse order - data = self.makeData(key) - d.put(key, data, txn) - - for x in range(500): - key = '%04d' % x # and now some in forward order - data = self.makeData(key) - d.put(key, data, txn) - - txn.commit() - - num = len(d) - if verbose: - print "created %d records" % num + BasicTestCase.populateDB(self, _txn=txn) self.txn = self.env.txn_begin() @@ -576,7 +621,7 @@ class BasicTransactionTestCase(BasicTestCase): if verbose and count % 100 == 0: print rec rec = c.next() - assert count == 1001 + assert count == self._numKeys+1 c.close() # Cursors *MUST* be closed before commit! self.txn.commit() @@ -805,7 +850,7 @@ class BasicMultiDBTestCase(BasicTestCase): if verbose and (count % 50) == 0: print rec rec = c1.next() - assert count == 1000 + assert count == self._numKeys count = 0 rec = c2.first() diff --git a/python/rpmdb/test/test_compat.py b/python/rpmdb/test/test_compat.py index 041081468..55acc8bcc 100644 --- a/python/rpmdb/test/test_compat.py +++ b/python/rpmdb/test/test_compat.py @@ -4,7 +4,6 @@ regression test suite. """ import sys, os, string -import rpmdb import unittest import tempfile @@ -12,6 +11,7 @@ from test_all import verbose from rpmdb import db, hashopen, btopen, rnopen + class CompatibilityTestCase(unittest.TestCase): def setUp(self): self.filename = tempfile.mktemp() diff --git a/python/rpmdb/test/test_dbtables.py b/python/rpmdb/test/test_dbtables.py index b90906515..dc4f7f027 100644 --- a/python/rpmdb/test/test_dbtables.py +++ b/python/rpmdb/test/test_dbtables.py @@ -18,7 +18,7 @@ # # -- Gregory P. Smith <greg@electricrain.com> # -# $Id: test_dbtables.py,v 1.1 2003/05/05 21:42:55 jbj Exp $ +# Id: test_dbtables.py,v 1.6 2003/09/21 00:08:14 greg Exp import sys, os, re try: diff --git a/python/rpmdb/test/test_join.py b/python/rpmdb/test/test_join.py index ab75ba196..9a46a891c 100644 --- a/python/rpmdb/test/test_join.py +++ b/python/rpmdb/test/test_join.py @@ -1,9 +1,115 @@ """TestCases for using the DB.join and DBCursor.join_item methods. """ +import sys, os, string +import tempfile +import time +from pprint import pprint + +try: + from threading import Thread, currentThread + have_threads = 1 +except ImportError: + have_threads = 0 + import unittest +from test_all import verbose + +from rpmdb import db, dbshelve + + +#---------------------------------------------------------------------- + +ProductIndex = [ + ('apple', "Convenience Store"), + ('blueberry', "Farmer's Market"), + ('shotgun', "S-Mart"), # Aisle 12 + ('pear', "Farmer's Market"), + ('chainsaw', "S-Mart"), # "Shop smart. Shop S-Mart!" + ('strawberry', "Farmer's Market"), +] + +ColorIndex = [ + ('blue', "blueberry"), + ('red', "apple"), + ('red', "chainsaw"), + ('red', "strawberry"), + ('yellow', "peach"), + ('yellow', "pear"), + ('black', "shotgun"), +] + +class JoinTestCase(unittest.TestCase): + keytype = '' + + def setUp(self): + self.filename = self.__class__.__name__ + '.db' + homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home') + self.homeDir = homeDir + try: os.mkdir(homeDir) + except os.error: pass + self.env = db.DBEnv() + self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK ) + + def tearDown(self): + self.env.close() + import glob + files = glob.glob(os.path.join(self.homeDir, '*')) + for file in files: + os.remove(file) + + def test01_join(self): + if verbose: + print '\n', '-=' * 30 + print "Running %s.test01_join..." % \ + self.__class__.__name__ + + # create and populate primary index + priDB = db.DB(self.env) + priDB.open(self.filename, "primary", db.DB_BTREE, db.DB_CREATE) + map(lambda t, priDB=priDB: apply(priDB.put, t), ProductIndex) + + # create and populate secondary index + secDB = db.DB(self.env) + secDB.set_flags(db.DB_DUP | db.DB_DUPSORT) + secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE) + map(lambda t, secDB=secDB: apply(secDB.put, t), ColorIndex) + + sCursor = None + jCursor = None + try: + # lets look up all of the red Products + sCursor = secDB.cursor() + # Don't do the .set() in an assert, or you can get a bogus failure + # when running python -O + tmp = sCursor.set('red') + assert tmp + + # FIXME: jCursor doesn't properly hold a reference to its + # cursors, if they are closed before jcursor is used it + # can cause a crash. + jCursor = priDB.join([sCursor]) + + if jCursor.get(0) != ('apple', "Convenience Store"): + self.fail("join cursor positioned wrong") + if jCursor.join_item() != 'chainsaw': + self.fail("DBCursor.join_item returned wrong item") + if jCursor.get(0)[0] != 'strawberry': + self.fail("join cursor returned wrong thing") + if jCursor.get(0): # there were only three red items to return + self.fail("join cursor returned too many items") + finally: + if jCursor: + jCursor.close() + if sCursor: + sCursor.close() + priDB.close() + secDB.close() def test_suite(): suite = unittest.TestSuite() + + suite.addTest(unittest.makeSuite(JoinTestCase)) + return suite diff --git a/python/rpmdb/test/test_thread.py b/python/rpmdb/test/test_thread.py index 3041557cc..4e7f9f01d 100644 --- a/python/rpmdb/test/test_thread.py +++ b/python/rpmdb/test/test_thread.py @@ -262,12 +262,12 @@ class SimpleThreadedBase(BaseThreadedTestCase): for loop in range(5): c = d.cursor() count = 0 - rec = c.first() + rec = dbutils.DeadlockWrap(c.first, max_retries=10) while rec: count += 1 key, data = rec self.assertEqual(self.makeData(key), data) - rec = c.next() + rec = dbutils.DeadlockWrap(c.next, max_retries=10) if verbose: print "%s: found %d records" % (name, count) c.close() |