# # # patch "tracvc/mtn/automate.py" # from [1f6b2e952bdfb6fc487f7fabf90f22bf32cd07dd] # to [1cb754d8f77cb7405b9e92facaa14dde58572c06] # # patch "tracvc/mtn/backend.py" # from [f61e9836fccd9c084387d569b0cc3f9d7c76b32c] # to [42e8c27bc7e7f79789b0951b2e56c1320bd17536] # # patch "tracvc/mtn/cache.py" # from [ee9520b88bf99131896bdb55ec7d977be849ef9f] # to [3cd067955720fe17bbb2d3e89337f6ca2ea7a951] # # patch "tracvc/mtn/util.py" # from [12e273ddf36684f07f66b88aad0bf7ddda3879d3] # to [0c74a27faa89fd3d7bb0e6a6803fdfbb096c4f2c] # ============================================================ --- tracvc/mtn/automate.py 1f6b2e952bdfb6fc487f7fabf90f22bf32cd07dd +++ tracvc/mtn/automate.py 1cb754d8f77cb7405b9e92facaa14dde58572c06 @@ -26,13 +26,13 @@ try: from subprocess import Popen, PIPE try: - from threading import Lock + from threading import Lock except ImportError: from dummy_threading import Lock #IGNORE:E0611 from tracvc.mtn import basic_io from tracvc.mtn.util import add_slash, to_unicode, natsort_key -from tracvc.mtn.cache import memoize + class AutomateException(Exception): """Thrown when the status of an automate command is not null, indicating that there is no valid result.""" @@ -49,7 +49,6 @@ class Automate(object): """General interface to the 'automate stdio' command.""" def __init__(self, database, binary): - import locale self.process = Popen( (binary, '--norc', '--root=.', '--automate-stdio-size=1048576', '--db=%s' % database, 'automate', 'stdio'), @@ -63,7 +62,7 @@ class Automate(object): def _flush(self): """Send flush to automate process.""" return self.process.stdin.flush() - + def _read(self, maxlen = -1): """Read maxlen bytes from automate process.""" return self.process.stdout.read(maxlen) @@ -77,7 +76,7 @@ class Automate(object): break result += char return result - + def _read_packet(self): """Read exactly one chunk of Monotone automate output.""" _ = self._read_until_colon() # ignore the cmd number @@ -142,15 +141,10 @@ class MTN(object): class MTN(object): """Connect to a Monotone repository using the automation interface.""" - def __init__(self, database, log, binary, cachespec): + def __init__(self, database, log, binary): self.automate = Automate(database, binary) self.log = log self.roots_cache = [] - self.cachespec = cachespec - - def get_cachespec(self): - """We use a method to hand the cachespec to Memoize at runtime.""" - return self.cachespec def leaves(self): """Returns a list containing the current leaves.""" @@ -164,7 +158,6 @@ class MTN(object): """Returns a list of the children of rev.""" return self.automate.command("children", [rev]).splitlines() - @memoize(get_cachespec) # IGNORE:E0602 def parents(self, rev): """Returns a list of the parents of rev.""" return self.automate.command("parents", [rev]).splitlines() @@ -199,10 +192,9 @@ class MTN(object): return self.automate.command("select", [selector.encode('utf-8')]).splitlines() - @memoize(get_cachespec) # IGNORE:E0602 def manifest(self, rev): """ Returns a processed manifest for rev. - + The manifest is a dictionary: path -> (kind, file_id, attrs), with kind being 'file' or 'dir', and attrs being a dictionary attr_name -> attr_value.""" @@ -225,14 +217,13 @@ class MTN(object): manifest[path] = (kind, content, attrs) return manifest - @memoize(get_cachespec) # IGNORE:E0602 def certs(self, rev): """Returns a dictionary of certs for rev. There might be more than one cert of the same name, so their values are collected in a list.""" raw_certs = self.automate.command("certs", [rev]) certs = {} - + for key, values in basic_io.items(raw_certs): if key == 'name': name = to_unicode(values[0]) @@ -244,13 +235,11 @@ class MTN(object): def file(self, file_id): """Returns the file contents for a given file id.""" return self.automate.command("get_file", [file_id]) - - @memoize(get_cachespec) # IGNORE:E0602 + def file_length(self, file_id): """Return the file length.""" return len(self.file(file_id)) - @memoize(get_cachespec) # IGNORE:E0602 def changesets(self, rev): """Parses a textual changeset into an instance of the Changeset class.""" @@ -286,7 +275,7 @@ class MTN(object): # fixme: what about 'set' and 'clear'? These are edits, # but not if applied to new files. return changesets - + def branchnames(self): """Returns a list of branch names.""" return map(to_unicode, #IGNORE:W0141 @@ -299,7 +288,7 @@ class MTN(object): for branch in self.branchnames(): revs = self.heads(branch) if revs: - branches.append((branch, revs[0])) + branches.append((branch, revs[0])) # multiple heads not supported return branches @@ -317,7 +306,7 @@ class MTN(object): """Returns a list of tags and their revs.""" raw_tags = self.automate.command("tags") tags = [] - + for key, values in basic_io.items(raw_tags): if key == 'tag': tag = to_unicode(values[0]) ============================================================ --- tracvc/mtn/backend.py f61e9836fccd9c084387d569b0cc3f9d7c76b32c +++ tracvc/mtn/backend.py 42e8c27bc7e7f79789b0951b2e56c1320bd17536 @@ -36,7 +36,9 @@ from tracvc.mtn.automate import MTN, Aut from pkg_resources import parse_version from time import strptime from tracvc.mtn.automate import MTN, AutomateException -from tracvc.mtn.util import get_oldpath, get_parent +from tracvc.mtn.util import get_oldpath, get_parent, Memoize +from tracvc.mtn.cache import CacheManager + import re try: @@ -74,7 +76,7 @@ class MonotoneConnector(Component): Component.__init__(self) self.repos = {} self.version = None - + def get_supported_types(self): """ Return the types of version control systems that are @@ -174,16 +176,16 @@ if parse_version(trac_version) < parse_v # Datetime handling changed somewhere between 0.10 and 0.11. We try to # support both variants for a while. if parse_version(trac_version) < parse_version("0.11dev"): - + def parse_date(rawdate): """ Convert a monotone date string into a unix timestamp. """ from calendar import timegm return timegm(strptime(rawdate + " UTC","%Y-%m-%dT%H:%M:%S %Z")) - + else: - + def parse_date(rawdate): """ Convert a monotone date string into a datetime object. @@ -207,20 +209,36 @@ def dates(certvals): return result +class CachedMTN(MTN): + + def __init__(self, database, log, binary, cachespec): + self.cachespec = cachespec + MTN.__init__(self, database, log, binary) + + def get_cache(self, realm): + return CacheManager.get_cache(realm, self.cachespec) + + parents = Memoize(MTN.parents, get_cache) + manifest = Memoize(MTN.manifest, get_cache) + certs = Memoize(MTN.certs, get_cache) + file_length = Memoize(MTN.file_length, get_cache) + changesets = Memoize(MTN.changesets, get_cache) + + class MonotoneRepository(Repository): """ Represents a Monotone repository. """ - + def close(self): """Close the connection to the repository.""" pass - + def __init__(self, path, log, binary, cachespec, revpropspec = None): Repository.__init__(self, 'mtn:%s' % path, None, log) - self.mtn = MTN(path, log, binary, cachespec) + self.mtn = CachedMTN(path, log, binary, cachespec) self.revpropspec = revpropspec or {} - + def get_changeset(self, rev): """ Retrieve a Changeset object that describes the changes made in @@ -271,10 +289,10 @@ class MonotoneRepository(Repository): def sync(self, feedback=None): """Perform a sync of the repository cache, if relevant. - + If given, `feedback` must be a callback taking a `rev` parameter. The backend will call this function for each `rev` it decided to - synchronize, once the synchronization changes are committed to the + synchronize, once the synchronization changes are committed to the cache. """ pass @@ -405,21 +423,21 @@ class MonotoneRepository(Repository): def get_quickjump_entries(self, from_rev): """ Generate a list of interesting places in the repositoy. - + `rev` might be used to restrict the list of available location, but in general it's best to produce all known locations. - + The generated results must be of the form (category, name, path, rev). """ result = [] - + for name, rev in self.get_branches(from_rev): result.append(('branches', name, '/', rev)) for name, rev in self.get_tags(from_rev): result.append(('tags', name, '/', rev)) - + return result def get_dates(self, rev): @@ -441,14 +459,14 @@ class MonotoneNode(Node): Represents a directory or file in the repository at a given revision. """ - + def __init__(self, mtn, rev, path, manifest = None): self.mtn = mtn self.manifest = manifest or self.mtn.manifest(rev) if not path in self.manifest: raise NoSuchNode(path, rev) - + self.content_id = self.manifest[path][1] self.created_path = path self.created_rev = rev @@ -463,7 +481,7 @@ class MonotoneNode(Node): # browser window, Node.path is used for the link behind # the path, but Node.rev is not, so better don't set # Node.path - + #marked = self.mtn.roster(rev)[1][curr.ident] #path = marked.name @@ -478,7 +496,7 @@ class MonotoneNode(Node): if self.isdir: return None return StringIO(self.mtn.file(self.content_id)) - + def get_entries(self): """ Generator that yields the immediate child entries of a @@ -496,7 +514,7 @@ class MonotoneNode(Node): for path in filter(ischild, self.manifest.keys()): # IGNORE:W0141 yield MonotoneNode(self.mtn, self.rev, path, self.manifest) - + def get_history(self, limit=None): """ Generator that yields (path, rev, chg) tuples, one for each @@ -526,12 +544,12 @@ class MonotoneNode(Node): if self.isdir: return None return '' - + def get_last_modified(self): # fixme: might be to pessimistic return dates(self.mtn.certs(self.rev).get('date', []))[-1] - + class MonotoneChangeset(Changeset): """ Represents the set of changes in one revision. @@ -547,15 +565,15 @@ class MonotoneChangeset(Changeset): # multiple dates not supported, so pick the first date date = self.dates[0] - + # Trac doesn't support multiple authors author = ', '.join(self.authors) - + # concatenate the commit messages message = '\n----\n'.join(self.messages) - + Changeset.__init__(self, rev, message, author, date) - + self.mtn = mtn self.revpropspec = revpropspec or {} @@ -594,7 +612,7 @@ class MonotoneChangeset(Changeset): for path in changeset.patched: oldpath = get_oldpath(path, changeset.renamed) yield path, Node.FILE, Changeset.EDIT, oldpath, oldrev - + if has_property_renderer: def get_properties(self): properties = {} ============================================================ --- tracvc/mtn/cache.py ee9520b88bf99131896bdb55ec7d977be849ef9f +++ tracvc/mtn/cache.py 3cd067955720fe17bbb2d3e89337f6ca2ea7a951 @@ -78,12 +78,12 @@ class Cache(object): """If key is not already in the cache, add the (key, value) pair. Fails silently otherwise.""" raise NotImplementedError - + def replace(self, key, value): """If key is already in the cache, replace it's value. Fails silently otherwise.""" raise NotImplementedError - + def flush_all(self): """Flush the cache.""" raise NotImplementedError @@ -196,7 +196,7 @@ class BSDDBShelve(Cache): def __init__(self, realm, dir, prefix=''): self.dbname = prefix + realm self.cache = dbshelve.open(self.dbname, dbenv=self.get_env(dir)) - + def set(self, key, value): self.cache[key] = value @@ -209,45 +209,3 @@ if db: if db: Cache.add_backend('bsddb', BSDDBShelve) - - -def memoize(get_cachespec, realm = None): - """Decorates a method with the Memoize decorator. get_cachespec is - a method of the same class which returns the desired cachespec, - realm is used to differentiate between different caches.""" - return lambda function: Memoize(function, get_cachespec, realm) - - -class Memoize(object): - """Caches return values of the decorated method.""" - - def __init__(self, function, get_cachespec, realm): - self.function = function - self.get_cachespec = get_cachespec - self.realm = realm or function.__name__ - - def __get__(self, instance, clazz = None): - self.instance = instance - return self - - def __call__(self, *args): - # Currently, we can only handle one single arg. Moreover, - # shelve only supports keys of type integer or string - if len(args) == 1: - key = args[0] - else: - key = args - - try: - return self.cache.get(key) - except KeyError: - # not yet in the cache - #print "CACHEMISS", self.realm, key - value = self.function(self.instance, *args) - self.cache.add(key, value) - return value - except AttributeError: - # oops, no cache yet, get one and restart - self.cache = CacheManager.get_cache(self.realm, \ - self.get_cachespec(self.instance)) - return self.__call__(*args) ============================================================ --- tracvc/mtn/util.py 12e273ddf36684f07f66b88aad0bf7ddda3879d3 +++ tracvc/mtn/util.py 0c74a27faa89fd3d7bb0e6a6803fdfbb096c4f2c @@ -37,7 +37,7 @@ def get_parent(path): def get_parent(path): """Returns the name of the directory containing path, or None if there is none (because path is '/' or None).""" - + path = path and path.rstrip('/') return path and (path[0:path.rfind('/')] or '/') or None @@ -82,3 +82,34 @@ def to_unicode(s): """Convert s from utf-8 to unicode, thereby replacing unknown characters with the unicode replace char.""" return unicode(s, "utf-8", "replace") + + +class Memoize(object): + """Caches return values of the decorated method.""" + + def __init__(self, method, get_cache, realm = None): + self.method = method + self.get_cache = get_cache + self.realm = realm or method.__name__ + self.instance = None + self.cache = None + + def __get__(self, instance, clazz = None): # IGNORE:W0613 + # non-data descriptor + self.instance = instance + return self + + def __call__(self, arg): + # Currently, we can only handle one single arg. Moreover, + # shelve only supports keys of type integer or string + try: + return self.cache.get(arg) + except KeyError: + # not yet in the cache + value = self.method(self.instance, arg) + self.cache.add(arg, value) + return value + except AttributeError: + # oops, no cache yet, get one and restart + self.cache = self.get_cache(self.instance, self.realm) + return self.__call__(arg)