[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Rdiff-backup-commits] Changes to rdiff-backup/testing/commontest.py [r1
From: |
Ben Escoto |
Subject: |
[Rdiff-backup-commits] Changes to rdiff-backup/testing/commontest.py [r1-0] |
Date: |
Fri, 25 Nov 2005 18:07:51 -0500 |
Index: rdiff-backup/testing/commontest.py
diff -u /dev/null rdiff-backup/testing/commontest.py:1.33.2.1
--- /dev/null Fri Nov 25 23:07:51 2005
+++ rdiff-backup/testing/commontest.py Fri Nov 25 23:07:51 2005
@@ -0,0 +1,388 @@
+"""commontest - Some functions and constants common to several test cases"""
+import os, sys, code
+from rdiff_backup.log import Log
+from rdiff_backup.rpath import RPath
+from rdiff_backup import Globals, Hardlink, SetConnections, Main, \
+ selection, lazy, Time, rpath, eas_acls, rorpiter
+
+RBBin = "../rdiff-backup"
+SourceDir = "../rdiff_backup"
+AbsCurdir = os.getcwd() # Absolute path name of current directory
+AbsTFdir = AbsCurdir+"/testfiles"
+MiscDir = "../misc"
+__no_execute__ = 1 # Keeps the actual rdiff-backup program from running
+
+def Myrm(dirstring):
+ """Run myrm on given directory string"""
+ root_rp = rpath.RPath(Globals.local_connection, dirstring)
+ for rp in selection.Select(root_rp).set_iter():
+ if rp.isdir(): rp.chmod(0700) # otherwise may not be able to
remove
+ assert not os.system("rm -rf %s" % (dirstring,))
+
+def re_init_dir(rp):
+ """Delete directory if present, then recreate"""
+ if rp.lstat():
+ Myrm(rp.path)
+ rp.setdata()
+ rp.mkdir()
+
+def Make():
+ """Make sure the rdiff-backup script in the source dir is up-to-date"""
+ os.chdir(SourceDir)
+ os.system("python ./Make")
+ os.chdir(AbsCurdir)
+
+def MakeOutputDir():
+ """Initialize the testfiles/output directory"""
+ Myrm("testfiles/output")
+ rp = rpath.RPath(Globals.local_connection, "testfiles/output")
+ rp.mkdir()
+ return rp
+
+def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
+ current_time = None, extra_options = "",
+ check_return_val = 1):
+ """Run rdiff-backup with the given options
+
+ source_local and dest_local are boolean values. If either is
+ false, then rdiff-backup will be run pretending that src_dir and
+ dest_dir, respectively, are remote. The server process will be
+ run in directories test1 and test2/tmp respectively.
+
+ src_dir and dest_dir are the source and destination
+ (mirror) directories, relative to the testing directory.
+
+ If current time is true, add the --current-time option with the
+ given number of seconds.
+
+ extra_options are just added to the command line.
+
+ """
+ if not source_local:
+ src_dir = ("'cd test1; ../%s --server'::../%s" % (RBBin,
src_dir))
+ if not dest_local:
+ dest_dir = ("'cd test2/tmp; ../../%s --server'::../../%s" %
+ (RBBin, dest_dir))
+
+ cmdargs = [RBBin, extra_options]
+ if not (source_local and dest_local): cmdargs.append("--remote-schema
%s")
+
+ if current_time: cmdargs.append("--current-time %s" % current_time)
+ cmdargs.extend([src_dir, dest_dir])
+ cmdline = " ".join(cmdargs)
+ print "Executing: ", cmdline
+ ret_val = os.system(cmdline)
+ if check_return_val: assert not ret_val, ret_val
+ return ret_val
+
+def cmd_schemas2rps(schema_list, remote_schema):
+ """Input list of file descriptions and the remote schema, return rps
+
+ File descriptions should be strings of the form 'hostname.net::foo'
+
+ """
+ return map(SetConnections.cmdpair2rp,
+ SetConnections.get_cmd_pairs(schema_list,
remote_schema))
+
+def InternalBackup(source_local, dest_local, src_dir, dest_dir,
+ current_time = None, eas = None, acls =
None):
+ """Backup src to dest internally
+
+ This is like rdiff_backup but instead of running a separate
+ rdiff-backup script, use the separate *.py files. This way the
+ script doesn't have to be rebuild constantly, and stacktraces have
+ correct line/file references.
+
+ """
+ Globals.current_time = current_time
+ #_reset_connections()
+ Globals.security_level = "override"
+ remote_schema = '%s'
+
+ if not source_local:
+ src_dir = "cd test1; python ../server.py ../%s::../%s" % \
+ (SourceDir, src_dir)
+ if not dest_local:
+ dest_dir = "cd test2/tmp; python ../../server.py
../../%s::../../%s" \
+ % (SourceDir, dest_dir)
+
+ rpin, rpout = cmd_schemas2rps([src_dir, dest_dir], remote_schema)
+ for attr in ('eas_active', 'eas_write', 'eas_conn'):
+ SetConnections.UpdateGlobal(attr, eas)
+ for attr in ('acls_active', 'acls_write', 'acls_conn'):
+ SetConnections.UpdateGlobal(attr, acls)
+ Main.misc_setup([rpin, rpout])
+ Main.Backup(rpin, rpout)
+ Main.cleanup()
+
+def InternalMirror(source_local, dest_local, src_dir, dest_dir):
+ """Mirror src to dest internally
+
+ like InternalBackup, but only mirror. Do this through
+ InternalBackup, but then delete rdiff-backup-data directory.
+
+ """
+ # Save attributes of root to restore later
+ src_root = rpath.RPath(Globals.local_connection, src_dir)
+ dest_root = rpath.RPath(Globals.local_connection, dest_dir)
+ dest_rbdir = dest_root.append("rdiff-backup-data")
+
+ InternalBackup(source_local, dest_local, src_dir, dest_dir)
+ dest_root.setdata()
+ Myrm(dest_rbdir.path)
+ # Restore old attributes
+ rpath.copy_attribs(src_root, dest_root)
+
+def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time,
+ eas = None, acls = None):
+ """Restore mirror_dir to dest_dir at given time
+
+ This will automatically find the increments.XXX.dir representing
+ the time specified. The mirror_dir and dest_dir are relative to
+ the testing directory and will be modified for remote trials.
+
+ """
+ Main.force = 1
+ Main.restore_root_set = 0
+ remote_schema = '%s'
+ Globals.security_level = "override"
+ #_reset_connections()
+ if not mirror_local:
+ mirror_dir = "cd test1; python ../server.py ../%s::../%s" % \
+ (SourceDir, mirror_dir)
+ if not dest_local:
+ dest_dir = "cd test2/tmp; python ../../server.py
../../%s::../../%s" \
+ % (SourceDir, dest_dir)
+
+ mirror_rp, dest_rp = cmd_schemas2rps([mirror_dir, dest_dir],
remote_schema)
+ for attr in ('eas_active', 'eas_write', 'eas_conn'):
+ SetConnections.UpdateGlobal(attr, eas)
+ for attr in ('acls_active', 'acls_write', 'acls_conn'):
+ SetConnections.UpdateGlobal(attr, acls)
+ Main.misc_setup([mirror_rp, dest_rp])
+ inc = get_increment_rp(mirror_rp, time)
+ if inc: Main.Restore(get_increment_rp(mirror_rp, time), dest_rp)
+ else: # use alternate syntax
+ Main.restore_timestr = str(time)
+ Main.Restore(mirror_rp, dest_rp, restore_as_of = 1)
+ Main.cleanup()
+
+def get_increment_rp(mirror_rp, time):
+ """Return increment rp matching time in seconds"""
+ data_rp = mirror_rp.append("rdiff-backup-data")
+ if not data_rp.isdir(): return None
+ for filename in data_rp.listdir():
+ rp = data_rp.append(filename)
+ if rp.isincfile() and rp.getincbase_str() == "increments":
+ if rp.getinctime() == time: return rp
+ return None # Couldn't find appropriate increment
+
+def _reset_connections(src_rp, dest_rp):
+ """Reset some global connection information"""
+ Globals.security_level = "override"
+ Globals.isbackup_reader = Globals.isbackup_writer = None
+ #Globals.connections = [Globals.local_connection]
+ #Globals.connection_dict = {0: Globals.local_connection}
+ SetConnections.UpdateGlobal('rbdir', None)
+ Main.misc_setup([src_rp, dest_rp])
+
+def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
+ equality_func = None, exclude_rbdir =
1,
+ ignore_tmp_files = None,
compare_ownership = 0,
+ compare_eas = 0, compare_acls = 0):
+ """Compare src_rp and dest_rp, which can be directories
+
+ This only compares file attributes, not the actual data. This
+ will overwrite the hardlink dictionaries if compare_hardlinks is
+ specified.
+
+ """
+ def get_selection_functions():
+ """Return generators of files in source, dest"""
+ src_rp.setdata()
+ dest_rp.setdata()
+ src_select = selection.Select(src_rp)
+ dest_select = selection.Select(dest_rp)
+
+ if ignore_tmp_files:
+ # Ignoring temp files can be useful when we want to
check the
+ # correctness of a backup which aborted in the middle.
In
+ # these cases it is OK to have tmp files lying around.
+ src_select.add_selection_func(src_select.regexp_get_sf(
+ ".*rdiff-backup.tmp.[^/]+$", 0))
+
dest_select.add_selection_func(dest_select.regexp_get_sf(
+ ".*rdiff-backup.tmp.[^/]+$", 0))
+
+ if exclude_rbdir: # Exclude rdiff-backup-data directory
+ src_select.parse_rbdir_exclude()
+ dest_select.parse_rbdir_exclude()
+
+ return src_select.set_iter(), dest_select.set_iter()
+
+ def preprocess(src_rorp, dest_rorp):
+ """Initially process src and dest_rorp"""
+ if compare_hardlinks and src_rorp:
+ Hardlink.add_rorp(src_rorp, dest_rorp)
+
+ def postprocess(src_rorp, dest_rorp):
+ """After comparison, process src_rorp and dest_rorp"""
+ if compare_hardlinks and src_rorp:
+ Hardlink.del_rorp(src_rorp)
+
+ def equality_func(src_rorp, dest_rorp):
+ """Combined eq func returns true iff two files compare same"""
+ if not src_rorp:
+ Log("Source rorp missing: " + str(dest_rorp), 3)
+ return 0
+ if not dest_rorp:
+ Log("Dest rorp missing: " + str(src_rorp), 3)
+ return 0
+ if not src_rorp.equal_verbose(dest_rorp,
+
compare_ownership = compare_ownership):
+ return 0
+ if compare_hardlinks and not Hardlink.rorp_eq(src_rorp,
dest_rorp):
+ Log("Hardlink compare failure", 3)
+ Log("%s: %s" % (src_rorp.index,
+
Hardlink.get_inode_key(src_rorp)), 3)
+ Log("%s: %s" % (dest_rorp.index,
+
Hardlink.get_inode_key(dest_rorp)), 3)
+ return 0
+ if compare_eas and not eas_acls.ea_compare_rps(src_rorp,
dest_rorp):
+ Log("Different EAs in files %s and %s" %
+ (src_rorp.get_indexpath(),
dest_rorp.get_indexpath()), 3)
+ return 0
+ if compare_acls and not eas_acls.acl_compare_rps(src_rorp,
dest_rorp):
+ Log("Different ACLs in files %s and %s" %
+ (src_rorp.get_indexpath(),
dest_rorp.get_indexpath()), 3)
+ return 0
+ return 1
+
+ Log("Comparing %s and %s, hardlinks %s, eas %s, acls %s" %
+ (src_rp.path, dest_rp.path, compare_hardlinks,
+ compare_eas, compare_acls), 3)
+ if compare_hardlinks: reset_hardlink_dicts()
+ src_iter, dest_iter = get_selection_functions()
+ for src_rorp, dest_rorp in rorpiter.Collate2Iters(src_iter, dest_iter):
+ preprocess(src_rorp, dest_rorp)
+ if not equality_func(src_rorp, dest_rorp): return 0
+ postprocess(src_rorp, dest_rorp)
+ return 1
+
+
+ def rbdir_equal(src_rorp, dest_rorp):
+ """Like hardlink_equal, but make allowances for data
directories"""
+ if not src_rorp.index and not dest_rorp.index: return 1
+ if (src_rorp.index and src_rorp.index[0] == 'rdiff-backup-data'
and
+ src_rorp.index == dest_rorp.index):
+ # Don't compare dirs - they don't carry significant info
+ if dest_rorp.isdir() and src_rorp.isdir(): return 1
+ if dest_rorp.isreg() and src_rorp.isreg():
+ # Don't compare gzipped files because it is
apparently
+ # non-deterministic.
+ if dest_rorp.index[-1].endswith('gz'): return 1
+ # Don't compare .missing increments because
they don't matter
+ if dest_rorp.index[-1].endswith('.missing'):
return 1
+ if compare_eas and not eas_acls.ea_compare_rps(src_rorp,
dest_rorp):
+ Log("Different EAs in files %s and %s" %
+ (src_rorp.get_indexpath(),
dest_rorp.get_indexpath()))
+ return None
+ if compare_acls and not eas_acls.acl_compare_rps(src_rorp,
dest_rorp):
+ Log("Different ACLs in files %s and %s" %
+ (src_rorp.get_indexpath(),
dest_rorp.get_indexpath()), 3)
+ return None
+ if compare_hardlinks:
+ if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
+ elif src_rorp.equal_verbose(dest_rorp,
+
compare_ownership = compare_ownership):
+ return 1
+ Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp,
1)), 3)
+ Log("%s: %s" % (dest_rorp.index,
+
Hardlink.get_indicies(dest_rorp, None)), 3)
+ return None
+
+
+def reset_hardlink_dicts():
+ """Clear the hardlink dictionaries"""
+ Hardlink._inode_index = {}
+
+def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
+ compare_hardlinks = 1,
+ dest_dirname =
"testfiles/output",
+ restore_dirname =
"testfiles/rest_out",
+ compare_backups = 1,
+ compare_eas = 0,
+ compare_acls = 0,
+ compare_ownership = 0):
+ """Test backing up/restoring of a series of directories
+
+ The dirnames correspond to a single directory at different times.
+ After each backup, the dest dir will be compared. After the whole
+ set, each of the earlier directories will be recovered to the
+ restore_dirname and compared.
+
+ """
+ Globals.set('preserve_hardlinks', compare_hardlinks)
+ time = 10000
+ dest_rp = rpath.RPath(Globals.local_connection, dest_dirname)
+ restore_rp = rpath.RPath(Globals.local_connection, restore_dirname)
+
+ Myrm(dest_dirname)
+ for dirname in list_of_dirnames:
+ src_rp = rpath.RPath(Globals.local_connection, dirname)
+ reset_hardlink_dicts()
+ _reset_connections(src_rp, dest_rp)
+
+ InternalBackup(source_local, dest_local, dirname, dest_dirname,
time,
+ eas = compare_eas, acls =
compare_acls)
+ time += 10000
+ _reset_connections(src_rp, dest_rp)
+ if compare_backups:
+ assert CompareRecursive(src_rp, dest_rp,
compare_hardlinks,
+
compare_eas = compare_eas,
+
compare_acls = compare_acls,
+
compare_ownership = compare_ownership)
+
+ time = 10000
+ for dirname in list_of_dirnames[:-1]:
+ reset_hardlink_dicts()
+ Myrm(restore_dirname)
+ InternalRestore(dest_local, source_local, dest_dirname,
+ restore_dirname, time,
+ eas = compare_eas, acls =
compare_acls)
+ src_rp = rpath.RPath(Globals.local_connection, dirname)
+ assert CompareRecursive(src_rp, restore_rp,
+ compare_eas =
compare_eas,
+ compare_acls =
compare_acls,
+
compare_ownership = compare_ownership)
+
+ # Restore should default back to newest time older than it
+ # with a backup then.
+ if time == 20000: time = 21000
+
+ time += 10000
+
+def MirrorTest(source_local, dest_local, list_of_dirnames,
+ compare_hardlinks = 1,
+ dest_dirname = "testfiles/output"):
+ """Mirror each of list_of_dirnames, and compare after each"""
+ Globals.set('preserve_hardlinks', compare_hardlinks)
+ dest_rp = rpath.RPath(Globals.local_connection, dest_dirname)
+ old_force_val = Main.force
+ Main.force = 1
+
+ Myrm(dest_dirname)
+ for dirname in list_of_dirnames:
+ src_rp = rpath.RPath(Globals.local_connection, dirname)
+ reset_hardlink_dicts()
+ _reset_connections(src_rp, dest_rp)
+
+ InternalMirror(source_local, dest_local, dirname, dest_dirname)
+ _reset_connections(src_rp, dest_rp)
+ assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
+ Main.force = old_force_val
+
+def raise_interpreter(use_locals = None):
+ """Start python interpreter, with local variables if locals is true"""
+ if use_locals: local_dict = locals()
+ else: local_dict = globals()
+ code.InteractiveConsole(local_dict).interact()
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Rdiff-backup-commits] Changes to rdiff-backup/testing/commontest.py [r1-0],
Ben Escoto <=