[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [Duplicity-talk] WebDAV backend
From: |
Jörg Mühlberger |
Subject: |
Re: [Duplicity-talk] WebDAV backend |
Date: |
Wed, 16 Aug 2006 21:24:06 +0200 |
User-agent: |
Thunderbird 1.5.0.5 (Windows/20060719) |
Hello!
is anybody using duplicity with a WebDAV backend? I found this patch
http://savannah.nongnu.org/patch/?func=detailitem&item_id=5185
but it doesn't work on my Debian 3.1 machine.
What do you mean? Does the patch not apply?
The patch itself is working, but I get errors of other subroutines that
this backend needs to start to make a connection via http.
So if I run the following command:
:~# duplicity /home/me/ webdav://address@hidden/
I get the following error message.
Traceback (most recent call last):
File "/usr/bin/duplicity", line 358, in ?
if __name__ == "__main__": main()
File "/usr/bin/duplicity", line 334, in main
globals.archive_dir).set_values()
File "/usr/lib/python2.3/site-packages/duplicity/collections.py", line
355, in set_values
backend_filename_list = self.backend.list()
File "/usr/lib/python2.3/site-packages/duplicity/backends.py", line
495, in list
dom = xml.dom.minidom.parseString(document)
File "/usr/lib/python2.3/xml/dom/minidom.py", line 1929, in parseString
return expatbuilder.parseString(string)
File "/usr/lib/python2.3/xml/dom/expatbuilder.py", line 940, in
parseString
return builder.parseString(string)
File "/usr/lib/python2.3/xml/dom/expatbuilder.py", line 223, in
parseString
parser.Parse(string, True)
xml.parsers.expat.ExpatError: syntax error: line 1, column 49
I have attached my backends.py for your information.
If there is anything else that I can help you with, to trace this error,
just let me know!
Thanks in advance
Jörg
# Copyright 2002 Ben Escoto
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""Provides functions and classes for getting/sending files to destination"""
import os, types, ftplib, tempfile
import log, path, dup_temp, file_naming
import time, base64, getpass
import xml.dom.minidom, httplib
from urllib import unquote
class BackendException(Exception): pass
class ParsingException(Exception): pass
def get_backend(url_string):
"""Return Backend object from url string, or None if not a url string
url strings are like
scp://foobar:address@hidden:124/usr/local. If a protocol
is unsupported a fatal error will be raised.
"""
global protocol_class_dict
try: pu = ParsedUrl(url_string)
except ParsingException: return None
try: backend_class = protocol_class_dict[pu.protocol]
except KeyError: log.FatalError("Unknown protocol '%s'" %
(pu.protocol,))
return backend_class(pu)
class ParsedUrl:
"""Contains information gleaned from a generic url"""
protocol = None # set to string like "ftp" indicating protocol
suffix = None # Set to everything after protocol://
server = None # First part of suffix (part before '/')
path = None # Second part of suffix (part after '/')
host = None # Set to host, if can be extracted
user = None # Set to user, as in ftp://address@hidden/whatever
port = None # Set to port, like scp://host:port/foo
def __init__(self, url_string):
"""Create ParsedUrl object, process url_string"""
self.url_string = url_string
self.set_protocol_suffix()
self.set_server_path()
self.set_host_user_port()
def bad_url(self, message = None):
"""Report a bad url, using message if given"""
if message:
err_string = "Bad URL string '%s': %s" %
(self.url_string, message)
else: err_string = "Bad URL string '%s'" % (self.url_string,)
raise ParsingException(err_string)
def set_protocol_suffix(self):
"""Parse self.url_string, setting self.protocol and
self.suffix"""
colon_position = self.url_string.find(":")
if colon_position < 1: self.bad_url("No colon (:) found")
self.protocol = self.url_string[:colon_position]
if self.url_string[colon_position+1:colon_position+3] != '//':
self.bad_url("first colon not followed by '//'")
self.suffix = self.url_string[colon_position+3:]
def set_server_path(self):
"""Set self.server and self.path from self.suffix"""
comps = self.suffix.split('/')
assert len(comps) > 0
self.server = comps[0]
if len(comps) > 1:
self.path = '/'.join(comps[1:])
def set_host_user_port(self):
"""Set self.host, self.user, and self.port from self.server"""
if not self.server: return
# Extract port
port_comps = self.server.split(":")
if len(port_comps) >= 2:
try: self.port = int(port_comps[-1])
except ValueError: user_host = self.server
else: user_host = ":".join(port_comps[:-1])
else: user_host = self.server
# Set user and host
user_comps = user_host.split("@")
if len(user_comps) >= 2:
self.user = user_comps[0]
self.host = "@".join(user_comps[1:])
else: self.host = user_host
class Backend:
"""Represent a connection to the destination device/computer
Classes that subclass this should implement the put, get, list,
and delete methods.
"""
def init(self, parsed_url): pass
def put(self, source_path, remote_filename = None):
"""Transfer source_path (Path object) to remote_filename
(string)
If remote_filename is None, get the filename from the last
path component of pathname.
"""
if not remote_filename: remote_filename =
source_path.get_filename()
pass
def get(self, remote_filename, local_path):
"""Retrieve remote_filename and place in local_path"""
local_path.setdata()
pass
def list(self):
"""Return list of filenames (strings) present in backend"""
pass
def delete(self, filename_list):
"""Delete each filename in filename_list, in order if
possible"""
pass
def run_command(self, commandline):
"""Run given commandline with logging and error detection"""
log.Log("Running '%s'" % commandline, 5)
if os.system(commandline):
raise BackendException("Error running '%s'" %
commandline)
def popen(self, commandline):
"""Run command and return stdout results"""
log.Log("Reading results of '%s'" % commandline, 5)
fout = os.popen(commandline)
results = fout.read()
if fout.close():
raise BackendException("Error running '%s'" %
commandline)
return results
def get_fileobj_read(self, filename, parseresults = None):
"""Return fileobject opened for reading of filename on backend
The file will be downloaded first into a temp file. When the
returned fileobj is closed, the temp file will be deleted.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, "Filename not correctly parsed"
tdp = dup_temp.new_tempduppath(parseresults)
self.get(filename, tdp)
tdp.setdata()
return tdp.filtered_open_with_delete("rb")
def get_fileobj_write(self, filename, parseresults = None,
sizelist = None):
"""Return fileobj opened for writing, write to backend on close
The file will be encoded as specified in parseresults (or as
read from the filename), and stored in a temp file until it
can be copied over and deleted.
If sizelist is not None, it should be set to an empty list.
The number of bytes will be inserted into the list.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, "Filename %s not correctly parsed"
% filename
tdp = dup_temp.new_tempduppath(parseresults)
def close_file_hook():
"""This is called when returned fileobj is closed"""
self.put(tdp, filename)
if sizelist is not None:
tdp.setdata()
sizelist.append(tdp.getsize())
tdp.delete()
fh = dup_temp.FileobjHooked(tdp.filtered_open("wb"))
fh.addhook(close_file_hook)
return fh
def get_data(self, filename, parseresults = None):
"""Retrieve a file from backend, process it, return contents"""
fin = self.get_fileobj_read(filename, parseresults)
buf = fin.read()
assert not fin.close()
return buf
def put_data(self, buffer, filename, parseresults = None):
"""Put buffer into filename on backend after processing"""
fout = self.get_fileobj_write(filename, parseresults)
fout.write(buffer)
assert not fout.close()
def close(self):
"""This is called when a connection is no longer needed"""
pass
class LocalBackend(Backend):
"""Use this backend when saving to local disk
Urls look like file://testfiles/output. Relative to root can be
gotten with extra slash (file:///usr/local).
"""
def __init__(self, parsed_url):
self.remote_pathdir = path.Path(parsed_url.suffix)
def put(self, source_path, remote_filename = None, rename = None):
"""If rename is set, try that first, copying if doesn't work"""
if not remote_filename: remote_filename =
source_path.get_filename()
target_path = self.remote_pathdir.append(remote_filename)
log.Log("Writing %s" % target_path.name, 6)
if rename:
try: source_path.rename(target_path)
except OSError: pass
else: return
target_path.writefileobj(source_path.open("rb"))
def get(self, filename, local_path):
"""Get file and put in local_path (Path object)"""
source_path = self.remote_pathdir.append(filename)
local_path.writefileobj(source_path.open("rb"))
def list(self):
"""List files in that directory"""
return self.remote_pathdir.listdir()
def delete(self, filename_list):
"""Delete all files in filename list"""
assert type(filename_list) is not types.StringType
try:
for filename in filename_list:
self.remote_pathdir.append(filename).delete()
except OSError, e: raise BackendException(str(e))
# The following can be redefined to use different shell commands from
# ssh or scp or to add more arguments. However, the replacements must
# have the same syntax. Also these strings will be executed by the
# shell, so shouldn't have strange characters in them.
ssh_command = "ssh"
scp_command = "scp"
class scpBackend(Backend):
"""This backend copies files using scp. List not supported"""
def __init__(self, parsed_url):
"""scpBackend initializer"""
self.host_string = parsed_url.server # of form
address@hidden:port
self.remote_dir = parsed_url.path # can be empty string
if self.remote_dir: self.remote_prefix = self.remote_dir + "/"
else: self.remote_prefix = ""
def put(self, source_path, remote_filename = None):
"""Use scp to copy source_dir/filename to remote computer"""
if not remote_filename: remote_filename =
source_path.get_filename()
commandline = "%s %s %s:%s%s" % \
(scp_command, source_path.name,
self.host_string,
self.remote_prefix, remote_filename)
self.run_command(commandline)
def get(self, remote_filename, local_path):
"""Use scp to get a remote file"""
commandline = "%s %s:%s%s %s" % \
(scp_command, self.host_string,
self.remote_prefix,
remote_filename, local_path.name)
self.run_command(commandline)
local_path.setdata()
if not local_path.exists():
raise BackendException("File %s not found" %
local_path.name)
def list(self):
"""List files available for scp
Note that this command can get confused when dealing with
files with newlines in them, as the embedded newlines cannot
be distinguished from the file boundaries.
"""
commandline = ("%s %s ls %s" %
(ssh_command, self.host_string,
self.remote_dir))
return filter(lambda x: x, self.popen(commandline).split("\n"))
def delete(self, filename_list):
"""Runs ssh rm to delete files. Files must not require
quoting"""
assert len(filename_list) > 0
pathlist = map(lambda fn: self.remote_prefix + fn,
filename_list)
del_prefix = "%s %s rm " % (ssh_command, self.host_string)
# Delete in groups of 10 to avoid overflowing command line
for i in range(0, len(pathlist), 10):
commandline = del_prefix + " ".join(pathlist[i:i+10])
self.run_command(commandline)
class sftpBackend(Backend):
"""This backend uses sftp to perform file operations"""
pass # Do this later
class ftpBackend(Backend):
"""Connect to remote store using File Transfer Protocol"""
SLEEP = 10 # time in seconds before we try to reconnect on temporary
errors
def __init__(self, parsed_url):
"""Create a new ftp backend object, log in to host"""
self.parsed_url = parsed_url
self.ftp = ftplib.FTP()
if parsed_url.port is None: self.error_wrap('connect',
parsed_url.host)
else: self.error_wrap('connect', parsed_url.host,
parsed_url.port)
if parsed_url.user is not None:
self.error_wrap('login', parsed_url.user,
self.get_password())
else: self.error_wrap('login')
self.ftp.cwd(parsed_url.path)
def error_wrap(self, command, *args):
"""Run self.ftp.command(*args), but raise BackendException on
error"""
try: return ftplib.FTP.__dict__[command](self.ftp, *args)
except ftplib.error_temp, e:
log.Log("Temporary error '%s'. Trying to reconnect in
%d seconds." %
(str(e), self.SLEEP), 3)
time.sleep(self.SLEEP)
self.__init__(self.parsed_url)
self.error_wrap(command, *args)
except ftplib.all_errors, e: raise BackendException(e)
def get_password(self):
"""Get ftp password using environment if possible"""
try: return os.environ['FTP_PASSWORD']
except KeyError:
log.Log("FTP_PASSWORD not set, using empty ftp
password", 3)
return ''
def put(self, source_path, remote_filename = None):
"""Transfer source_path to remote_filename"""
if not remote_filename: remote_filename =
source_path.get_filename()
source_file = source_path.open("rb")
log.Log("Saving %s on FTP server" % (remote_filename,), 5)
self.error_wrap('storbinary', "STOR "+remote_filename,
source_file)
assert not source_file.close()
def get(self, remote_filename, local_path):
"""Get remote filename, saving it to local_path"""
target_file = local_path.open("wb")
log.Log("Retrieving %s from FTP server" % (remote_filename,), 5)
self.error_wrap('retrbinary', "RETR "+remote_filename,
target_file.write)
assert not target_file.close()
local_path.setdata()
def list(self):
"""List files in directory"""
log.Log("Listing files on FTP server", 5)
return self.error_wrap('nlst')
def delete(self, filename_list):
"""Delete files in filename_list"""
for filename in filename_list:
log.Log("Deleting %s from FTP server" % (filename,), 5)
self.error_wrap('delete', filename)
def close(self):
"""Shut down connection"""
self.error_wrap('quit')
class rsyncBackend(Backend):
"""Connect to remote store using rsync
rsync backend contributed by Sebastian Wilhelmi <address@hidden>
"""
def __init__(self, parsed_url):
"""rsyncBackend initializer"""
self.url_string = parsed_url.url_string
if self.url_string[-1] != '/':
self.url_string += '/'
def put(self, source_path, remote_filename = None):
"""Use rsync to copy source_dir/filename to remote computer"""
if not remote_filename: remote_filename =
source_path.get_filename()
remote_path = os.path.join (self.url_string, remote_filename)
commandline = "rsync %s %s" % (source_path.name, remote_path)
self.run_command(commandline)
def get(self, remote_filename, local_path):
"""Use rsync to get a remote file"""
remote_path = os.path.join (self.url_string, remote_filename)
commandline = "rsync %s %s" % (remote_path, local_path.name)
self.run_command(commandline)
local_path.setdata()
if not local_path.exists():
raise BackendException("File %s not found" %
local_path.name)
def list(self):
"""List files"""
def split (str):
line = str.split ()
if len (line) > 4 and line[4] != '.':
return line[4]
else:
return None
commandline = "rsync %s" % self.url_string
return filter (lambda x: x, map (split,
self.popen(commandline).split('\n')))
def delete(self, filename_list):
"""Delete files."""
delete_list = filename_list
dont_delete_list = []
for file in self.list ():
if file in delete_list:
delete_list.remove (file)
else:
dont_delete_list.append (file)
if len (delete_list) > 0:
raise BackendException("Files %s not found" % str
(delete_list))
dir = tempfile.mktemp ()
exclude_name = tempfile.mktemp ()
exclude = open (exclude_name, 'w')
to_delete = [exclude_name]
os.mkdir (dir)
for file in dont_delete_list:
path = os.path.join (dir, file)
to_delete.append (path)
f = open (path, 'w')
f.close ()
print >>exclude, file
exclude.close ()
commandline = ("rsync --recursive --delete --exclude-from=%s
%s/ %s" %
(exclude_name, dir, self.url_string))
self.run_command(commandline)
for file in to_delete:
os.unlink (file)
os.rmdir (dir)
class webdavBackend(Backend):
directory = None
conn = None
headers = {}
"""Connect to remote store using WebDAV Protocol"""
SLEEP = 10 # time in seconds before we try to reconnect on temporary
errors
def __init__(self, parsed_url):
self.parsed_url = parsed_url
self.directory = '/' + parsed_url.path.rstrip('/') + '/'
log.Log("Using WebDAV host %s" % (parsed_url.host,), 5)
log.Log("Using WebDAV direcotry %s" % (self.directory,), 5)
self.conn = httplib.HTTPConnection( parsed_url.host )
try: password = os.environ['FTP_PASSWORD']
except KeyError:
password = getpass.getpass('Password for
'+parsed_url.user+'@'+parsed_url.host+': ')
self.headers['Authorization'] = 'Basic ' + base64.encodestring(
parsed_url.user+':'+ password)
def _getText(self,nodelist):
rc = ""
for node in nodelist:
if node.nodeType == node.TEXT_NODE:
rc = rc + node.data
return rc
def close(self):
self.conn.close()
def list(self):
"""List files in directory"""
log.Log("Listing directory %s on WebDAV server" %
(self.directory,), 5)
self.conn.request( "PROPFIND", self.directory, None,
self.headers )
response = self.conn.getresponse()
document = response.read()
dom = xml.dom.minidom.parseString(document)
result = []
for href in dom.getElementsByTagName( '__NS1:href' ):
filename =
unquote(self._getText(href.childNodes).strip())
if filename.startswith( self.directory ):
filename = filename.replace(self.directory,'',1)
result.append(filename)
return result
def get(self, remote_filename, local_path):
"""Get remote filename, saving it to local_path"""
url = self.directory + remote_filename
log.Log("Retrieving %s from FTP server" % ( url ,), 5)
target_file = local_path.open("wb")
self.conn.request( "GET", url, None, self.headers )
response = self.conn.getresponse()
target_file.write( response.read() )
assert not target_file.close()
local_path.setdata()
def put(self, source_path, remote_filename = None):
"""Transfer source_path to remote_filename"""
if not remote_filename:
remote_filename = source_path.get_filename()
url = self.directory + remote_filename
source_file = source_path.open("rb")
log.Log("Saving %s on WebDAV server" % (url,), 5)
self.conn.request( "PUT", url, source_file.read(), self.headers
)
response = self.conn.getresponse()
response.read()
assert not source_file.close()
def delete(self, filename_list):
"""Delete files in filename_list"""
for filename in filename_list:
url = self.directory + filename
log.Log("Deleting %s from WebDAV server" % (url,), 5)
self.conn.request( "DELETE", url, None, self.headers )
response = self.conn.getresponse()
response.read()
# Dictionary relating protocol strings to backend_object classes.
protocol_class_dict = {"scp": scpBackend,
"ssh": scpBackend,
"file": LocalBackend,
"ftp": ftpBackend,
"rsync": rsyncBackend,
"webdav": webdavBackend}