[Savannah-cvs] /srv/bzr/administration r260: Generate backup script from a simpler spec (but not from #file: tags in the docs) + check that files are correctly downloaded

[Sylvain Beucler]

------------------------------------------------------------
revno: 260
committer: Sylvain Beucler <address@hidden>
branch nick: infra
timestamp: Sun 2011-02-13 15:59:24 +0100
message:
  Generate backup script from a simpler spec (but not from #file: tags in the 
docs) + check that files are correctly downloaded
added:
  backup/base.chb
  backup/cherry-backup.py
modified:
  backup/dl.sh

=== added file 'backup/base.chb'
--- a/backup/base.chb   1970-01-01 00:00:00 +0000
+++ b/backup/base.chb   2011-02-13 14:59:24 +0000
@@ -0,0 +1,126 @@
+# These are the files necessary to rebuild the system.  They come as a
+# complement to the .txt instructions files from the 'administration'
+# repository.  Ideally we should automatically generate this file
+# using the '#file:' annotations in those files.
+
+# Do not backup generatable or sed-able files.  Document how to
+# produce them instead.
+
+# We should make these files public so that people could easily
+# reproduce the Savannah configuration.  Backup confidential files
+# (such as 'authorized_files') using 'dl-confidential.sh'.
+
address@hidden:/ -> colonialone.fsf.org/
+  /root/.profile
+  /root/remote_backup.sh
+
+  # blah
+  /home/syncaliases/00_aliases/aliases
+  /home/syncaliases/00_aliases/README
+  /home/syncaliases/.ssh/authorized_keys
+
+  /etc/aliases
+  /etc/cron.daily/backup-bind
+  /etc/diffmon/diffmon.cf
+ 
+  /etc/xen/xend-config.sxp
+  /etc/xen/auto/***
+  /etc/xen/disabled/***
+ 
+  /etc/network/interfaces
+  /etc/network/firewall.sh
+
address@hidden:/ -> frontend.in.sv.gnu.org/
+  /etc/cron.daily/sv_list_groups
+  /etc/cron.d/sv_export
+
+  /etc/savane/.savane.conf.php
+  /etc/savane/savane.conf.pl
+
+  /etc/apache2/sites-available/***
+  /etc/apache2/conf.d/detect_bot.conf
+
address@hidden:/ -> vcs-noshell.in.sv.gnu.org/
+  /etc/init.d/cvs-permissions
+  /etc/init.d/cvs_lockdirs
+  /etc/libnss-mysql.cfg
+  /etc/libnss-mysql-root.cfg
+
+  /etc/cron.hourly/bzr_commit_mail_notification
+  /etc/cron.d/rsync_external_cvs_repositories
+  /etc/cron.d/sv
+  /etc/cron.d/cvs2git
+  /etc/cron.d/truncate-gitcvs-db-log
+
+  /etc/apache2/sites-available/***
+  /etc/apache2/conf.d/detect_bot.conf
+  /etc/apache2/conf.d/rlimit
+  /etc/apache2/conf.d/status
+
+  --exclude /var/www/*/webalizer/*
+  /var/www/bzr/***
+  /var/www/cvs/***
+  /var/www/git/***
+  /var/www/hg/***
+  /var/www/svn/***
+  /var/www/off-site/README
+  /var/www/off-site/hgweb/***
+  /var/www/off-site/viewvc/viewvc.conf
+  /var/www/off-site/viewvc/templates/include/header.ezt
+
+  /etc/gitweb.conf
+  /etc/cgitrc
+  /etc/mercurial/***
+
+[shell]
+# Mangle passwords (TODO: split them in separate file)
+sed -i -e 's/^password.*/password XXXXX/' \
+  vcs-noshell.in.sv.gnu.org/etc/libnss-mysql.cfg \
+  vcs-noshell.in.sv.gnu.org/etc/libnss-mysql-root.cfg
+# TODO: document hgweb/viewvc/cgit/etc. configurations instead of
+# copying them.
+[/shell]
+
address@hidden:/ -> sftp.in.sv.gnu.org/
+  /etc/cron.d/download-tidyperms
+  /etc/cron.d/download-timestamp
+  /etc/cron.d/sv
+ 
+  /etc/mirmon.conf
+ 
+  /etc/apache2/sites-available/***
+  /etc/apache2/conf.d/sv_dotsig
+ 
+  --exclude /var/www/*/webalizer/*
+  --exclude /var/www/arch/google*
+  /var/www/arch/***
+  /var/www/download/***
+  /var/www/audio-video/***
+ 
+  /srv/download/00_MIRRORS.*
+
address@hidden:/ -> internal.in.sv.gnu.org/
+  /usr/sbin/rmlist
+  /usr/sbin/config_list
+  /usr/lib/mailman/bin/change_pw
+ 
+  /etc/mysql/my.cnf
+  /etc/mysql/my-huge.cnf
+  /etc/exim4/update-exim4.conf.conf
+  /etc/munin/munin.conf
+ 
+  /etc/cron.d/sv
+
address@hidden:/ -> builder.in.sv.gnu.org/
+  /usr/src/patched/README
+  /usr/src/patched/cvs-patches/***
+  /usr/src/patched/webalizer.changelog
+  /usr/src/patched/debs/README
+
address@hidden:/ -> savannah-backup.gnu.org/
+  /root/restore.sh
+
+  /etc/cron.daily/check-savannah-backup
+  /etc/cron.d/savannah-backup
+ 
+  /mnt/backup/backup.sh

=== added file 'backup/cherry-backup.py'
--- a/backup/cherry-backup.py   1970-01-01 00:00:00 +0000
+++ b/backup/cherry-backup.py   2011-02-13 14:59:24 +0000
@@ -0,0 +1,180 @@
+#!/usr/bin/python
+
+# Cherry Backup
+# Copyright (C) 2011  Sylvain Beucler
+#
+# Cherry Backup is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# Cherry Backup is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Backup script to get the minimum set of files necessary to rebuild a
+# system.  We also check if all files are well rsync'd, so we can
+# track typos and renamed files.
+
+# Do not backup generatable or sed-able files.  Document how to
+# produce them instead.
+
+
+# Input file (base.chb) format:
+"""
address@hidden:/ -> mybox-backup/
+  /etc/aliases
+  /etc/mysql/my.cnf
+  /etc/exim4/update-exim4.conf.conf
+  /etc/munin/munin.conf
+
+  /etc/apache2/sites-available/***
+
+  --exclude /var/www/*/webalizer/*
+  /var/www/site1/***
+  /var/www/site2/***
+
+  /etc/libnss-mysql.cfg
+  /etc/libnss-mysql-root.cfg
+
+[shell]
+# Mangle passwords
+sed -i -e 's/^password.*/password XXXXX/' \
+  mybox-backup/etc/libnss-mysql.cfg \
+  mybox-backup/etc/libnss-mysql-root.cfg
+[/shell]
+"""
+
+import re
+import os.path
+import subprocess
+import glob
+
+filename_in = 'base.chb'
+fin = open(filename_in)
+fout = open('dl.sh', 'w')
+cur_dir = None
+mode_shell_input = False
+mode_files = False
+print_separation_line = False
+already_seen_paths = {}
+files = {}
+
+fout.write("#!/bin/bash\n\n")
+fout.write("# Generated by Cherry Backup - do not modify directly, edit '%s' 
instead\n" % filename_in)
+
+while True:
+    # "Parse" line-by-line
+    line = fin.readline()
+    if not line:
+        break
+
+    # Shell throughput mode
+    if mode_shell_input:
+        if line.startswith('[/shell]'):
+            mode_shell_input = False
+        else:
+            fout.write(line)
+    # Config mode
+    else:
+        # comments
+        if line.strip().startswith('#'):
+            pass
+        elif line.strip() == '':
+            print_separation_line = True
+        # section
+        elif not line.startswith(' ') and not line.startswith('\t'):
+            # shell subsection
+            if line.startswith('[shell]'):
+                fout.write("  \\\n  --exclude '*'\n")
+                mode_shell_input = True
+                mode_files = False
+            # host section
+            else:
+                match = re.match(r'^([^ ]+) -> ([^ ]+)\n', line)
+                if match:
+                    src = match.groups()[0]
+                    cur_dir = match.groups()[1]
+                    if not cur_dir.endswith('/'):
+                        cur_dir += '/'
+                    if mode_files:
+                        fout.write("  \\\n  --exclude '*'\n")
+                        mode_files = False
+                    fout.write("\n")
+                    fout.write("\n")
+                    fout.write("echo '* %s -> %s'\n" % (src, cur_dir))
+                    fout.write("rsync -zavHS --delete-excluded %s %s \\\n" % 
(src, cur_dir))
+                    fout.write("  \\\n")
+                    fout.write("  --exclude '*~' \\\n")
+                    already_seen_paths = {}
+                    files[cur_dir] = []
+                    print_separation_line = True
+        # Files: include or rsync option, with leading space
+        else:
+            mode_files = True
+            if print_separation_line:
+                    fout.write("  \\\n")
+                    print_separation_line = False
+            line = line.strip()
+            # rsync option
+            if line.startswith('-'):
+                fout.write("  " + line + " \\\n")
+            # include
+            else:
+                fullpath_elts = line.split('/')  # /usr/src/a -> ['', 'usr', 
'src', 'a']
+                i = 0
+                # Write each path directory
+                nb = len(fullpath_elts)
+                if fullpath_elts[-1] == '***':
+                    nb -= 1
+                for i in range(2, nb):
+                    cur_path = '/'.join(fullpath_elts[0:i])
+                    if (i < len(fullpath_elts)):
+                        cur_path += '/'
+                    if not already_seen_paths.has_key(cur_path):
+                        already_seen_paths[cur_path] = True
+                        fout.write("  --include '%s' \\\n" % 
cur_path.replace("'", "'\\''"))
+                fout.write("  --include '%s' \\\n" % line.replace("'", 
"'\\''"))
+                files[cur_dir].append(line)
+if mode_files:
+    fout.write("  \\\n  --exclude '*'\n")
+    mode_files = False
+fout.close()
+
+
+# Run the generated script
+subprocess.call(['bash', 'dl.sh'])
+
+
+# Check if specified files were truly rsync'd
+print ""
+for host in files:
+    for bakpath in files[host]:
+        bakpath = re.sub('^/*', '', bakpath)
+        # There's a wildcard:
+        if bakpath.find('*') != -1 or bakpath.find('?') != -1:
+            bakdir = re.sub('\**$', '', bakpath)
+            if bakpath.endswith('*') and bakdir.endswith('/'):
+                if not os.path.isdir(os.path.join(host,bakdir)):
+                    print "ERROR: " + os.path.join(host,bakdir) + " is not a 
directory"
+            elif not glob.glob(os.path.join(host,bakpath)):
+                print "ERROR: " + os.path.join(host,bakpath) + " has no match"
+            # TODO: we don't support several situations ('**'
+            # notation; trailing '*' in addition to other wildcard
+            # usage; ...?)
+        # Bare file or (empty) directory:
+        elif not os.path.exists(os.path.join(host,bakpath)):
+            print "ERROR: " + os.path.join(host,bakpath) + " is missing"
+
+
+# TODO: If target directory is a SVN/Git/Bzr repository, prepare it
+# for commit (deleted files marked as deleted, new files marked as
+# added, etc.).  Do not commit, but support being run several times
+# (aka support files already marked for addition/suppression but
+# non-committed, possibly with a simple 'git checkout' before the
+# rsync).

=== modified file 'backup/dl.sh'
--- a/backup/dl.sh      2010-12-21 19:18:00 +0000
+++ b/backup/dl.sh      2011-02-13 14:59:24 +0000
@@ -1,19 +1,10 @@
 #!/bin/bash
 
-# These are the files necessary to rebuild the system.  They come as a
-# complement to the .txt instructions files from the 'administration'
-# repository.  Ideally we should automatically generate this file
-# using the '#file:' annotations in those files (we could also check
-# if all files are well rsync'd, so we could track typos).
-
-# Do not backup generatable or sed-able files.  Document how to
-# produce them instead.
-
-# We should make these files public so that people could easily
-# reproduce the Savannah configuration.  Backup confidential files
-# (such as 'authorized_files') using 'dl-confidential.sh'.
-
-rsync -avHS --delete-excluded address@hidden:/ colonialone.fsf.org/ \
+# Generated by Cherry Backup - do not modify directly, edit 'base.chb' instead
+
+
+echo '* address@hidden:/ -> colonialone.fsf.org/'
+rsync -zavHS --delete-excluded address@hidden:/ colonialone.fsf.org/ \
   \
   --exclude '*~' \
   \
@@ -36,42 +27,43 @@
   --include '/etc/diffmon/' \
   --include '/etc/diffmon/diffmon.cf' \
   \
-  --include '/etc/' \
   --include '/etc/xen/' \
   --include '/etc/xen/xend-config.sxp' \
   --include '/etc/xen/auto/***' \
   --include '/etc/xen/disabled/***' \
   \
-  --include '/etc/' \
   --include '/etc/network/' \
   --include '/etc/network/interfaces' \
   --include '/etc/network/firewall.sh' \
   \
   --exclude '*'
 
-rsync -avHS --delete-excluded address@hidden:/ frontend.in.sv.gnu.org/ \
+
+echo '* address@hidden:/ -> frontend.in.sv.gnu.org/'
+rsync -zavHS --delete-excluded address@hidden:/ frontend.in.sv.gnu.org/ \
   \
   --exclude '*~' \
   \
   --include '/etc/' \
   --include '/etc/cron.daily/' \
   --include '/etc/cron.daily/sv_list_groups' \
+  --include '/etc/cron.d/' \
   --include '/etc/cron.d/sv_export' \
   \
-  --include '/etc/' \
   --include '/etc/savane/' \
   --include '/etc/savane/.savane.conf.php' \
   --include '/etc/savane/savane.conf.pl' \
   \
-  --include '/etc/' \
   --include '/etc/apache2/' \
-  --include '/etc/apache2/sites-availables/***' \
+  --include '/etc/apache2/sites-available/***' \
   --include '/etc/apache2/conf.d/' \
   --include '/etc/apache2/conf.d/detect_bot.conf' \
   \
   --exclude '*'
 
-rsync -avHS --delete-excluded address@hidden:/ vcs-noshell.in.sv.gnu.org/ \
+
+echo '* address@hidden:/ -> vcs-noshell.in.sv.gnu.org/'
+rsync -zavHS --delete-excluded address@hidden:/ vcs-noshell.in.sv.gnu.org/ \
   \
   --exclude '*~' \
   \
@@ -82,7 +74,6 @@
   --include '/etc/libnss-mysql.cfg' \
   --include '/etc/libnss-mysql-root.cfg' \
   \
-  --include '/etc/' \
   --include '/etc/cron.hourly/' \
   --include '/etc/cron.hourly/bzr_commit_mail_notification' \
   --include '/etc/cron.d/' \
@@ -91,7 +82,6 @@
   --include '/etc/cron.d/cvs2git' \
   --include '/etc/cron.d/truncate-gitcvs-db-log' \
   \
-  --include '/etc/' \
   --include '/etc/apache2/' \
   --include '/etc/apache2/sites-available/***' \
   --include '/etc/apache2/conf.d/' \
@@ -99,9 +89,9 @@
   --include '/etc/apache2/conf.d/rlimit' \
   --include '/etc/apache2/conf.d/status' \
   \
+  --exclude /var/www/*/webalizer/* \
   --include '/var/' \
   --include '/var/www/' \
-  --exclude '/var/www/*/webalizer/*' \
   --include '/var/www/bzr/***' \
   --include '/var/www/cvs/***' \
   --include '/var/www/git/***' \
@@ -116,7 +106,6 @@
   --include '/var/www/off-site/viewvc/templates/include/' \
   --include '/var/www/off-site/viewvc/templates/include/header.ezt' \
   \
-  --include '/etc/' \
   --include '/etc/gitweb.conf' \
   --include '/etc/cgitrc' \
   --include '/etc/mercurial/***' \
@@ -130,7 +119,8 @@
 # copying them.
 
 
-rsync -avHS --delete-excluded address@hidden:/ sftp.in.sv.gnu.org/ \
+echo '* address@hidden:/ -> sftp.in.sv.gnu.org/'
+rsync -zavHS --delete-excluded address@hidden:/ sftp.in.sv.gnu.org/ \
   \
   --exclude '*~' \
   \
@@ -140,19 +130,17 @@
   --include '/etc/cron.d/download-timestamp' \
   --include '/etc/cron.d/sv' \
   \
-  --include '/etc/' \
   --include '/etc/mirmon.conf' \
   \
-  --include '/etc/' \
   --include '/etc/apache2/' \
   --include '/etc/apache2/sites-available/***' \
   --include '/etc/apache2/conf.d/' \
   --include '/etc/apache2/conf.d/sv_dotsig' \
   \
+  --exclude /var/www/*/webalizer/* \
+  --exclude /var/www/arch/google* \
   --include '/var/' \
   --include '/var/www/' \
-  --exclude '/var/www/*/webalizer/*' \
-  --exclude '/var/www/arch/google*' \
   --include '/var/www/arch/***' \
   --include '/var/www/download/***' \
   --include '/var/www/audio-video/***' \
@@ -163,7 +151,9 @@
   \
   --exclude '*'
 
-rsync -avHS --delete-excluded address@hidden:/ internal.in.sv.gnu.org/ \
+
+echo '* address@hidden:/ -> internal.in.sv.gnu.org/'
+rsync -zavHS --delete-excluded address@hidden:/ internal.in.sv.gnu.org/ \
   \
   --exclude '*~' \
   \
@@ -171,7 +161,6 @@
   --include '/usr/sbin/' \
   --include '/usr/sbin/rmlist' \
   --include '/usr/sbin/config_list' \
-  --include '/usr/' \
   --include '/usr/lib/' \
   --include '/usr/lib/mailman/' \
   --include '/usr/lib/mailman/bin/' \
@@ -180,17 +169,20 @@
   --include '/etc/' \
   --include '/etc/mysql/' \
   --include '/etc/mysql/my.cnf' \
+  --include '/etc/mysql/my-huge.cnf' \
+  --include '/etc/exim4/' \
   --include '/etc/exim4/update-exim4.conf.conf' \
+  --include '/etc/munin/' \
   --include '/etc/munin/munin.conf' \
   \
-  --include '/etc/' \
   --include '/etc/cron.d/' \
   --include '/etc/cron.d/sv' \
   \
   --exclude '*'
 
 
-rsync -avHS --delete-excluded address@hidden:/ builder.in.sv.gnu.org/ \
+echo '* address@hidden:/ -> builder.in.sv.gnu.org/'
+rsync -zavHS --delete-excluded address@hidden:/ builder.in.sv.gnu.org/ \
   \
   --exclude '*~' \
   \
@@ -205,13 +197,17 @@
   \
   --exclude '*'
 
-rsync -avHS address@hidden:/ savannah-backup.gnu.org/ \
+
+echo '* address@hidden:/ -> savannah-backup.gnu.org/'
+rsync -zavHS --delete-excluded address@hidden:/ savannah-backup.gnu.org/ \
+  \
+  --exclude '*~' \
   \
   --include '/root/' \
   --include '/root/restore.sh' \
   \
   --include '/etc/' \
-  --include '/etc/cron.daily' \
+  --include '/etc/cron.daily/' \
   --include '/etc/cron.daily/check-savannah-backup' \
   --include '/etc/cron.d/' \
   --include '/etc/cron.d/savannah-backup' \