[GNUnet-SVN] r18970 - in gnunet-update: . etc gnunet_update test test/new/test-package/src

[gnunet]

Author: harsha
Date: 2012-01-04 11:23:04 +0100 (Wed, 04 Jan 2012)
New Revision: 18970

Added:
   gnunet-update/test/new/test-package/src/group1-binary.c
   gnunet-update/test/test_install_manifest.py
Modified:
   gnunet-update/README
   gnunet-update/etc/gnunet-update.conf
   gnunet-update/gnunet_update/install.py
   gnunet-update/gnunet_update/install_manifest.py
   gnunet-update/gnunet_update/package.py
   gnunet-update/gnunet_update/update.py
   gnunet-update/test/new/test-package/src/Makefile.am
Log:
-added package groups support in packager, installer and updater

Modified: gnunet-update/README
===================================================================
--- gnunet-update/README        2012-01-04 10:21:55 UTC (rev 18969)
+++ gnunet-update/README        2012-01-04 10:23:04 UTC (rev 18970)
@@ -38,7 +38,7 @@
 There could be some problems in the way the dependencies are
 handled. The dependencies are tracked using their version
 information. Although most of the libraries follow the version
-numbering descibed by libtool. Some, e.g. libc, may not include
+numbering described by libtool. Some, e.g. libc, may not include
 version numbers as recommended by libtool. Such libraries must be explicitly
 handled. In case of libc only one number is present in its version. We 
 deem a libc library dependency compatible only if there is a libc with

Modified: gnunet-update/etc/gnunet-update.conf
===================================================================
--- gnunet-update/etc/gnunet-update.conf        2012-01-04 10:21:55 UTC (rev 
18969)
+++ gnunet-update/etc/gnunet-update.conf        2012-01-04 10:23:04 UTC (rev 
18970)
@@ -1,5 +1,5 @@
 # This file is part of GNUnet.
-# (C) 2001--2011 Christian Grothoff (and other contributing authors)
+# (C) 2011 Christian Grothoff (and other contributing authors)
 #
 # GNUnet is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published

Modified: gnunet-update/gnunet_update/install.py
===================================================================
--- gnunet-update/gnunet_update/install.py      2012-01-04 10:21:55 UTC (rev 
18969)
+++ gnunet-update/gnunet_update/install.py      2012-01-04 10:23:04 UTC (rev 
18970)
@@ -1,5 +1,5 @@
 # This file is part of GNUnet.
-# (C) 2001--2011 Christian Grothoff (and other contributing authors)
+# (C) 2011, 2012 Christian Grothoff (and other contributing authors)
 #
 # GNUnet is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published
@@ -33,17 +33,23 @@
 import install_manifest
 from metadata import Metadata
 from config import GnunetUpdateConfig
+from file import ExecutableFileObject, FileObject
 
 def usage():
     """Print helpful usage information."""    
     print """
-Usage arguments: [options] <package_file> </install/location>
+Usage arguments: [options] <package_file> [install/location]
 This script tries to install the contents in the package file in the given 
 location.
 
+    package_file              : The package file
+    install/location          : The directory where to install the package. It
+                                is optional only for group listings, see -l
 Options:
     -h, --help                : prints this message
     -c, --config=FILENAME     : use configuration file FILENAME
+    -l, --list-groups         : list the available groups
+    -g, --group=GROUP         : install GROUP (multiple options are allowed)
 """
 
 def shared_library_setup(shared_library_paths):
@@ -69,11 +75,16 @@
 def main():
     """Execution start point."""
     external_config_file = None
+    list_groups = False
+    install_group_names = set()
 
     try:
         opts, args = getopt.getopt(sys.argv[1:], 
-                                   "c:h", 
-                                   ["help", "config="])
+                                   "c:g:lh", 
+                                   ["help", 
+                                    "config=", 
+                                    "list-groups",
+                                    "group="])
     except getopt.GetoptError, err:
         print err
         print "Execption occured"
@@ -86,8 +97,17 @@
             sys.exit(0)
         elif option in ("-c", "--config"):
             external_config_file = value
+        elif option in ("-l", "--list-groups"):
+            list_groups = True
+        elif option in ("-g", "--group"):
+            install_group_names.add(value)
     
-    if len(args) != 2:
+    if list_groups:
+        if len(args) < 1:
+            print "Incorrect number of arguments"
+            usage()
+            sys.exit(1)
+    elif len(args) != 2:
         print "Incorrect number of arguments"
         usage()
         sys.exit(1)
@@ -98,6 +118,27 @@
     if metadata is None:
         sys.exit(2)
 
+    def print_available_groups():
+        print "Available groups in the package:"
+        for group in metadata.groups:
+            print "    " + group.name
+
+    if list_groups:
+        # Only list the available groups and exit
+        print_available_groups()
+        sys.exit(0)
+        
+    selected_groups = list()
+    for group_name in install_group_names:
+        matched_groups = filter(lambda group: group.name == group_name,
+                                metadata.groups)
+        if len(matched_groups) == 0:
+            print group_name + " is not available in the package"
+            print_available_groups()
+            exit(2)
+        else:
+            selected_groups.extend (matched_groups)
+
     #Platform check is done; now unpack the tarfile into destination directory
     package_tarfile = tarfile.open(args[0],'r')
     install_dir = args[1]
@@ -108,12 +149,31 @@
     
     available_libs = util.get_available_libs()    # already available 
dependencies
     installed_files = list()    # List of files that are installed from package
-    needed_deps = metadata.dependencies.keys() # Required dependencies
+    all_objects = metadata.binary_objects + metadata.other_objects
+    not_selected_groups = [ group for group in metadata.groups
+                            if group not in selected_groups ]
+    not_needed_objects = list(set(
+            reduce(list.__add__,
+                   map(lambda group: group.file_objects,
+                       not_selected_groups),
+                   [])))
+    to_be_installed_objects = [ file_object for file_object in all_objects
+                                if file_object not in not_needed_objects ]
+    
+    needed_deps = set(
+        reduce(lambda list1, list2: list1 + list2,
+               map(lambda binary_object: binary_object.deps,
+                   filter(lambda install_object:
+                              isinstance(install_object, ExecutableFileObject),
+                          to_be_installed_objects))))
+
+    needed_deps = list(needed_deps) # Convert from set to list
     to_be_installed_deps = util.filter_needed_deps(needed_deps,
                                                    available_libs)
 
+
     # Extract the package's main software contents
-    util.extract_install_prefix(metadata.binary_objects + 
metadata.other_objects,
+    util.extract_install_prefix(to_be_installed_objects,
                                 package_tarfile,
                                 install_dir,
                                 installed_files)
@@ -161,7 +221,8 @@
     # Write install manifest file
     install_manifest.write_to_file(install_dir, 
                                    installed_files,
-                                   installed_dep_files)
+                                   installed_dep_files,
+                                   map(lambda group: group.name, 
selected_groups))
     print "Installation Successful!"
     print "GNUNET has been installed at: " + install_dir
     shared_library_setup([os.path.join(install_dir, "lib"), 

Modified: gnunet-update/gnunet_update/install_manifest.py
===================================================================
--- gnunet-update/gnunet_update/install_manifest.py     2012-01-04 10:21:55 UTC 
(rev 18969)
+++ gnunet-update/gnunet_update/install_manifest.py     2012-01-04 10:23:04 UTC 
(rev 18970)
@@ -1,5 +1,5 @@
 # This file is part of GNUnet.
-# (C) 2001--2011 Christian Grothoff (and other contributing authors)
+# (C) 2011, 2012 Christian Grothoff (and other contributing authors)
 #
 # GNUnet is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published
@@ -23,13 +23,13 @@
 
 import os
 
-from file import FileObject
+from file import FileObject, DependencyFileObject
 
 
 """Class for holding install manifest information."""
 install_manifest_file_path = "share/gnunet-update/install-manifest"
 
-def write_to_file(install_dir, file_objects, dep_objects):
+def write_to_file(install_dir, file_objects, dep_objects, group_names):
     """Writes the contents of file_objects to a file in install_base_dir. Only
     the object name and hash are stored. Object path is not stored.
     
@@ -40,6 +40,7 @@
         manifest file
     dep_objects: The dependency objects which are to be included in the install
         manifest file
+    group_names: A list containing names of package groups installed
     """
     # Create the directory which holds install manifest file if not present
     install_manifest_file_dir = os.path.dirname(install_manifest_file_path)
@@ -62,6 +63,9 @@
             f.write(dep_object.name + ";" + dep_object.realname 
                     + ";" + dep_object.hash + '\n'),
         dep_objects)
+    f.write("%%\n")             # The section seperator
+    map(lambda group_name:f.write(group_name + '\n'), 
+        group_names) 
         
     f.close()
 
@@ -96,7 +100,7 @@
     dep_objects = list()
     while True:
         read_line = f.readline()
-        if len(read_line) == 0: # Reached EOF?
+        if read_line == "%%\n": # Reached EOF?
             break
         tokens = read_line.split(';')
         dep_name = tokens[0]
@@ -107,5 +111,13 @@
                                           dep_hash)
         dep_object.realname = dep_realname
         dep_objects.append(dep_object)
+
+    group_names = list()
+    while True:
+        read_line = f.readline()
+        if len(read_line) == 0: # Reached EOF?
+            break
+        group_names.append(read_line[:-1]);
+
     f.close()
-    return (file_objects, dep_objects)
+    return (file_objects, dep_objects, group_names)

Modified: gnunet-update/gnunet_update/package.py
===================================================================
--- gnunet-update/gnunet_update/package.py      2012-01-04 10:21:55 UTC (rev 
18969)
+++ gnunet-update/gnunet_update/package.py      2012-01-04 10:23:04 UTC (rev 
18970)
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # This file is part of GNUnet.
-# (C) 2001--2011 Christian Grothoff (and other contributing authors)
+# (C) 2011 Christian Grothoff (and other contributing authors)
 #
 # GNUnet is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published
@@ -69,8 +69,8 @@
                                 script in the given source tree. Multiple
                                 number of such options can be specified. These
                                 must be enclosed in double quotes.  
-                                  E.g: -c "--prefix=/opt/gnunet" 
-                                       -c "--with-extractor=/opt/Extractor"
+                                  E.g: -o "--prefix=/opt/gnunet" 
+                                       -o "--with-extractor=/opt/Extractor"
                         
 """
 
@@ -119,6 +119,7 @@
 def get_deps(install_dir):
     """Extract dependencies from ldd output."""
 
+    # A mapping from a file name to a group
     reverse_group_map = dict()
     for group_name in config.groups:
         config.groups[group_name].matching_files = config.groups[
@@ -131,6 +132,7 @@
                 reverse_group_map[matching_file].add(config.groups[group_name])
                         
     def add_to_matching_group(file_object):
+        """ Function to add a given file object to corresponding group."""
         if file_object.name in reverse_group_map:
             for group in reverse_group_map[file_object.name]:
                 if group.file_objects is None:

Modified: gnunet-update/gnunet_update/update.py
===================================================================
--- gnunet-update/gnunet_update/update.py       2012-01-04 10:21:55 UTC (rev 
18969)
+++ gnunet-update/gnunet_update/update.py       2012-01-04 10:23:04 UTC (rev 
18970)
@@ -1,5 +1,5 @@
 # This file is part of GNUnet.
-# (C) 2001--2011 Christian Grothoff (and other contributing authors)
+# (C) 2011, 2012 Christian Grothoff (and other contributing authors)
 #
 # GNUnet is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published
@@ -19,19 +19,24 @@
 # File:    gnunet_update/update.py
 # Author:  Sree Harsha Totakura
 #
-# Updates an existing installation of gnunet to a new release
+""" 
+Updates an existing installation of gnunet to a new release.
+Only the package groups in the existing installation are updated. If an
+existing group is not available in the update, then that group is
+ignored(removed) 
+"""
 
 import os
 import getopt
 import sys
 import shutil
-import subprocess
 import tarfile
 
 import util
 import install_manifest
 import metadata
 from config import GnunetUpdateConfig
+from file import ExecutableFileObject
 
 def usage():
     """Print helpful usage information."""
@@ -47,8 +52,11 @@
 
 def find_expired_objects(old_install_objects, new_install_objects):
     """
-    Compared old_install_objects with new_install_objects to find which older
-    install objects are not needed/updated in the new install objects
+    Compares old_install_objects with new_install_objects to find which older
+    install objects are not needed/updated in the new install objects. When an
+    object in old_install_object matches its name and hash a
+    new_install_objects, the older object is deemed to be unmodified in
+    new_install_objects and is removed from new_install_objects
 
     old_install_objects: list of objects which are existing in the installation
     new_install_objects: list of objects from the new installation
@@ -56,6 +64,7 @@
     """
 
     expired_objects = list()
+    # Sorting for keeing complexity to O(nlogn)
     old_install_objects.sort(key=(lambda f_object: f_object.name))
     new_install_objects.sort(key=(lambda f_object: f_object.name))
     old_obj_cnt = new_obj_cnt = 0
@@ -128,16 +137,34 @@
     # files. install_dir must point to an already installed location, if not
     # we'll fail
     (old_install_objects,
-     old_install_deps) = install_manifest.read_from_file(install_dir)
+     old_install_deps,
+     old_group_names) = install_manifest.read_from_file(install_dir)
 
     # To be refreshed objects are the objects which are present in the older
     # version and the newer version, however, in the newer version their hash
     # is different which means they have new updated content
 
-    new_install_objects = (metadata.binary_objects +
-                           metadata.other_objects)
+    selected_groups = list()
+    for group_name in old_group_names:
+        matched_groups = filter(lambda group: group.name == group_name,
+                                metadata.groups)
+        if len(matched_groups) == 0:
+            print group_name + " is not available in the package -- Ignoring"
+        else:
+            selected_groups.extend (matched_groups)
+    not_selected_groups = [ group for group in metadata.groups
+                            if group not in selected_groups ]
+    not_needed_objects = list(set(
+            reduce(list.__add__,
+                   map(lambda group: group.file_objects,
+                       not_selected_groups),
+                   [])))
+    all_objects = metadata.binary_objects + metadata.other_objects
+    to_be_installed_objects = [ file_object for file_object in all_objects
+                                if file_object not in not_needed_objects ]
+    # IMPORTANT: find_expired_objects modified the argument 2 list
     to_be_deleted_objects = find_expired_objects(old_install_objects,
-                                                 new_install_objects)
+                                                 to_be_installed_objects)
 
     # Delete objects in the to_be_deleted list
     deleted_objects = list()
@@ -163,13 +190,19 @@
     map(lambda f_object: installed_files.append(f_object), retained_objects)
     
     # Extract the files we need
-    util.extract_install_prefix(new_install_objects,
+    util.extract_install_prefix(to_be_installed_objects,
                                 package_tarfile,
                                 install_dir,
                                 installed_files)
 
     # Dependency update and handling
-    needed_deps = metadata.dependencies.keys() # Required dependencies
+    needed_deps = list(set(
+        reduce(lambda list1, list2: list1 + list2,
+               map(lambda binary_object: binary_object.deps,
+                   filter(lambda install_object:
+                              isinstance(install_object, ExecutableFileObject),
+                          to_be_installed_objects)))))
+
     expired_deps = find_expired_objects(old_install_deps,
                                         needed_deps)
     retained_deps = [dep for dep in old_install_deps
@@ -223,7 +256,8 @@
     # Write install manifest file
     install_manifest.write_to_file(install_dir, 
                                    installed_files,
-                                   installed_dep_files)
+                                   installed_dep_files,
+                                   map(lambda group: group.name, 
selected_groups))
     
 if "__main__" == __name__:
     main()

Modified: gnunet-update/test/new/test-package/src/Makefile.am
===================================================================
--- gnunet-update/test/new/test-package/src/Makefile.am 2012-01-04 10:21:55 UTC 
(rev 18969)
+++ gnunet-update/test/new/test-package/src/Makefile.am 2012-01-04 10:23:04 UTC 
(rev 18970)
@@ -54,7 +54,8 @@
                binary-libfunadd \
                binary-libfundel \
                binary-libfunmod \
-               binary-libnochange
+               binary-libnochange \
+               group1-binary
 
 test_binary_SOURCES = test-binary.c 
 
@@ -73,3 +74,5 @@
 
 binary_libnochange_SOURCES = binary-libnochange.c libnochange.h
 binary_libnochange_LDADD = libnochange.la
+
+group1_binary_SOURCES = group1-binary.c

Added: gnunet-update/test/new/test-package/src/group1-binary.c
===================================================================
--- gnunet-update/test/new/test-package/src/group1-binary.c                     
        (rev 0)
+++ gnunet-update/test/new/test-package/src/group1-binary.c     2012-01-04 
10:23:04 UTC (rev 18970)
@@ -0,0 +1,35 @@
+/*
+     This file is part of GNUnet.
+     (C) 2011, 2012 Christian Grothoff (and other contributing authors)
+
+     GNUnet is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 3, or (at your
+     option) any later version.
+
+     GNUnet is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with GNUnet; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+*/
+
+
+/**
+ * @file test/new/test-package/src/group1-binary.c
+ * @brief binary which is to be grouped under group1 and updated in
+           the new version
+ * @author Sree Harsha Totakura <address@hidden> 
+ */
+
+/**
+ * The main execution function
+ */
+int main()
+{
+  return 1; //return 0 (older)
+}

Added: gnunet-update/test/test_install_manifest.py
===================================================================
--- gnunet-update/test/test_install_manifest.py                         (rev 0)
+++ gnunet-update/test/test_install_manifest.py 2012-01-04 10:23:04 UTC (rev 
18970)
@@ -0,0 +1,133 @@
+# This file is part of GNUnet.
+# (C) 2011, 2012 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+#File:     test/test_metadata.py
+#Author:   Sree Harsha Totakura
+#
+
+"""Test case file for metadata file utility."""
+
+
+import unittest
+import tempfile
+import platform
+import shutil
+
+import __init__
+import gnunet_update.file as file
+from gnunet_update.group import Group
+# from gnunet_update.file import FileObject, DependencyFileObject, 
ExecutableFileObject
+import gnunet_update.install_manifest as install_manifest
+
+class TestInstallManifest(unittest.TestCase):
+    """Metadata test case class."""
+    deps_1 = list()
+    deps_2 = list()
+    binary_objects = list()
+    other_objects = list()
+    groups = list()
+
+    deps_1.append(file.DependencyFileObject("dep1_1",
+                                            
"/tmp/dependencies/dep1_1.so.1.0.0",
+                                            "hash-dep_1_0"))
+    deps_1[-1].realname = "dep1_1.so.1.0.0"
+    deps_1.append(file.DependencyFileObject("dep1_2",
+                                            
"/tmp/dependencies/dep1_2.so.1.1.0",
+                                            "hash-dep_1_1"))
+    deps_1[-1].realname = "dep1_2.so.1.1.0"
+
+    deps_2.append(file.DependencyFileObject("dep2_1",
+                                            
"/tmp/dependencies/dep2_1.so.1.0.0",
+                                            "hash-dep_2_1"))
+    deps_2[-1].realname = "dep2_1.so.1.0.0"
+    deps_2.append(file.DependencyFileObject("dep2_2",
+                                            
"/tmp/dependencies/dep2_2.so.1.1.0",
+                                            "hash-dep_2_2"))
+    deps_2[-1].realname = "dep2_2.so.1.1.0"
+
+    binary_objects.append(file.ExecutableFileObject("test-binary1"))
+    for dep in deps_1:
+        binary_objects[-1].add_dependency(dep)
+    binary_objects[-1].hash = "hash-binary_object_1"
+    
+    binary_objects.append(file.ExecutableFileObject("test-binary2"))
+    for dep in deps_2:
+        binary_objects[-1].add_dependency(dep)
+    binary_objects[-1].hash = "hash-binary_object_2"
+
+    other_objects.append(file.FileObject("other1_1",
+                                         "hash-other1_1"))
+    other_objects.append(file.FileObject("other1_2",
+                                         "hash-other1_2"))
+
+    groups.append(Group("group1"))
+    groups[-1].file_objects = [binary_objects[0], other_objects[0]]
+    
+    groups.append(Group("group2"))
+    groups[-1].file_objects = [binary_objects[1], other_objects[1]]
+
+
+    def test_install_manifest_read_write(self):
+        """Test case to test install manifest file writing and reading."""
+
+        # Make a temporary directory
+        tempdir = tempfile.mkdtemp(prefix='gnunet-test-')
+
+        # Test install manifest file writing
+        install_manifest.write_to_file(tempdir, 
+                                       self.binary_objects + 
self.other_objects,
+                                       self.deps_1 + self.deps_2,
+                                       map(lambda group: group.name,
+                                           self.groups))
+        # Test install manifest file reading
+        (objects, deps, groups) = install_manifest.read_from_file(tempdir)
+        
+        # Assert all objects are present
+        for file_object in self.binary_objects + self.other_objects:
+            self.assertTrue(file_object in objects)
+        # Assert all dependencies are present
+        for dep in self.deps_1 + self.deps_2:
+            self.assertTrue(dep in deps)
+        
+        for group in self.groups:
+            self.assertTrue(group.name in groups)
+
+        # Remove the temporary directory
+        shutil.rmtree(tempdir)
+
+    def test_install_manifest_null_read_write(self):
+        """Test case to test install manifest file with minimal values."""
+
+        tempdir = tempfile.mkdtemp(prefix='gnunet-test-')
+        # Write the install manifest
+        install_manifest.write_to_file(tempdir,
+                                       [],
+                                       [],
+                                       [])
+        # Read the install manifest
+        (objects, deps, groups) = install_manifest.read_from_file(tempdir)
+        # Assert the values are empty
+        self.assertTrue(len(objects) == 0)
+        self.assertTrue(len(deps) == 0)
+        self.assertTrue(len(groups) == 0)
+        # Remove the temp directory
+        shutil.rmtree(tempdir)
+        
+
+if __name__ == '__main__':
+    unittest.main()


Property changes on: gnunet-update/test/test_install_manifest.py
___________________________________________________________________
Added: svn:executable
   + *