[Savannah-cvs] [SCM] Savane-cleanup framework branch, master, updated. fc09677073cbbd7e46d9329b8efec8d2a8c32067

[Sylvain Beucler]

This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Savane-cleanup framework".

The branch, master has been updated
       via  fc09677073cbbd7e46d9329b8efec8d2a8c32067 (commit)
      from  7552a50da01bd07ab62f229f407c399fbfbb1d67 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
http://git.savannah.gnu.org/cgit/savane-cleanup/framework.git/commit/?id=fc09677073cbbd7e46d9329b8efec8d2a8c32067

commit fc09677073cbbd7e46d9329b8efec8d2a8c32067
Author: Sylvain Beucler <address@hidden>
Date:   Sun Aug 2 23:38:26 2009 +0200

    LDIF export: more efficient (direct MySQL access) and cleaner (queries in 
the models)

diff --git a/TODO b/TODO
index 30ca8ce..50f71d8 100644
--- a/TODO
+++ b/TODO
@@ -1,7 +1,6 @@
 - models
 
-  - add DB indexes (db_model=True - but is that possible for
-    auth_user.username? :/)
+  - add DB indexes (db_model=True)
 
 - now we need the screens for users to modify them
 
diff --git a/src/savane/backend/auth_ldif_export.py 
b/src/savane/backend/auth_ldif_export.py
index 2e4b759..dbb0f7e 100644
--- a/src/savane/backend/auth_ldif_export.py
+++ b/src/savane/backend/auth_ldif_export.py
@@ -17,7 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Recommended indexes:
-# index                uid,sn,uidNumber,gidNumber,memberUid,shadowExpire eq
+# index                uid,uidNumber,gidNumber,memberUid,shadowExpire eq
 
 # TODO: most settings are hard-coded and need to be made configurable
 # - base: dc=savannah,dc=gnu,dc=org
@@ -27,10 +27,12 @@
 # - min uid: 1000
 # - min gid: 1000
 # - default group: cn=svusers / gid=1000
+# - loginShell: /usr/local/bin/sv_membersh
 
 import sys
 import codecs
 import base64, binascii
+from django.db import connection, models
 import savane.svmain.models as svmain_models
 
 # Convert stdout to UTF-8 - if the stdout is redirected to a file
@@ -71,21 +73,51 @@ userPassword:: e2NyeXB0fWt0YVZ1TFNDaEg0Wi4=
 structuralObjectClass: organizationalRole
 """
 
-#count = svmain_models.ExtendedUser.objects.count()
-#print str(count) + " users in the database."
-
-uidNumber=1000
-for user in svmain_models.ExtendedUser.objects.only('username', 'first_name', 
'last_name', 'email',
-                                                    'password', 'uidNumber', 
'gidNumber'):
-    uidNumber=uidNumber+1
-    ##if uidNumber == 0: # either non-assigned, or mistakenly assigned to root
-    #if uidNumber < 1000: # either non-assigned, or mistakenly assigned to 
privileged user
-    #    uidn = UidNumber()
-    #    uidn.save()
-    #    user.uidNumber = uidn
-    #    user.save()
-
-    cleanup = [user.first_name, user.last_name, user.email]
+import MySQLdb, settings
+MySQLdb.charset = 'UTF-8'
+conn = MySQLdb.connect(user=settings.DATABASE_USER,
+                       passwd=settings.DATABASE_PASSWORD,
+                       db=settings.DATABASE_NAME,
+                       use_unicode=True)
+
+# Alternatively:
+#from django.db import connection
+#connection.cursor() # establish connection - well looks like it does
+#conn = connection.connection # MySQL-specific connection - now using mysqldb
+
+
+##
+# Users
+##
+
+max_uid = 
svmain_models.ExtendedUser.objects.all().aggregate(models.Max('uidNumber'))['uidNumber__max']
+if max_uid < 1000: max_uid = 1000
+
+users_with_group = {}
+group_users = {}
+svmain_models.Membership.query_active_memberships_raw(conn, ('group_id', 
'username'))
+res = conn.store_result()
+for row in res.fetch_row(maxrows=0, how=1):
+    users_with_group[row['username']] = 1
+    if group_users.has_key(row['group_id']):
+        group_users[row['group_id']].append(row['username'])
+    else:
+        group_users[row['group_id']] = [row['username'],]
+
+user_saves = []
+svmain_models.ExtendedUser.query_active_users_raw(conn, ('username', 
'first_name', 'last_name', 'email',
+                                                         'password', 
'uidNumber', 'gidNumber'))
+res = conn.store_result()
+for row in res.fetch_row(maxrows=0):
+    (username, first_name, last_name, email, password, uidNumber, gidNumber) = 
row
+
+    #if uidNumber == 0: # either non-assigned, or mistakenly assigned to root
+    if uidNumber < 1000: # either non-assigned, or mistakenly assigned to 
privileged user
+        max_uid = max_uid + 1
+        user_saves.append((username, max_uid))
+        uidNumber = max_uid
+
+    cleanup = [first_name, last_name, email]
     for i in range(0, len(cleanup)):
         cleanup[i] = cleanup[i].replace('\n', ' ')
         cleanup[i] = cleanup[i].replace('\r', ' ')
@@ -93,33 +125,34 @@ for user in 
svmain_models.ExtendedUser.objects.only('username', 'first_name', 'l
     (first_name, last_name, email) = cleanup
 
     ldap_password = '{CRYPT}!'  # default = unusable password
-    if user.password.startswith('sha1$'):
-        # Django-specific algorithm: it sums 5-char-salt+pass instead
-        # of SSHA's pass+4-bytes-salt, so we can't store it in LDAP -
-        # /me curses django devs
-        pass
-    elif user.password.startswith('md5$$'):
-        # MD5 without salt
-        algo, empty, hash_hex = user.password.split('$')
-        if (len(hash_hex) == 32): # filter out empty or disabled passwords
-            ldap_password = "{MD5}" + 
base64.b64encode(binascii.a2b_hex(hash_hex))
-    elif user.password.startswith('md5$'):
-        # md5$salt$ vs. {SMD5} is similar to sha1$salt$ vs. {SSHA} -
-        # cf. above
-        pass
-    elif user.password.startswith('crypt$'):
-        # glibc crypt has improved algorithms, but where salt contains
-        # three '$'s, which Django doesn't support (since '$' is
-        # already the salt field separator). So this is only weak,
-        # passwd-style (not shadow-style) crypt.
-        algo, salt_hex, hash_hex = user.password.split('$')
-        # salt_hex is 2-chars long and already prepended to hash_hex
-        ldap_password = "{CRYPT}" + 
base64.b64encode(binascii.a2b_hex(hash_hex))
-    elif '$' not in user.password:
-        # MD5 without salt, alternate Django syntax
-        hash_hex = user.password
-        if (len(hash_hex) == 32): # filter out empty or disabled passwords
-            ldap_password = "{MD5}" + 
base64.b64encode(binascii.a2b_hex(hash_hex))
+    if users_with_group.has_key(username):
+        if password.startswith('sha1$'):
+            # Django-specific algorithm: it sums 5-char-salt+pass instead
+            # of SSHA's pass+4-bytes-salt, so we can't store it in LDAP -
+            # /me curses django devs
+            pass
+        elif password.startswith('md5$$'):
+            # MD5 without salt
+            algo, empty, hash_hex = password.split('$')
+            if (len(hash_hex) == 32): # filter out empty or disabled passwords
+                ldap_password = "{MD5}" + 
base64.b64encode(binascii.a2b_hex(hash_hex))
+        elif password.startswith('md5$'):
+            # md5$salt$ vs. {SMD5} is similar to sha1$salt$ vs. {SSHA} -
+            # cf. above
+            pass
+        elif password.startswith('crypt$'):
+            # glibc crypt has improved algorithms, but where salt contains
+            # three '$'s, which Django doesn't support (since '$' is
+            # already the salt field separator). So this is only weak,
+            # passwd-style (not shadow-style) crypt.
+            algo, salt_hex, hash_hex = password.split('$')
+            # salt_hex is 2-chars long and already prepended to hash_hex
+            ldap_password = "{CRYPT}" + 
base64.b64encode(binascii.a2b_hex(hash_hex))
+        elif '$' not in password:
+            # MD5 without salt, alternate Django syntax
+            hash_hex = password
+            if (len(hash_hex) == 32): # filter out empty or disabled passwords
+                ldap_password = "{MD5}" + 
base64.b64encode(binascii.a2b_hex(hash_hex))
 
     # Object classes:
     # - posixAccount: base class for libnss-ldap/pam-ldap support
@@ -135,28 +168,38 @@ userPassword: %(ldap_password)s
 uidNumber: %(uidNumber)d
 gidNumber: %(gidNumber)d
 homeDirectory: %(homedir)s
+loginShell: /usr/local/bin/sv_membersh
 objectClass: shadowAccount
 objectClass: posixAccount
 objectClass: inetOrgPerson
 objectClass: top
 structuralObjectClass: inetOrgPerson""" % {
-        'username' : user.username,
-        'full_name' : base64.b64encode((first_name + u' ' + 
last_name).encode('UTF-8')),
-        'last_name' : base64.b64encode((last_name or u'-').encode('UTF-8')),
+        'username' : username,
+        'full_name' : base64.b64encode((first_name + ' ' + 
last_name).encode('UTF-8')),
+        'last_name' : base64.b64encode((last_name or '-').encode('UTF-8')),
         'email' : email,
         'ldap_password' : ldap_password,
         'uidNumber' : uidNumber,
         'gidNumber' : 1000,
-        'homedir' : u'/home/' + user.username[:1] + u'/' + user.username[:2] + 
u'/' + user.username,
+        'homedir' : '/home/' + username[:1] + '/' + username[:2] + '/' + 
username,
         }
     # non-mandatory fields - slapadd doesn't accept empty fields apparently
     if len(first_name) > 0:
         print "givenName::" + base64.b64encode(first_name.encode('UTF-8'))
     # disallow login for users that are not part of any group
-    #if user.extendedgroup_set.count() == 0:
-    #    print "shadowExpire: 10" # timestamp - avoid 0 as it may be
-    #                             # interpreted at 'no expiration'
+    if not users_with_group.has_key(username):
+        # shadowExpire is a timestamp - avoid 0 as it may be
+        # interpreted as 'no expiration'
+        print "shadowExpire: 10"
+
+##
+# Groups
+##
 
+max_gid = 
svmain_models.ExtendedGroup.objects.all().aggregate(models.Max('gidNumber'))['gidNumber__max']
+if max_gid < 1000: max_gid = 1000
+
+# Create base 'svusers' group
 print u"""
 dn: cn=svusers,ou=groups,dc=savannah,dc=gnu,dc=org
 cn: svusers
@@ -164,9 +207,20 @@ gidNumber: 1000
 objectClass: posixGroup
 objectClass: top
 structuralObjectClass: posixGroup"""
-i=1000
-for group in svmain_models.ExtendedGroup.objects.only('name'):
-    i=i+1
+
+# Dump groups
+group_saves = []
+svmain_models.ExtendedGroup.query_active_groups_raw(conn, ('group_id', 'name', 
'gidNumber'))
+res = conn.store_result()
+#for group in svmain_models.ExtendedGroup.objects.only('name'):
+for row in res.fetch_row(maxrows=0):
+    (group_id, name, gidNumber) = row
+
+    if gidNumber < 1000: # either non-assigned, or mistakenly assigned to 
privileged user
+        max_gid = max_gid + 1
+        group_saves.append((group_id, max_gid))
+        gidNumber = max_gid
+
     print u"""
 dn: cn=%(name)s,ou=groups,dc=savannah,dc=gnu,dc=org
 cn: %(name)s
@@ -174,8 +228,14 @@ gidNumber: %(gidNumber)s
 objectClass: posixGroup
 objectClass: top
 structuralObjectClass: posixGroup""" % {
-     'name' : group.name,
-     'gidNumber' : i,
+     'name' : name,
+     'gidNumber' : gidNumber,
      }
-    for user in group.extendeduser_set.only('username'):
-        print "memberUid: " + user.username
+    if group_users.has_key(group_id):
+        for username in group_users[group_id]:
+            print "memberUid: " + username
+
+# TODO
+# - user_saves
+# - group_saves
+# with multi-line UPDATEs
diff --git a/src/savane/svmain/models.py b/src/savane/svmain/models.py
index b2fd83b..0eb6f6b 100644
--- a/src/savane/svmain/models.py
+++ b/src/savane/svmain/models.py
@@ -106,6 +106,18 @@ class ExtendedUser(auth_models.User):
     # Inherit specialized models.Manager with convenience functions
     objects = auth_models.UserManager()
 
+    @staticmethod
+    def query_active_users_raw(conn, fields):
+        """
+        Return efficient query with all the users; used by LDIF export
+        """
+        return conn.query("SELECT "
+                          + ",".join(fields)
+                          + " FROM auth_user JOIN svmain_extendeduser"
+                          + " ON auth_user.id = 
svmain_extendeduser.user_ptr_id"
+                          + " WHERE status = 'A'"
+                          )
+
     @models.permalink
     def get_absolute_url(self):
         return ('savane.svmain.user_detail', [self.username])
@@ -463,6 +475,18 @@ class ExtendedGroup(auth_models.Group):
     #patch_private_exclude_address text
     #cookbook_private_exclude_address text
 
+    @staticmethod
+    def query_active_groups_raw(conn, fields):
+        """
+        Return efficient query with all the users; used by LDIF export
+        """
+        return conn.query("SELECT "
+                          + ",".join(fields)
+                          + " FROM auth_group JOIN svmain_extendedgroup"
+                          + " ON auth_group.id = 
svmain_extendedgroup.group_ptr_id"
+                          + " WHERE status = 'A'"
+                          )
+
     def __unicode__(self):
         return self.name
 
@@ -508,6 +532,18 @@ class Membership(models.Model):
     # Deprecated
     #forum_flags int(11) default NULL
 
+    @staticmethod
+    def query_active_memberships_raw(conn):
+        """
+        Return efficient query with all the users; used by LDIF export
+        """
+        return conn.query("SELECT "
+                          + ",".join(fields)
+                          + " FROM svmain_membership JOIN auth_user"
+                          + " ON user_id = auth_user.id"
+                          + " WHERE admin_flags<>'P'"
+                          )
+
     def __unicode__(self):
         return "[%s is a member of %s]" % (self.user.username, self.group.name)
 

-----------------------------------------------------------------------

Summary of changes:
 TODO                                   |    3 +-
 src/savane/backend/auth_ldif_export.py |  174 +++++++++++++++++++++-----------
 src/savane/svmain/models.py            |   36 +++++++
 3 files changed, 154 insertions(+), 59 deletions(-)


hooks/post-receive
-- 
Savane-cleanup framework