[GNUnet-SVN] r16350 - in gnunet/src: datastore include

[gnunet]

Author: grothoff
Date: 2011-08-03 22:59:58 +0200 (Wed, 03 Aug 2011)
New Revision: 16350

Modified:
   gnunet/src/datastore/plugin_datastore_mysql.c
   gnunet/src/datastore/plugin_datastore_sqlite.c
   gnunet/src/include/gnunet_datastore_plugin.h
   gnunet/src/include/gnunet_datastore_service.h
Log:
improve API, speed up mysql

Modified: gnunet/src/datastore/plugin_datastore_mysql.c
===================================================================
--- gnunet/src/datastore/plugin_datastore_mysql.c       2011-08-03 19:53:43 UTC 
(rev 16349)
+++ gnunet/src/datastore/plugin_datastore_mysql.c       2011-08-03 20:59:58 UTC 
(rev 16350)
@@ -194,34 +194,34 @@
   /**
    * Prepared statements.
    */
-#define INSERT_ENTRY "INSERT INTO gn090 
(repl,type,prio,anonLevel,expire,rvalue,hash,vhash,value) VALUES 
(?,?,?,?,?,RAND(),?,?,?)"
+#define INSERT_ENTRY "INSERT INTO gn090 
(repl,type,prio,anonLevel,expire,rvalue,hash,vhash,value) VALUES 
(?,?,?,?,?,?,?,?,?)"
   struct GNUNET_MysqlStatementHandle *insert_entry;
   
 #define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?"
   struct GNUNET_MysqlStatementHandle *delete_entry_by_uid;
 
-#define COUNT_ENTRY_BY_HASH "SELECT count(*) FROM gn090 WHERE hash=?"
+#define COUNT_ENTRY_BY_HASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash) 
WHERE hash=?"
   struct GNUNET_MysqlStatementHandle *count_entry_by_hash;
   
-#define SELECT_ENTRY_BY_HASH "SELECT type,prio,anonLevel,expire,hash,value,uid 
FROM gn090 WHERE hash=? ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH "SELECT type,prio,anonLevel,expire,hash,value,uid 
FROM gn090 FORCE INDEX (idx_hash) WHERE hash=? ORDER BY uid LIMIT 1 OFFSET ?"
   struct GNUNET_MysqlStatementHandle *select_entry_by_hash;
 
-#define COUNT_ENTRY_BY_HASH_AND_VHASH "SELECT count(*) FROM gn090 WHERE hash=? 
AND vhash=?"
+#define COUNT_ENTRY_BY_HASH_AND_VHASH "SELECT count(*) FROM gn090 FORCE INDEX 
(idx_hash_vhash) WHERE hash=? AND vhash=?"
   struct GNUNET_MysqlStatementHandle *count_entry_by_hash_and_vhash;
 
-#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE hash=? AND vhash=? 
ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX 
(idx_hash_vhash) WHERE hash=? AND vhash=? ORDER BY uid LIMIT 1 OFFSET ?"
   struct GNUNET_MysqlStatementHandle *select_entry_by_hash_and_vhash;
  
-#define COUNT_ENTRY_BY_HASH_AND_TYPE "SELECT count(*) FROM gn090 WHERE hash=? 
AND type=?"
+#define COUNT_ENTRY_BY_HASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX 
(idx_hash_type_uid) WHERE hash=? AND type=?"
   struct GNUNET_MysqlStatementHandle *count_entry_by_hash_and_type;
 
-#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE hash=? AND type=? 
ORDER BY uid LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX 
(idx_hash_type_uid) WHERE hash=? AND type=? ORDER BY uid LIMIT 1 OFFSET ?"
   struct GNUNET_MysqlStatementHandle *select_entry_by_hash_and_type;
  
-#define COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT count(*) FROM gn090 WHERE 
hash=? AND vhash=? AND type=?"
+#define COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE 
INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=?"
   struct GNUNET_MysqlStatementHandle *count_entry_by_hash_vhash_and_type;
   
-#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE hash=? AND vhash=? 
AND type=? ORDER BY uid ASC LIMIT 1 OFFSET ?"
+#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX 
(idx_hash_vhash) WHERE hash=? AND vhash=? AND type=? ORDER BY uid ASC LIMIT 1 
OFFSET ?"
   struct GNUNET_MysqlStatementHandle *select_entry_by_hash_vhash_and_type;
 
 #define UPDATE_ENTRY "UPDATE gn090 SET 
prio=prio+?,expire=IF(expire>=?,expire,?) WHERE uid=?"
@@ -233,27 +233,32 @@
 #define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn090"
   struct GNUNET_MysqlStatementHandle *get_size;
 
-#define SELECT_IT_NON_ANONYMOUS "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE anonLevel=0 AND 
type=? ORDER BY uid DESC LIMIT 1 OFFSET ?"
+#define SELECT_IT_NON_ANONYMOUS "SELECT 
type,prio,anonLevel,expire,hash,value,uid "\
+   "FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) "\
+   "WHERE anonLevel=0 AND type=? AND "\
+   "(rvalue >= ? OR"\
+   "  NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) 
WHERE anonLevel=0 AND type=? AND rvalue>=?)) "\
+   "ORDER BY rvalue ASC LIMIT 1"
   struct GNUNET_MysqlStatementHandle *zero_iter;
 
-#define SELECT_IT_EXPIRATION "(SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM gn090 WHERE expire < ? ORDER BY 
prio ASC LIMIT 1) "\
-  "UNION "\
-  "(SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 ORDER BY prio 
ASC LIMIT 1) "\
-  "ORDER BY expire ASC LIMIT 1"
+#define SELECT_IT_EXPIRATION "SELECT type,prio,anonLevel,expire,hash,value,uid 
FROM gn090 FORCE INDEX (idx_expire) WHERE expire < ? ORDER BY expire ASC LIMIT 
1"
   struct GNUNET_MysqlStatementHandle *select_expiration;
 
-  // select type from (select rand() as v) AS t1 INNER JOIN gn090 ON 
expire>=t1.v limit 1;
+#define SELECT_IT_PRIORITY "SELECT type,prio,anonLevel,expire,hash,value,uid 
FROM gn090 FORCE INDEX (idx_prio) ORDER BY prio ASC LIMIT 1"
+  struct GNUNET_MysqlStatementHandle *select_priority;
 
-#define SELECT_IT_REPLICATION "SELECT 
type,prio,anonLevel,expire,hash,value,uid FROM "\
-  "(SELECT RAND() AS v) AS t1 INNER JOIN "\
-  "(SELECT MAX(repl) AS m FROM gn090) AS t2 INNER JOIN "\
-  "gn090 ON repl=t2.m AND"\
-  "       (rvalue>=t1.v OR"\
-  "        NOT EXISTS (SELECT 1 FROM gn090 WHERE repl=t2.m AND rvalue>=t1.v))"\
+#define SELECT_IT_REPLICATION "SELECT 
type,prio,anonLevel,expire,hash,value,uid "\
+  "FROM gn090 FORCE INDEX (idx_repl_rvalue) "\
+  "WHERE repl=? AND "\
+  " (rvalue>=? OR"\
+  "  NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_repl_rvalue) WHERE 
repl=? AND rvalue>=?)) "\
   "ORDER BY rvalue ASC "\
   "LIMIT 1"
   struct GNUNET_MysqlStatementHandle *select_replication;
 
+#define SELECT_MAX_REPL "SELECT MAX(repl) FROM gn090"
+  struct GNUNET_MysqlStatementHandle *max_repl;
+
 };
 
 
@@ -854,6 +859,8 @@
   unsigned int ipriority = priority;
   unsigned int ianonymity = anonymity;
   unsigned long long lexpiration = expiration.abs_value;
+  unsigned long long lrvalue = (unsigned long long) GNUNET_CRYPTO_random_u64 
(GNUNET_CRYPTO_QUALITY_WEAK,
+                                                                             
UINT64_MAX);
   unsigned long hashSize;
   unsigned long hashSize2;
   unsigned long lsize;
@@ -877,6 +884,7 @@
                              MYSQL_TYPE_LONG, &ipriority, GNUNET_YES,
                              MYSQL_TYPE_LONG, &ianonymity, GNUNET_YES,
                              MYSQL_TYPE_LONGLONG, &lexpiration, GNUNET_YES,
+                             MYSQL_TYPE_LONGLONG, &lrvalue, GNUNET_YES,
                              MYSQL_TYPE_BLOB, key, hashSize, &hashSize,
                              MYSQL_TYPE_BLOB, &vhash, hashSize2, &hashSize2,
                              MYSQL_TYPE_BLOB, data, lsize, &lsize, 
@@ -1234,16 +1242,16 @@
                                 PluginDatumProcessor proc, void *proc_cls)
 {
   struct Plugin *plugin = cls;
-  unsigned long long off;
-
-  off = (unsigned long long) offset;
+  unsigned long long rvalue = (unsigned long long) GNUNET_CRYPTO_random_u64 
(GNUNET_CRYPTO_QUALITY_WEAK,
+                                                                            
UINT64_MAX);
   execute_select (plugin,
                  plugin->zero_iter,
                  proc, proc_cls,
                  MYSQL_TYPE_LONG, &type, GNUNET_YES,
-                 MYSQL_TYPE_LONGLONG, &off, GNUNET_YES,
+                 MYSQL_TYPE_LONGLONG, &rvalue, GNUNET_YES,
+                 MYSQL_TYPE_LONG, &type, GNUNET_YES,
+                 MYSQL_TYPE_LONGLONG, &rvalue, GNUNET_YES,
                  -1);
-
 }
 
 
@@ -1348,19 +1356,120 @@
 {
   struct Plugin *plugin = cls;
   struct ReplCtx rc;
+  unsigned long long rvalue;
+  unsigned long repl;
+  MYSQL_BIND results;
   
   rc.plugin = plugin;
   rc.proc = proc;
   rc.proc_cls = proc_cls;
+  memset (&results, 0, sizeof (results));
+  results.buffer_type = MYSQL_TYPE_LONG;
+  results.buffer = &repl;
+  results.is_unsigned = GNUNET_YES;
+
+  if (1 !=
+      prepared_statement_run_select (plugin,
+                                    plugin->max_repl,
+                                    1,
+                                    &results,
+                                    -1))
+    {
+      proc (proc_cls, 
+           NULL, 0, NULL, 0, 0, 0, 
+           GNUNET_TIME_UNIT_ZERO_ABS, 0);
+      return;      
+    }
+
+  rvalue = (unsigned long long) GNUNET_CRYPTO_random_u64 
(GNUNET_CRYPTO_QUALITY_WEAK,
+                                                         UINT64_MAX);
   execute_select (plugin,
                  plugin->select_replication, 
                  &repl_proc, &rc,
+                 MYSQL_TYPE_LONG, &repl, GNUNET_YES, 
+                 MYSQL_TYPE_LONGLONG, &rvalue, GNUNET_YES, 
+                 MYSQL_TYPE_LONG, &repl, GNUNET_YES, 
+                 MYSQL_TYPE_LONGLONG, &rvalue, GNUNET_YES, 
                  -1);
 
 }
 
 
 /**
+ * Context for 'expi_proc' function.
+ */
+struct ExpiCtx
+{
+  
+  /**
+   * Plugin handle.
+   */
+  struct Plugin *plugin;
+  
+  /**
+   * Function to call for the result (or the NULL).
+   */
+  PluginDatumProcessor proc;
+  
+  /**
+   * Closure for proc.
+   */
+  void *proc_cls;
+};
+
+
+
+/**
+ * Wrapper for the processor for 'mysql_plugin_get_expiration'.
+ * If no expired value was found, we do a second query for
+ * low-priority content.
+ *
+ * @param cls closure
+ * @param key key for the content
+ * @param size number of bytes in data
+ * @param data content stored
+ * @param type type of the content
+ * @param priority priority of the content
+ * @param anonymity anonymity-level for the content
+ * @param expiration expiration time for the content
+ * @param uid unique identifier for the datum;
+ *        maybe 0 if no unique identifier is available
+ *
+ * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue
+ *         (continue on call to "next", of course),
+ *         GNUNET_NO to delete the item and continue (if supported)
+ */
+static int
+expi_proc (void *cls,
+          const GNUNET_HashCode *key,
+          uint32_t size,
+          const void *data,
+          enum GNUNET_BLOCK_Type type,
+          uint32_t priority,
+          uint32_t anonymity,
+          struct GNUNET_TIME_Absolute expiration, 
+          uint64_t uid)
+{
+  struct ExpiCtx *rc = cls;
+  struct Plugin *plugin = rc->plugin;
+  
+  if (NULL == key)
+    {
+      execute_select (plugin,
+                     plugin->select_priority, 
+                     rc->proc, rc->proc_cls,
+                     -1);
+      return GNUNET_SYSERR;
+    }
+  return rc->proc (rc->proc_cls,
+                  key,
+                  size, data, 
+                  type, priority, anonymity, expiration,
+                  uid);
+}
+
+
+/**
  * Get a random item for expiration.
  * Call 'proc' with all values ZERO or NULL if the datastore is empty.
  *
@@ -1374,11 +1483,15 @@
 {
   struct Plugin *plugin = cls;
   long long nt;
-
+  struct ExpiCtx rc;
+  
+  rc.plugin = plugin;
+  rc.proc = proc;
+  rc.proc_cls = proc_cls;
   nt = (long long) GNUNET_TIME_absolute_get().abs_value;
   execute_select (plugin,
                  plugin->select_expiration, 
-                 proc, proc_cls,
+                 expi_proc, &rc,
                  MYSQL_TYPE_LONGLONG, &nt, GNUNET_YES, 
                  -1);
 
@@ -1433,7 +1546,7 @@
              " prio INT(11) UNSIGNED NOT NULL DEFAULT 0,"
              " anonLevel INT(11) UNSIGNED NOT NULL DEFAULT 0,"
              " expire BIGINT UNSIGNED NOT NULL DEFAULT 0,"
-             " rvalue DOUBLE UNSIGNED NOT NULL,"
+             " rvalue BIGINT UNSIGNED NOT NULL,"
              " hash BINARY(64) NOT NULL DEFAULT '',"
              " vhash BINARY(64) NOT NULL DEFAULT '',"
              " value BLOB NOT NULL DEFAULT '',"
@@ -1442,11 +1555,11 @@
              " INDEX idx_hash (hash(64)),"
              " INDEX idx_hash_uid (hash(64),uid),"
              " INDEX idx_hash_vhash (hash(64),vhash(64)),"
-             " INDEX idx_hash_type_uid (hash(64),type,uid),"
+             " INDEX idx_hash_type_rvalue (hash(64),type,rvalue),"
              " INDEX idx_prio (prio),"
              " INDEX idx_repl_rvalue (repl,rvalue),"
-             " INDEX idx_expire_prio (expire,prio),"
-             " INDEX idx_anonLevel_uid (anonLevel,uid)"
+             " INDEX idx_expire (expire),"
+             " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)"
              ") ENGINE=InnoDB") ||
       MRUNS ("SET AUTOCOMMIT = 1") ||
       PINIT (plugin->insert_entry, INSERT_ENTRY) ||
@@ -1466,6 +1579,8 @@
       || PINIT (plugin->dec_repl, DEC_REPL)
       || PINIT (plugin->zero_iter, SELECT_IT_NON_ANONYMOUS) 
       || PINIT (plugin->select_expiration, SELECT_IT_EXPIRATION) 
+      || PINIT (plugin->select_priority, SELECT_IT_PRIORITY) 
+      || PINIT (plugin->max_repl, SELECT_MAX_REPL) 
       || PINIT (plugin->select_replication, SELECT_IT_REPLICATION) )
     {
       iclose (plugin);

Modified: gnunet/src/datastore/plugin_datastore_sqlite.c
===================================================================
--- gnunet/src/datastore/plugin_datastore_sqlite.c      2011-08-03 19:53:43 UTC 
(rev 16349)
+++ gnunet/src/datastore/plugin_datastore_sqlite.c      2011-08-03 20:59:58 UTC 
(rev 16350)
@@ -186,7 +186,7 @@
        sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_comb ON gn090 
(anonLevel ASC,expire ASC,prio,type,hash)",
                      NULL, NULL, NULL)) ||
        (SQLITE_OK != 
-       sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_anon_type_exp ON 
gn090 (anonLevel ASC,type,hash)",
+       sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_anon_type_hash ON 
gn090 (anonLevel ASC,type,hash)",
                      NULL, NULL, NULL)) ||
        (SQLITE_OK != 
        sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 
(expire ASC)",
@@ -357,7 +357,7 @@
                    &plugin->selExpi) != SQLITE_OK) ||
       (sq_prepare (plugin->dbh, 
                   "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ "
-                  "FROM gn090 INDEXED BY idx_anon_type_exp "
+                  "FROM gn090 INDEXED BY idx_anon_type_hash "
                   "WHERE (anonLevel = 0 AND type=?1) "
                   "ORDER BY hash DESC LIMIT 1 OFFSET ?2",
                   &plugin->selZeroAnon) != SQLITE_OK) ||

Modified: gnunet/src/include/gnunet_datastore_plugin.h
===================================================================
--- gnunet/src/include/gnunet_datastore_plugin.h        2011-08-03 19:53:43 UTC 
(rev 16349)
+++ gnunet/src/include/gnunet_datastore_plugin.h        2011-08-03 20:59:58 UTC 
(rev 16350)
@@ -284,6 +284,9 @@
 
   /**
    * Get datum (of the specified type) with anonymity level zero.
+   * This function is allowed to ignore the 'offset' argument
+   * and instead return a random result (with zero anonymity of
+   * the correct type) if implementing an offset is expensive.
    */
   PluginGetType get_zero_anonymity;
 
@@ -297,8 +300,9 @@
   PluginGetRandom get_replication;
 
   /**
-   * Function to get a random expired item or, if none are expired, one
-   * with a low priority.
+   * Function to get a random expired item or, if none are expired,
+   * either the oldest entry or one with a low priority (depending
+   * on what was efficiently implementable).
    */
   PluginGetRandom get_expiration;
 

Modified: gnunet/src/include/gnunet_datastore_service.h
===================================================================
--- gnunet/src/include/gnunet_datastore_service.h       2011-08-03 19:53:43 UTC 
(rev 16349)
+++ gnunet/src/include/gnunet_datastore_service.h       2011-08-03 20:59:58 UTC 
(rev 16350)
@@ -321,6 +321,10 @@
 
 /**
  * Get a single zero-anonymity value from the datastore.
+ * Note that some implementations can ignore the 'offset' and
+ * instead return a random zero-anonymity value.  In that case,
+ * detecting the wrap-around based on a repeating UID is at best
+ * probabilistic.
  *
  * @param h handle to the datastore
  * @param offset offset of the result (modulo num-results); set to