[GNUnet-SVN] r8596 - gnunet/src/datastore

[gnunet]

Author: grothoff
Date: 2009-06-16 20:50:43 -0600 (Tue, 16 Jun 2009)
New Revision: 8596

Modified:
   gnunet/src/datastore/gnunet-service-datastore.c
   gnunet/src/datastore/perf_datastore_api.c
   gnunet/src/datastore/perf_datastore_api_iterators.c
   gnunet/src/datastore/test_datastore_api_data.conf
Log:
making tests compile

Modified: gnunet/src/datastore/gnunet-service-datastore.c
===================================================================
--- gnunet/src/datastore/gnunet-service-datastore.c     2009-06-17 02:23:24 UTC 
(rev 8595)
+++ gnunet/src/datastore/gnunet-service-datastore.c     2009-06-17 02:50:43 UTC 
(rev 8596)
@@ -789,8 +789,7 @@
  * Function called when the service shuts
  * down.  Unloads our datastore plugin.
  *
- * @param cls closure
- * @param cfg configuration to use
+ * @param plug plugin to unload
  */
 static void
 unload_plugin (struct DatastorePlugin *plug)

Modified: gnunet/src/datastore/perf_datastore_api.c
===================================================================
--- gnunet/src/datastore/perf_datastore_api.c   2009-06-17 02:23:24 UTC (rev 
8595)
+++ gnunet/src/datastore/perf_datastore_api.c   2009-06-17 02:50:43 UTC (rev 
8596)
@@ -31,14 +31,6 @@
  * strategy alternates between "lowest priority" and "earliest expiration".
  * Priorities and expiration dates are set using a pseudo-random value
  * within a realistic range.
- * <p>
- *
- * Note that the disk overhead calculations are not very sane for
- * MySQL: we take the entire /var/lib/mysql directory (best we can
- * do for ISAM), which may contain other data and which never
- * shrinks.  The scanning of the entire mysql directory during
- * each report is also likely to be the cause of a minor
- * slowdown compared to sqlite.<p>
  */
 
 #include "platform.h"
@@ -103,13 +95,6 @@
  */
 #define ITERATIONS 100
 
-/**
- * Name of the database on disk.
- * You may have to adjust this path and the access
- * permission to the respective directory in order
- * to obtain all of the performance information.
- */
-#define DB_NAME "/tmp/gnunet-datastore-test/data/fs/"
 
 static unsigned long long stored_bytes;
 
@@ -186,12 +171,12 @@
 {
   int j;
   unsigned long long size;
-  int have_file;
-  struct stat sbuf;
   int i;
 
   datastore = GNUNET_DATASTORE_connect (cfg, sched);
-  have_file = 0 == stat (DB_NAME, &sbuf);
+  /* FIXME: change loop to use CPS; current
+     datastore API will likely react negative to
+     us ignoring the callbacks... */
   for (i = 0; i < ITERATIONS; i++)
     {
 #if REPORT_ID
@@ -205,22 +190,15 @@
       if ((i % 2) == 0)
         GNUNET_DATASTORE_get_random (datastore, &iterate_delete, NULL);
       size = 0;
-      if (have_file)
-        GNUNET_disk_file_size (NULL, DB_NAME, &size, GNUNET_NO);
       printf (
 #if REPORT_ID
                "\n"
 #endif
-               "Useful %llu, disk %llu (%.2f%%) / %lluk ops / %llu ops/s\n", 
+               "Stored %llu kB / %lluk ops / %llu ops/s\n", 
               stored_bytes / 1024,     /* used size in k */
-               size / 1024,     /* disk size in kb */
-               (100.0 * size / stored_bytes) - 100,     /* overhead */
                (stored_ops * 2 - stored_entries) / 1024,        /* total 
operations (in k) */
-               1000 * (stored_ops * 2 - stored_entries) / (1 + GNUNET_get_time 
() - start_time));       /* operations per second */
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
+               1000 * (stored_ops * 2 - stored_entries) / (1 + 
GNUNET_TIME_absolute_get_duration(start_time).value));       /* operations per 
second */
     }
-
   GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES);
 }
 
@@ -246,7 +224,7 @@
 #if VERBOSE
                                  "-L", "DEBUG",
 #endif
-                                 "-c", "perf_datastore_api_data.conf", NULL);
+                                 "-c", "test_datastore_api_data.conf", NULL);
   sleep (1);
   GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
                       argv, "perf-datastore-api", "nohelp",
@@ -281,7 +259,4 @@
 }
 
 
-
-
-
 /* end of perf_datastore_api.c */

Modified: gnunet/src/datastore/perf_datastore_api_iterators.c
===================================================================
--- gnunet/src/datastore/perf_datastore_api_iterators.c 2009-06-17 02:23:24 UTC 
(rev 8595)
+++ gnunet/src/datastore/perf_datastore_api_iterators.c 2009-06-17 02:50:43 UTC 
(rev 8596)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2004, 2005, 2006, 2007 Christian Grothoff (and other contributing 
authors)
+     (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other 
contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -18,16 +18,15 @@
      Boston, MA 02111-1307, USA.
 */
 /*
- * @file applications/sqstore_sqlite/sqlitetest3.c
- * @brief Profile sqstore iterators.
+ * @file perf_datastore_api_iterators.c
+ * @brief Profile database plugin directly, focusing on iterators.
  * @author Christian Grothoff
  */
 
 #include "platform.h"
-#include "gnunet_util.h"
+#include "gnunet_util_lib.h"
 #include "gnunet_protocols.h"
-#include "gnunet_sqstore_service.h"
-#include "core.h"
+#include "plugin_datastore.h"
 
 /**
  * Target datastore size (in bytes).  Realistic sizes are
@@ -50,159 +49,267 @@
 
 static unsigned long long stored_ops;
 
-static GNUNET_CronTime start_time;
+static struct GNUNET_CONFIGURATION_Handle *cfg;
 
+static struct GNUNET_SCHEDULER_Handle *sched;
+            
 static int
-putValue (GNUNET_SQstore_ServiceAPI * api, int i, int k)
+putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k)
 {
-  GNUNET_DatastoreValue *value;
+  char value[65536];
   size_t size;
   static GNUNET_HashCode key;
   static int ic;
+  char *msg;
 
   /* most content is 32k */
-  size = sizeof (GNUNET_DatastoreValue) + 32 * 1024;
+  size = 32 * 1024;
 
-  if (GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, 16) == 0)  /* but some of 
it is less! */
-    size =
-      sizeof (GNUNET_DatastoreValue) +
-      GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, 32 * 1024);
+  if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0)  /* but 
some of it is less! */
+    size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
   size = size - (size & 7);     /* always multiple of 8 */
 
   /* generate random key */
-  key.bits[0] = (unsigned int) GNUNET_get_time ();
-  GNUNET_hash (&key, sizeof (GNUNET_HashCode), &key);
-  value = GNUNET_malloc (size);
-  value->size = htonl (size);
-  value->type = htonl (i);
-  value->priority =
-    htonl (GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, 100));
-  value->anonymity_level = htonl (i);
-  value->expiration_time =
-    GNUNET_htonll (GNUNET_get_time () + 60 * GNUNET_CRON_HOURS +
-                   GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, 1000));
-  memset (&value[1], i, size - sizeof (GNUNET_DatastoreValue));
+  key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().value;
+  GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key);
+  memset (value, i, size);
   if (i > 255)
-    memset (&value[1], i - 255, (size - sizeof (GNUNET_DatastoreValue)) / 2);
-  ((char *) &value[1])[0] = k;
-  if (GNUNET_OK != api->put (&key, value))
+    memset (value, i - 255, size / 2);
+  value[0] = k;
+  msg = NULL;
+  if (GNUNET_OK != api->put (api->cls,
+                            &key, 
+                            size,
+                            value,
+                            i,
+                            GNUNET_CRYPTO_random_u32 
(GNUNET_CRYPTO_QUALITY_WEAK, 100),
+                            i,
+                            GNUNET_TIME_relative_to_absolute 
+                            (GNUNET_TIME_relative_multiply 
(GNUNET_TIME_UNIT_MILLISECONDS,
+                                                            60 * 60 * 60 * 
1000 +
+                                                            
GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
+                            &msg));
     {
-      GNUNET_free (value);
-      fprintf (stderr, "E");
+      fprintf (stderr, "E: `%s'", msg);
+      GNUNET_free_non_null (msg);
       return GNUNET_SYSERR;
     }
   ic++;
-  stored_bytes += ntohl (value->size);
+  stored_bytes += size;
   stored_ops++;
   stored_entries++;
   GNUNET_free (value);
   return GNUNET_OK;
 }
 
+
 static int
-iterateDummy (const GNUNET_HashCode * key, const GNUNET_DatastoreValue * val,
-              void *cls, unsigned long long uid)
+iterateDummy (void *cls,
+             void *next_cls,
+             const GNUNET_HashCode * key,
+             uint32_t size,
+             const void *data,
+             uint32_t type,
+             uint32_t priority,
+             uint32_t anonymity,
+             struct GNUNET_TIME_Absolute
+             expiration, 
+             uint64_t uid)
 {
-  if (GNUNET_shutdown_test () == GNUNET_YES)
-    return GNUNET_SYSERR;
   return GNUNET_OK;
 }
 
 static int
-test (GNUNET_SQstore_ServiceAPI * api)
-{
+test (struct GNUNET_DATASTORE_PluginFunctions * api)
+{  
   int i;
   int j;
-  int ret;
-  GNUNET_CronTime start;
-  GNUNET_CronTime end;
+  struct GNUNET_TIME_Absolute start;
+  struct GNUNET_TIME_Absolute end;
 
+  /* FIXME: CPS the loop! */
   for (i = 0; i < ITERATIONS; i++)
     {
       /* insert data equivalent to 1/10th of MAX_SIZE */
-      start = GNUNET_get_time ();
+      start = GNUNET_TIME_absolute_get ();
       for (j = 0; j < PUT_10; j++)
         {
           if (GNUNET_OK != putValue (api, j, i))
             break;
-          if (GNUNET_shutdown_test () == GNUNET_YES)
-            break;
         }
-      end = GNUNET_get_time ();
-      printf ("%3u insertion              took %20llums\n", i, end - start);
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
-      start = GNUNET_get_time ();
-      ret = api->iterateLowPriority (0, &iterateDummy, api);
-      end = GNUNET_get_time ();
-      printf ("%3u low priority iteration took %20llums (%d)\n", i,
-              end - start, ret);
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
-      start = GNUNET_get_time ();
-      ret = api->iterateExpirationTime (0, &iterateDummy, api);
-      end = GNUNET_get_time ();
-      printf ("%3u expiration t iteration took %20llums (%d)\n", i,
-              end - start, ret);
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
-      start = GNUNET_get_time ();
-      ret = api->iterateNonAnonymous (0, &iterateDummy, api);
-      end = GNUNET_get_time ();
-      printf ("%3u non anonymou iteration took %20llums (%d)\n", i,
-              end - start, ret);
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
-      start = GNUNET_get_time ();
-      ret = api->iterateMigrationOrder (&iterateDummy, api);
-      end = GNUNET_get_time ();
-      printf ("%3u migration or iteration took %20llums (%d)\n", i,
-              end - start, ret);
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
-      start = GNUNET_get_time ();
-      ret = api->iterateAllNow (&iterateDummy, api);
-      end = GNUNET_get_time ();
-      printf ("%3u all now      iteration took %20llums (%d)\n", i,
-              end - start, ret);
-      if (GNUNET_shutdown_test () == GNUNET_YES)
-        break;
+      end = GNUNET_TIME_absolute_get ();
+      printf ("%3u insertion              took %20llums\n", i, end.value - 
start.value);
+      start = end;
+      api->iter_low_priority (api->cls, 0, &iterateDummy, api);
+      end = GNUNET_TIME_absolute_get ();
+      printf ("%3u low priority iteration took %20llums\n", i,
+              end.value - start.value);
+      start = end;
+      api->iter_ascending_expiration (api->cls, 0, &iterateDummy, api);
+      end = GNUNET_TIME_absolute_get ();
+      printf ("%3u expiration t iteration took %20llums\n", i,
+              end.value - start.value);
+      start = end;
+      api->iter_zero_anonymity (api->cls, 0, &iterateDummy, api);
+      end = GNUNET_TIME_absolute_get ();
+      printf ("%3u non anonymou iteration took %20llums\n", i,
+              end.value - start.value);
+      start = end;
+      api->iter_migration_order (api->cls, 0, &iterateDummy, api);
+      end = GNUNET_TIME_absolute_get ();
+      printf ("%3u migration or iteration took %20llums\n", i,
+              end.value - start.value);
+      start = end;
+      api->iter_all_now (api->cls, 0, &iterateDummy, api);
+      end = GNUNET_TIME_absolute_get ();
+      printf ("%3u all now      iteration took %20llums\n", i,
+              end.value - start.value);
     }
-  api->drop ();
+  api->drop (api->cls);
   return GNUNET_OK;
 }
 
-int
-main (int argc, char *argv[])
+
+/**
+ * Load the datastore plugin.
+ */
+static struct GNUNET_DATASTORE_PluginFunctions *
+load_plugin ()
 {
-  GNUNET_SQstore_ServiceAPI *api;
-  int ok;
-  struct GNUNET_GC_Configuration *cfg;
-  struct GNUNET_CronManager *cron;
+  static struct GNUNET_DATASTORE_PluginEnvironment env;
+  struct GNUNET_DATASTORE_PluginFunctions * ret; 
+  char *name;
+  char *libname;
 
-  cfg = GNUNET_GC_create ();
-  if (-1 == GNUNET_GC_parse_configuration (cfg, "check.conf"))
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             "DATASTORE", "DATABASE", &name))
     {
-      GNUNET_GC_free (cfg);
-      return -1;
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("No `%s' specified for `%s' in configuration!\n"),
+                 "DATABASE",
+                 "DATASTORE");
+      return NULL;
     }
-  cron = GNUNET_cron_create (NULL);
-  GNUNET_CORE_init (NULL, cfg, cron, NULL);
-  api = GNUNET_CORE_request_service ("sqstore");
-  if (api != NULL)
+  env.cfg = cfg;
+  env.sched = sched;  
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Loading `%s' datastore plugin\n"), name);
+  GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
+  GNUNET_assert (NULL != (ret = GNUNET_PLUGIN_load (libname, &env)));
+  GNUNET_free (libname);
+  GNUNET_free (name);
+  return ret;
+}
+
+
+/**
+ * Function called when the service shuts
+ * down.  Unloads our datastore plugin.
+ *
+ * @param api api to unload
+ */
+static void
+unload_plugin (struct GNUNET_DATASTORE_PluginFunctions * api)
+{
+  char *name;
+  char *libname;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             "DATASTORE", "DATABASE", &name))
     {
-      start_time = GNUNET_get_time ();
-      ok = test (api);
-      GNUNET_CORE_release_service (api);
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("No `%s' specified for `%s' in configuration!\n"),
+                 "DATABASE",
+                 "DATASTORE");
+      return;
     }
-  else
-    ok = GNUNET_SYSERR;
-  GNUNET_CORE_done ();
-  GNUNET_cron_destroy (cron);
-  GNUNET_GC_free (cfg);
-  if (ok == GNUNET_SYSERR)
-    return 1;
-  return 0;
+  GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
+  GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
+  GNUNET_free (libname);
+  GNUNET_free (name);
 }
 
-/* end of sqlitetest3.c */
+
+
+/**
+ * Last task run during shutdown.  Disconnects us from
+ * the transport and core.
+ */
+static void
+cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_DATASTORE_PluginFunctions *api = cls;
+
+  unload_plugin (api);
+}
+
+
+
+static void
+run (void *cls,
+     struct GNUNET_SCHEDULER_Handle *s,
+     char *const *args,
+     const char *cfgfile, struct GNUNET_CONFIGURATION_Handle *c)
+{
+  struct GNUNET_DATASTORE_PluginFunctions *api;
+
+  cfg = c;
+  sched = s;
+  api = load_plugin ();
+  test(api);
+  GNUNET_SCHEDULER_add_delayed (sched,
+                                GNUNET_YES,
+                                GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
+                                GNUNET_TIME_UNIT_FOREVER_REL,
+                                &cleaning_task, api);
+}
+
+
+static int
+check ()
+{
+  int ok = 1 + 2 + 4 + 8;
+  char *const argv[] = { "perf-datastore-api-iterators",
+    "-c",
+    "test_datastore_api_data.conf",
+#if VERBOSE
+    "-L", "DEBUG",
+#endif
+    NULL
+  };
+  struct GNUNET_GETOPT_CommandLineOption options[] = {
+    GNUNET_GETOPT_OPTION_END
+  };
+  GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
+                      argv, "perf-datastore-api-iterators", "nohelp",
+                      options, &run, &ok);
+  if (ok != 0)
+    fprintf (stderr, "Missed some testcases: %u\n", ok);
+  return ok;
+}
+
+
+int
+main (int argc, char *argv[])
+{
+  int ret;
+
+  GNUNET_log_setup ("perf-datastore-api-iterators",
+#if VERBOSE
+                    "DEBUG",
+#else
+                    "WARNING",
+#endif
+                    NULL);
+  ret = check ();
+
+  return ret;
+}
+
+
+/* end of perf_datastore_api_iterators.c */
+
+

Modified: gnunet/src/datastore/test_datastore_api_data.conf
===================================================================
--- gnunet/src/datastore/test_datastore_api_data.conf   2009-06-17 02:23:24 UTC 
(rev 8595)
+++ gnunet/src/datastore/test_datastore_api_data.conf   2009-06-17 02:50:43 UTC 
(rev 8596)
@@ -12,6 +12,7 @@
 ALLOW_SHUTDOWN = YES
 QUOTA = 1000000
 BLOOMFILTER = $SERVICEHOME/fs/bloomfilter
+DATABASE = sqlite
 # USERNAME = 
 # MAXBUF =
 # TIMEOUT =