[GNUnet-SVN] [gnunet] branch master updated: Rewrite gnunet-datastore to dump to a file

[gnunet]

This is an automated email from the git hooks/post-receive script.

david-barksdale pushed a commit to branch master
in repository gnunet.

The following commit(s) were added to refs/heads/master by this push:
     new cacd64d86 Rewrite gnunet-datastore to dump to a file
cacd64d86 is described below

commit cacd64d8635201459e59bf2cd8a2ea8fd0699b84
Author: David Barksdale <address@hidden>
AuthorDate: Thu Apr 13 21:22:29 2017 -0500

    Rewrite gnunet-datastore to dump to a file
---
 doc/man/gnunet-datastore.1       |  24 ++-
 src/datastore/gnunet-datastore.c | 452 ++++++++++++++++++++++++++++++---------
 2 files changed, 364 insertions(+), 112 deletions(-)

diff --git a/doc/man/gnunet-datastore.1 b/doc/man/gnunet-datastore.1
index 14372a49b..1605006b3 100644
--- a/doc/man/gnunet-datastore.1
+++ b/doc/man/gnunet-datastore.1
@@ -1,6 +1,6 @@
-.TH gnunet\-datastore "1" "30 May 2013" "GNUnet"
+.TH gnunet\-datastore "1" "13 April 2017" "GNUnet"
 .SH NAME
-gnunet\-datastore \- merge or convert GNUnet datastore databases
+gnunet\-datastore \- dump or insert (restore) GNUnet datastore databases
 
 .SH SYNOPSIS
 .B gnunet\-datastore
@@ -8,26 +8,32 @@ gnunet\-datastore \- merge or convert GNUnet datastore 
databases
 .SH DESCRIPTION
 .PP
 
-gnunet\-datastore can be used to convert or merge GNUnet datastores.  This is 
useful if a datastore is to be migrated between SQL databases, i.e. from sqlite 
to postgres or vice versa.  gnunet\-datastore basically takes two configuration 
files (which must specify different databases) and reads in all of the data 
from the datasource (\-s option) and copies it to the destination (\-c option). 
 Note that replication level information is lost in the process at this time.
+gnunet\-datastore can be used to backup and restore or merge GNUnet 
datastores.  This is useful if a datastore is to be migrated between SQL 
databases, i.e. from sqlite to postgres or vice versa.  gnunet\-datastore will 
dump the entire contents of the database or insert a dump file into the 
database.
 
 .TP
 \fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR
-configuration file to use for the destination database
+configuration file to use
+.TP
+\fB\-d\fR, \fB\-\-dump\fR
+dump all records to a file
+.TP
+\fB\-f \fIFILENAME\fR, \fB\-\-file=FILENAME\fR
+file to dump to or insert from.  Otherwise stdin/stdout are used.
 .TP
 \fB\-h\fR, \fB\-\-help\fR
 print help page
 .TP
+\fB\-i\fR, \fB\-\-insert\fR
+insert from dump file
+.TP
 \fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR
 Change the loglevel.  Possible values for LOGLEVEL are ERROR, WARNING, INFO 
and DEBUG.
 .TP
-\fB\-s \fIFILENAME\fR, \fB\-\-sourcecfg=FILENAME\fR
-configuration file to use for the source database
+\fB\-l \fIFILENAME\fR, \fB\-\-logfile=FILENAME\fR
+configure logging to write logs to FILENAME
 .TP
 \fB\-v\fR, \fB\-\-version\fR
 print the version number
-.TP
-\fB\-V\fR, \fB\-\-verbose\fR
-be verbose
 
 .SH NOTES
 
diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c
index 9e0ee205e..891343e17 100644
--- a/src/datastore/gnunet-datastore.c
+++ b/src/datastore/gnunet-datastore.c
@@ -23,101 +23,136 @@
  * @brief tool to manipulate datastores
  * @author Christian Grothoff
  */
+#include <inttypes.h>
 #include "platform.h"
 #include "gnunet_util_lib.h"
 #include "gnunet_datastore_service.h"
 
+GNUNET_NETWORK_STRUCT_BEGIN
+
+struct DataRecord
+{
+  /**
+   * Number of bytes in the item (NBO).
+   */
+  uint32_t size GNUNET_PACKED;
+
+  /**
+   * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
+   */
+  uint32_t type GNUNET_PACKED;
+
+  /**
+   * Priority of the item (NBO).
+   */
+  uint32_t priority GNUNET_PACKED;
+
+  /**
+   * Desired anonymity level (NBO).
+   */
+  uint32_t anonymity GNUNET_PACKED;
+
+  /**
+   * Desired replication level (NBO).
+   */
+  uint32_t replication GNUNET_PACKED;
+
+  /**
+   * Expiration time (NBO).
+   */
+  struct GNUNET_TIME_AbsoluteNBO expiration;
+
+  /**
+   * Key under which the item can be found.
+   */
+  struct GNUNET_HashCode key;
+
+};
+GNUNET_NETWORK_STRUCT_END
+
 
 /**
- * Name of the second configuration file.
+ * Length of our magic header.
  */
-static char *alternative_cfg;
+static const size_t MAGIC_LEN = 16;
 
 /**
- * Global return value.
+ * Magic header bytes.
  */
-static int ret;
+static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
+
+/**
+ * Dump the database.
+ */
+static int dump;
 
 /**
- * Our offset on 'get'.
+ * Insert into the database.
  */
-static uint64_t offset;
+static int insert;
 
 /**
- * First UID ever returned.
+ * Dump file name.
  */
-static uint64_t first_uid;
+static char *file_name;
 
 /**
- * Configuration for the source database.
+ * Dump file handle.
  */
-static struct GNUNET_CONFIGURATION_Handle *scfg;
+static struct GNUNET_DISK_FileHandle *file_handle;
 
 /**
- * Handle for database source.
+ * Global return value.
  */
-static struct GNUNET_DATASTORE_Handle *db_src;
+static int ret;
 
 /**
- * Handle for database destination.
+ * Handle for datastore.
  */
-static struct GNUNET_DATASTORE_Handle *db_dst;
+static struct GNUNET_DATASTORE_Handle *datastore;
 
 /**
  * Current operation.
  */
 static struct GNUNET_DATASTORE_QueueEntry *qe;
 
+/**
+ * Record count.
+ */
+static uint64_t record_count;
+
 
 static void
 do_shutdown (void *cls)
 {
   if (NULL != qe)
     GNUNET_DATASTORE_cancel (qe);
-  GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO);
-  GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO);
-  GNUNET_CONFIGURATION_destroy (scfg);
+  if (NULL != datastore)
+    GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO);
+  if (NULL != file_handle)
+    GNUNET_DISK_file_close (file_handle);
 }
 
 
 /**
- * Perform next GET operation.
+ * Begin dumping the database.
  */
 static void
-do_get (void);
+start_dump (void);
 
 
 /**
- * Continuation called to notify client about result of the
- * operation.
- *
- * @param cls closure
- * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
- *                GNUNET_NO if content was already there
- *                GNUNET_YES (or other positive value) on success
- * @param min_expiration minimum expiration time required for 0-priority 
content to be stored
- *                by the datacache at this time, zero for unknown, forever if 
we have no
- *                space for 0-priority content
- * @param msg NULL on success, otherwise an error message
+ * Begin inserting into the database.
  */
 static void
-do_finish (void *cls,
-          int32_t success,
-          struct GNUNET_TIME_Absolute min_expiration,
-          const char *msg)
-{
-  qe = NULL;
-  if (GNUNET_SYSERR == success)
-  {
-    fprintf (stderr,
-            _("Failed to store item: %s, aborting\n"),
-            msg);
-    ret = 1;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  do_get ();
-}
+start_insert (void);
+
+
+/**
+ * Perform next GET operation.
+ */
+static void
+do_get (const uint64_t next_uid);
 
 
 /**
@@ -136,7 +171,7 @@ do_finish (void *cls,
  *        maybe 0 if no unique identifier is available
  */
 static void
-do_put (void *cls,
+get_cb (void *cls,
         const struct GNUNET_HashCode *key,
         size_t size,
         const void *data,
@@ -144,33 +179,63 @@ do_put (void *cls,
         uint32_t priority,
         uint32_t anonymity,
         uint32_t replication,
-        struct GNUNET_TIME_Absolute
-        expiration,
+        struct GNUNET_TIME_Absolute expiration,
         uint64_t uid)
 {
   qe = NULL;
-  if ( (0 != offset) &&
-       (uid == first_uid) )
+  if (NULL == key)
   {
+    FPRINTF (stderr,
+             _("Dumped %" PRIu64 " records\n"),
+             record_count);
+    GNUNET_DISK_file_close (file_handle);
+    file_handle = NULL;
+    if (insert)
+      start_insert();
+    else
+    {
+      ret = 0;
+      GNUNET_SCHEDULER_shutdown ();
+    }
+    return;
+  }
+
+  struct DataRecord dr;
+  dr.size = htonl ((uint32_t) size);
+  dr.type = htonl (type);
+  dr.priority = htonl (priority);
+  dr.anonymity = htonl (anonymity);
+  dr.replication = htonl (replication);
+  dr.expiration = GNUNET_TIME_absolute_hton (expiration);
+  dr.key = *key;
+
+  ssize_t len;
+  len = GNUNET_DISK_file_write (file_handle, &dr, sizeof (dr));
+  if (sizeof (dr) != len)
+  {
+    FPRINTF (stderr,
+             _("Short write to file: %zd bytes expecting %zd\n"),
+             len,
+             sizeof (dr));
+    ret = 1;
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  if (0 == offset)
-    first_uid = uid;
-  qe = GNUNET_DATASTORE_put (db_dst,
-                             0,
-                             key,
-                             size,
-                             data,
-                             type,
-                             priority,
-                             anonymity,
-                             replication,
-                             expiration,
-                             0,
-                             1,
-                             &do_finish,
-                             NULL);
+
+  len = GNUNET_DISK_file_write (file_handle, data, size);
+  if (size != len)
+  {
+    FPRINTF (stderr,
+             _("Short write to file: %zd bytes expecting %zd\n"),
+             len,
+             size);
+    ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+
+  record_count++;
+  do_get(uid + 1);
 }
 
 
@@ -178,64 +243,236 @@ do_put (void *cls,
  * Perform next GET operation.
  */
 static void
-do_get ()
+do_get (const uint64_t next_uid)
 {
-  qe = GNUNET_DATASTORE_get_key (db_src,
-                                0, false,
-                                NULL, GNUNET_BLOCK_TYPE_ANY,
-                                0, 1,
-                                &do_put, NULL);
+  GNUNET_assert (NULL == qe);
+  qe = GNUNET_DATASTORE_get_key (datastore,
+                                 next_uid,
+                                 false /* random */,
+                                 NULL /* key */,
+                                 GNUNET_BLOCK_TYPE_ANY,
+                                 0 /* queue_priority */,
+                                 1 /* max_queue_size */,
+                                 &get_cb,
+                                 NULL /* proc_cls */);
+  if (NULL == qe)
+  {
+    FPRINTF (stderr,
+             _("Error queueing datastore GET operation\n"));
+    ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+  }
 }
 
 
+/**
+ * Begin dumping the database.
+ */
+static void
+start_dump ()
+{
+  record_count = 0;
+
+  if (NULL != file_name)
+  {
+    file_handle = GNUNET_DISK_file_open (file_name,
+                                         GNUNET_DISK_OPEN_WRITE |
+                                         GNUNET_DISK_OPEN_TRUNCATE |
+                                         GNUNET_DISK_OPEN_CREATE,
+                                         GNUNET_DISK_PERM_USER_READ |
+                                         GNUNET_DISK_PERM_USER_WRITE);
+    if (NULL == file_handle)
+    {
+      FPRINTF (stderr,
+               _("Unable to open dump file: %s\n"),
+               file_name);
+      ret = 1;
+      GNUNET_SCHEDULER_shutdown ();
+      return;
+    }
+  }
+  else
+  {
+    file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
+  }
+  GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN);
+  do_get(0);
+}
+
 
 /**
- * Main function that will be run by the scheduler.
+ * Continuation called to notify client about result of the
+ * operation.
  *
  * @param cls closure
- * @param args remaining command-line arguments
- * @param cfgfile name of the configuration file used
- * @param cfg configuration -- for destination datastore
+ * @param success GNUNET_SYSERR on failure (including timeout/queue drop)
+ *                GNUNET_NO if content was already there
+ *                GNUNET_YES (or other positive value) on success
+ * @param min_expiration minimum expiration time required for 0-priority 
content to be stored
+ *                by the datacache at this time, zero for unknown, forever if 
we have no
+ *                space for 0-priority content
+ * @param msg NULL on success, otherwise an error message
  */
 static void
-run (void *cls, char *const *args, const char *cfgfile,
-     const struct GNUNET_CONFIGURATION_Handle *cfg)
+put_cb (void *cls,
+        int32_t success,
+        struct GNUNET_TIME_Absolute min_expiration,
+        const char *msg)
 {
-  if (NULL == alternative_cfg)
-    return; /* nothing to be done */
-  if (0 == strcmp (cfgfile, alternative_cfg))
+  qe = NULL;
+  if (GNUNET_SYSERR == success)
   {
-    fprintf (stderr,
-            _("Cannot use the same configuration for source and 
destination\n"));
+    FPRINTF (stderr,
+             _("Failed to store item: %s, aborting\n"),
+             msg);
     ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  scfg = GNUNET_CONFIGURATION_create ();
-  if (GNUNET_OK !=
-      GNUNET_CONFIGURATION_load (scfg,
-                                alternative_cfg))
+
+  struct DataRecord dr;
+  ssize_t len;
+
+  len = GNUNET_DISK_file_read (file_handle, &dr, sizeof (dr));
+  if (0 == len)
+  {
+    FPRINTF (stderr,
+             _("Inserted %" PRIu64 " records\n"),
+             record_count);
+    ret = 0;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  else if (sizeof (dr) != len)
   {
-    GNUNET_CONFIGURATION_destroy (scfg);
+    FPRINTF (stderr,
+             _("Short read from file: %zd bytes expecting %zd\n"),
+             len,
+             sizeof (dr));
     ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  db_src = GNUNET_DATASTORE_connect (scfg);
-  if (NULL == db_src)
+
+  const size_t size = ntohl (dr.size);
+  uint8_t data[size];
+  len = GNUNET_DISK_file_read (file_handle, data, size);
+  if (size != len)
   {
-    GNUNET_CONFIGURATION_destroy (scfg);
+    FPRINTF (stderr,
+             _("Short read from file: %zd bytes expecting %zd\n"),
+             len,
+             size);
     ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  db_dst = GNUNET_DATASTORE_connect (cfg);
-  if (NULL == db_dst)
+
+  record_count++;
+  qe = GNUNET_DATASTORE_put (datastore,
+                             0,
+                             &dr.key,
+                             size,
+                             data,
+                             ntohl (dr.type),
+                             ntohl (dr.priority),
+                             ntohl (dr.anonymity),
+                             ntohl (dr.replication),
+                             GNUNET_TIME_absolute_ntoh (dr.expiration),
+                             0,
+                             1,
+                             &put_cb,
+                             NULL);
+  if (NULL == qe)
   {
-    GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO);
-    GNUNET_CONFIGURATION_destroy (scfg);
+    FPRINTF (stderr,
+             _("Error queueing datastore PUT operation\n"));
     ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+  }
+}
+
+
+/**
+ * Begin inserting into the database.
+ */
+static void
+start_insert ()
+{
+  record_count = 0;
+
+  if (NULL != file_name)
+  {
+    file_handle = GNUNET_DISK_file_open (file_name,
+                                         GNUNET_DISK_OPEN_READ,
+                                         GNUNET_DISK_PERM_NONE);
+    if (NULL == file_handle)
+    {
+      FPRINTF (stderr,
+               _("Unable to open dump file: %s\n"),
+               file_name);
+      ret = 1;
+      GNUNET_SCHEDULER_shutdown ();
+      return;
+    }
+  }
+  else
+  {
+    file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
+  }
+
+  uint8_t buf[MAGIC_LEN];
+  ssize_t len;
+
+  len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
+  if (len != MAGIC_LEN ||
+      0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN))
+  {
+    FPRINTF (stderr,
+             _("Input file is not of a supported format\n"));
     return;
   }
+  put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL);
+}
+
+
+/**
+ * Main function that will be run by the scheduler.
+ *
+ * @param cls closure
+ * @param args remaining command-line arguments
+ * @param cfgfile name of the configuration file used
+ * @param cfg configuration
+ */
+static void
+run (void *cls,
+     char *const *args,
+     const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
   GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
-  do_get ();
+  datastore = GNUNET_DATASTORE_connect (cfg);
+  if (NULL == datastore)
+  {
+    FPRINTF (stderr,
+             _("Failed connecting to the datastore.\n"));
+    ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (dump)
+    start_dump();
+  else if (insert)
+    start_insert();
+  else
+  {
+    FPRINTF (stderr,
+             _("Please choose at least one operation: %s, %s\n"),
+             "dump",
+             "insert");
+    ret = 1;
+    GNUNET_SCHEDULER_shutdown ();
+  }
 }
 
 
@@ -247,14 +484,23 @@ run (void *cls, char *const *args, const char *cfgfile,
  * @return 0 ok, 1 on error
  */
 int
-main (int argc, char *const *argv)
+main (int argc,
+      char *const *argv)
 {
   struct GNUNET_GETOPT_CommandLineOption options[] = {
-    GNUNET_GETOPT_option_filename ('s',
-                                   "sourcecfg",
+    GNUNET_GETOPT_option_flag ('d',
+                               "dump",
+                               gettext_noop ("Dump all records from the 
datastore"),
+                               &dump),
+    GNUNET_GETOPT_option_flag ('i',
+                               "insert",
+                               gettext_noop ("Insert records into the 
datastore"),
+                               &insert),
+    GNUNET_GETOPT_option_filename ('f',
+                                   "file",
                                    "FILENAME",
-                                   gettext_noop ("specifies the configuration 
to use to access an alternative datastore; will merge that datastore into our 
current datastore"),
-                                       &alternative_cfg),
+                                   gettext_noop ("File to dump or insert"),
+                                   &file_name),
     GNUNET_GETOPT_OPTION_END
   };
   if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))

-- 
To stop receiving notification emails like this one, please contact
address@hidden