[GNUnet-SVN] r19512 - gnunet/src/fs

[gnunet]

Author: grothoff
Date: 2012-01-29 22:20:17 +0100 (Sun, 29 Jan 2012)
New Revision: 19512

Modified:
   gnunet/src/fs/
   gnunet/src/fs/fs_dirmetascan.c
   gnunet/src/fs/gnunet-helper-fs-publish.c
   gnunet/src/fs/gnunet-publish.c
Log:
-towards external meta scanner process


Property changes on: gnunet/src/fs
___________________________________________________________________
Modified: svn:ignore
   - fs.conf
gnunet-fs
perf_gnunet_service_fs_p2p_trust
perf_gnunet_service_fs_p2p_dht
perf_gnunet_service_fs_p2p_index
perf_gnunet_service_fs_p2p
test_fs_namespace_list_updateable
test_gnunet_service_fs_migration
test_fs_download_indexed
test_fs_unindex_persistence
test_fs_search_persistence
test_fs_publish_persistence
test_fs_download_persistence
test_gnunet_fs_rec.py
test_gnunet_fs_psd.py
test_gnunet_fs_ns.py
test_gnunet_fs_idx.py
test_gnunet_service_fs_p2p
test_fs_test_lib
gnunet-service-fs.gcda
test_fs_unindex.gcno
test_fs_unindex.gcda
test_fs_start_stop.gcno
test_fs_start_stop.gcda
test_fs_search.gcno
test_fs_search.gcda
test_fs_publish.gcno
test_fs_publish.gcda
test_fs_namespace.gcno
test_fs_list_indexed.gcno
test_fs_list_indexed.gcda
test_fs_file_information.gcno
test_fs_file_information.gcda
test_fs_download.gcno
test_fs_download.gcda
test_fs_publish
test_fs_list_indexed
test_fs_file_information
test_fs_unindex
test_fs_namespace
test_fs_start_stop
test_fs_search
test_fs_download
test_fs_uri.gcno
test_fs_uri.gcda
test_fs_getopt.gcno
test_fs_directory.gcno
test_fs_directory.gcda
test_fs_collection.gcno
gnunet-unindex.gcno
gnunet-service-fs.gcno
gnunet-search.gcno
gnunet-publish.gcno
gnunet-pseudonym.gcno
gnunet-download.gcno
gnunet-directory.gcno
fs_uri.gcno
fs_uri.gcda
fs_unindex.gcno
fs_unindex.gcda
fs_tree.gcno
fs_tree.gcda
fs_search.gcno
fs_search.gcda
fs_publish.gcno
fs_publish.gcda
fs_namespace.gcno
fs_namespace.gcda
fs_list_indexed.gcno
fs_list_indexed.gcda
fs_getopt.gcno
fs_getopt.gcda
fs_file_information.gcno
fs_file_information.gcda
fs_download.gcno
fs_download.gcda
fs_directory.gcno
fs_directory.gcda
fs_collection.gcno
fs_collection.gcda
fs.gcno
fs.gcda
test_fs_uri
test_fs_getopt
test_fs_directory
test_fs_collection
gnunet-service-fs
gnunet-pseudonym
gnunet-directory
gnunet-unindex
gnunet-search
gnunet-download
gnunet-publish
Makefile.in
Makefile
.deps

   + gnunet-helper-fs-publish
fs.conf
gnunet-fs
perf_gnunet_service_fs_p2p_trust
perf_gnunet_service_fs_p2p_dht
perf_gnunet_service_fs_p2p_index
perf_gnunet_service_fs_p2p
test_fs_namespace_list_updateable
test_gnunet_service_fs_migration
test_fs_download_indexed
test_fs_unindex_persistence
test_fs_search_persistence
test_fs_publish_persistence
test_fs_download_persistence
test_gnunet_fs_rec.py
test_gnunet_fs_psd.py
test_gnunet_fs_ns.py
test_gnunet_fs_idx.py
test_gnunet_service_fs_p2p
test_fs_test_lib
gnunet-service-fs.gcda
test_fs_unindex.gcno
test_fs_unindex.gcda
test_fs_start_stop.gcno
test_fs_start_stop.gcda
test_fs_search.gcno
test_fs_search.gcda
test_fs_publish.gcno
test_fs_publish.gcda
test_fs_namespace.gcno
test_fs_list_indexed.gcno
test_fs_list_indexed.gcda
test_fs_file_information.gcno
test_fs_file_information.gcda
test_fs_download.gcno
test_fs_download.gcda
test_fs_publish
test_fs_list_indexed
test_fs_file_information
test_fs_unindex
test_fs_namespace
test_fs_start_stop
test_fs_search
test_fs_download
test_fs_uri.gcno
test_fs_uri.gcda
test_fs_getopt.gcno
test_fs_directory.gcno
test_fs_directory.gcda
test_fs_collection.gcno
gnunet-unindex.gcno
gnunet-service-fs.gcno
gnunet-search.gcno
gnunet-publish.gcno
gnunet-pseudonym.gcno
gnunet-download.gcno
gnunet-directory.gcno
fs_uri.gcno
fs_uri.gcda
fs_unindex.gcno
fs_unindex.gcda
fs_tree.gcno
fs_tree.gcda
fs_search.gcno
fs_search.gcda
fs_publish.gcno
fs_publish.gcda
fs_namespace.gcno
fs_namespace.gcda
fs_list_indexed.gcno
fs_list_indexed.gcda
fs_getopt.gcno
fs_getopt.gcda
fs_file_information.gcno
fs_file_information.gcda
fs_download.gcno
fs_download.gcda
fs_directory.gcno
fs_directory.gcda
fs_collection.gcno
fs_collection.gcda
fs.gcno
fs.gcda
test_fs_uri
test_fs_getopt
test_fs_directory
test_fs_collection
gnunet-service-fs
gnunet-pseudonym
gnunet-directory
gnunet-unindex
gnunet-search
gnunet-download
gnunet-publish
Makefile.in
Makefile
.deps


Modified: gnunet/src/fs/fs_dirmetascan.c
===================================================================
--- gnunet/src/fs/fs_dirmetascan.c      2012-01-29 20:21:09 UTC (rev 19511)
+++ gnunet/src/fs/fs_dirmetascan.c      2012-01-29 21:20:17 UTC (rev 19512)
@@ -68,11 +68,20 @@
   /**
    * After the scan is finished, it will contain a pointer to the
    * top-level directory entry in the directory tree built by the
-   * scanner.  Must only be manipulated by the thread for the
-   * duration of the thread's runtime.
+   * scanner. 
    */
   struct GNUNET_FS_ShareTreeItem *toplevel;
 
+  /**
+   * Current position during processing.
+   */
+  struct GNUNET_FS_ShareTreeItem *pos;
+
+  /**
+   * Arguments for helper.
+   */
+  char *args[4];
+
 };
 
 
@@ -121,6 +130,76 @@
 
 
 /**
+ * Move in the directory from the given position to the next file
+ * in DFS traversal.
+ *
+ * @param pos current position
+ * @return next file, NULL for none
+ */
+static struct GNUNET_FS_ShareTreeItem *
+advance (struct GNUNET_FS_ShareTreeItem *pos)
+{
+  int moved;
+  
+  GNUNET_assert (NULL != pos);
+  moved = 0; /* must not terminate, even on file, otherwise "normal" */
+  while ( (pos->is_directory) ||
+         (0 == moved) )
+  {
+    if ( (moved != -1) &&
+        (NULL != pos->children_head) )
+    {
+      pos = pos->children_head;
+      moved = 1; /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->next)
+    {
+      pos = pos->next;
+      moved = 1; /* can terminate if file */
+      continue;
+    }
+    if (NULL != pos->parent)
+    {
+      pos = pos->parent;
+      moved = -1; /* force move to 'next' or 'parent' */
+      continue;
+    }
+    /* no more options, end of traversal */
+    return NULL;
+  }
+  return pos;
+}
+
+
+/**
+ * Add another child node to the tree.
+ *
+ * @param parent parent of the child, NULL for top level
+ * @param filename name of the file or directory
+ * @param is_directory GNUNET_YES for directories
+ * @return new entry that was just created
+ */
+static struct GNUNET_FS_ShareTreeItem *
+expand_tree (struct GNUNET_FS_ShareTreeItem *parent,
+            const char *filename,
+            int is_directory)
+{
+  struct GNUNET_FS_ShareTreeItem *chld;
+
+  chld = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem));
+  chld->parent = parent;
+  chld->filename = GNUNET_strdup (filename);
+  chld->is_directory = is_directory;
+  if (NULL != parent)
+      GNUNET_CONTAINER_DLL_insert (parent->children_head,
+                                  parent->children_tail,
+                                  chld);  
+  return chld;
+}
+
+
+/**
  * Called every time there is data to read from the scanner.
  * Calls the scanner progress handler.
  *
@@ -134,35 +213,145 @@
                     const struct GNUNET_MessageHeader *msg)
 {
   struct GNUNET_FS_DirScanner *ds = cls;
-  ds++;
-#if 0
-  enum GNUNET_FS_DirScannerProgressUpdateReason reason;
-  size_t filename_len;
-  int is_directory;
-  char *filename;
+  const char *filename;
+  size_t left;
 
-  /* Process message. If message is malformed or can't be read, end the 
scanner */
-  /* read successfully, notify client about progress */
+  left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
+  filename = (const char*) &msg[1];
+  switch (ntohs (msg->type))
+  {
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
+    if (filename[left-1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                          filename, GNUNET_NO,
+                          GNUNET_FS_DIRSCANNER_FILE_START);
+    expand_tree (ds->pos,
+                filename, GNUNET_NO);
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
+    if (filename[left-1] != '\0')
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 == strcmp ("..", filename))
+    {
+      if (NULL == ds->pos)
+      {
+       GNUNET_break (0);
+       break;
+      }
+      ds->pos = ds->pos->parent;
+      return;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                          filename, GNUNET_YES,
+                          GNUNET_FS_DIRSCANNER_FILE_START);
+    ds->pos = expand_tree (ds->pos,
+                          filename, GNUNET_YES);
+    if (NULL == ds->toplevel)
+      ds->toplevel = ds->pos;
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
+    break;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
+    if (filename[left-1] != '\0')
+      break;
+    ds->progress_callback (ds->progress_callback_cls, 
+                          filename, GNUNET_SYSERR,
+                          GNUNET_FS_DIRSCANNER_FILE_IGNORED);
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                          NULL, GNUNET_SYSERR,
+                          GNUNET_FS_DIRSCANNER_ALL_COUNTED);
+    ds->pos = ds->toplevel;
+    if (ds->pos->is_directory)
+      ds->pos = advance (ds->pos);
+    return;
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA:
+    {
+      size_t nlen;
+      const char *end;
+      
+      if (NULL == ds->pos)
+      {
+       GNUNET_break (0);
+       break;
+      }
+      end = memchr (filename, 0, left);
+      if (NULL == end)
+      {
+       GNUNET_break (0);
+       break;
+      }
+      end++;
+      nlen = end - filename;
+      left -= nlen;
+      if (0 != strcmp (filename,
+                      ds->pos->filename))
+      {
+       GNUNET_break (0);
+       break;
+      }
+      ds->progress_callback (ds->progress_callback_cls, 
+                            filename, GNUNET_YES,
+                            GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
+      if (0 < left)
+      {
+       ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left);
+       if (NULL == ds->pos->meta)
+       {
+         GNUNET_break (0);
+         break;
+       }
+       /* having full filenames is too dangerous; always make sure we clean 
them up */
+       ds->pos->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name 
(filename));
+       GNUNET_CONTAINER_meta_data_delete (ds->pos->meta, 
+                                          EXTRACTOR_METATYPE_FILENAME,
+                                          NULL, 0);
+       GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>",
+                                          EXTRACTOR_METATYPE_FILENAME,
+                                          EXTRACTOR_METAFORMAT_UTF8, 
"text/plain",
+                                          ds->pos->short_filename, 
+                                          strlen (ds->pos->short_filename) + 
1);
+      }
+      ds->pos = advance (ds->pos);      
+      return;
+    }
+  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
+    if (NULL != ds->pos)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    if (0 != left)
+    {
+      GNUNET_break (0);
+      break;
+    }
+    ds->progress_callback (ds->progress_callback_cls, 
+                          NULL, GNUNET_SYSERR,
+                          GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
+    
+    return;
+  default:
+    GNUNET_break (0);
+    break;
+  }
   ds->progress_callback (ds->progress_callback_cls, 
-                        ds, 
-                        filename,
-                        is_directory, 
-                        reason);
-  GNUNET_free (filename);
+                        NULL, GNUNET_SYSERR,
+                        GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
 
-
-  /* having full filenames is too dangerous; always make sure we clean them up 
*/
-  item->short_filename = GNUNET_strdup (GNUNET_STRINGS_get_short_name 
(filename));
-
-  GNUNET_CONTAINER_meta_data_delete (item->meta, 
-                                    EXTRACTOR_METATYPE_FILENAME,
-                                    NULL, 0);
-  GNUNET_CONTAINER_meta_data_insert (item->meta, "<libgnunetfs>",
-                                     EXTRACTOR_METATYPE_FILENAME,
-                                     EXTRACTOR_METAFORMAT_UTF8, "text/plain",
-                                     item->short_filename, 
-                                    strlen (item->short_filename) + 1);
-#endif
 }
 
 
@@ -185,7 +374,6 @@
   struct stat sbuf;
   char *filename_expanded;
   struct GNUNET_FS_DirScanner *ds;
-  char *args[4];
 
   if (0 != STAT (filename, &sbuf))
     return NULL;
@@ -199,13 +387,16 @@
   ds->progress_callback = cb;
   ds->progress_callback_cls = cb_cls;
   ds->filename_expanded = filename_expanded;
-  ds->ex_arg = GNUNET_strdup ((disable_extractor) ? "-" : ex);
-  args[0] = "gnunet-helper-fs-publish";
-  args[1] = ds->filename_expanded;
-  args[2] = ds->ex_arg;
-  args[3] = NULL;
+  if (disable_extractor)  
+    ds->ex_arg = GNUNET_strdup ("-");
+  else 
+    ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
+  ds->args[0] = "gnunet-helper-fs-publish";
+  ds->args[1] = ds->filename_expanded;
+  ds->args[2] = ds->ex_arg;
+  ds->args[3] = NULL;
   ds->helper = GNUNET_HELPER_start ("gnunet-helper-fs-publish",
-                                   args,
+                                   ds->args,
                                    &process_helper_msgs,
                                    ds);
   if (NULL == ds->helper)

Modified: gnunet/src/fs/gnunet-helper-fs-publish.c
===================================================================
--- gnunet/src/fs/gnunet-helper-fs-publish.c    2012-01-29 20:21:09 UTC (rev 
19511)
+++ gnunet/src/fs/gnunet-helper-fs-publish.c    2012-01-29 21:20:17 UTC (rev 
19512)
@@ -223,9 +223,10 @@
     return GNUNET_SYSERR;
   }
   chld->parent = rc->parent;
-  GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
-                              rc->parent->children_tail,
-                              chld);
+  if (NULL != rc->parent)
+    GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
+                                rc->parent->children_tail,
+                                chld);
   return GNUNET_OK;
 }
 
@@ -282,6 +283,10 @@
       free_tree (item);
       return GNUNET_SYSERR;
     }
+    if (GNUNET_OK !=
+       write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY,
+                      "..", 3))
+      return GNUNET_SYSERR;
   }
   *dst = item;
   return GNUNET_OK;
@@ -379,7 +384,8 @@
   }
   filename_expanded = argv[1];
   ex = argv[2];
-  if (0 != strcmp (ex, "-"))
+  if ( (ex == NULL) ||
+       (0 != strcmp (ex, "-")) )
   {
     plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
     if (NULL != ex)

Modified: gnunet/src/fs/gnunet-publish.c
===================================================================
--- gnunet/src/fs/gnunet-publish.c      2012-01-29 20:21:09 UTC (rev 19511)
+++ gnunet/src/fs/gnunet-publish.c      2012-01-29 21:20:17 UTC (rev 19512)
@@ -425,50 +425,67 @@
 }
 
 
+/**
+ * Function called by the directory scanner as we build the tree
+ * that we will need to publish later.
+ *
+ * @param cls closure
+ * @param filename which file we are making progress on
+ * @param is_directory GNUNET_YES if this is a directory,
+ *                     GNUNET_NO if this is a file
+ *                     GNUNET_SYSERR if it is neither (or unknown)
+ * @param reason kind of progress we are making
+ */
 static void
-directory_scan_cb (void *cls, struct GNUNET_FS_DirScanner *ds,
+directory_scan_cb (void *cls, 
                   const char *filename, 
                   int is_directory,
                   enum GNUNET_FS_DirScannerProgressUpdateReason reason)
 {
   switch (reason)
   {
-    case GNUNET_FS_DIRSCANNER_FILE_START:
+  case GNUNET_FS_DIRSCANNER_FILE_START:
+    if (verbose > 1)
+    {
       if (is_directory)
        FPRINTF (stdout, _("Scanning directory `%s'.\n"), filename);
       else
        FPRINTF (stdout, _("Scanning file `%s'.\n"), filename);      
-      break;
-    case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
+    }
+    break;
+  case GNUNET_FS_DIRSCANNER_FILE_IGNORED:
+    FPRINTF (stderr, 
+            _("There was trouble processing file `%s', skipping it.\n"),
+            filename);
+    break;
+  case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
+    if (verbose)
       FPRINTF (stdout, "%s", _("Preprocessing complete.\n"));      
-      break;
-    case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
+    break;
+  case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
+    if (verbose > 2)
       FPRINTF (stdout, _("Extracting meta data from file `%s' complete.\n"), 
filename);      
-      break;
-    case GNUNET_FS_DIRSCANNER_DOES_NOT_EXIST:
-      FPRINTF (stdout, 
-              _("There was trouble processing file `%s', skipping it.\n"),
-              filename);
-      break;
-    case GNUNET_FS_DIRSCANNER_FINISHED:
-      FPRINTF (stdout, "%s", _("Scanner has finished.\n"));
-      directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
-      ds = NULL;
-      GNUNET_FS_share_tree_trim (directory_scan_result);
-      directory_trim_complete ();
-      break;
-    case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
-      FPRINTF (stdout, "%s", _("Internal error scanning directory.\n"));
-      GNUNET_FS_directory_scan_abort (ds);
-      ds = NULL;
-      if (namespace != NULL)
-       GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
-      GNUNET_FS_stop (ctx);
-      ret = 1;
-      break;
-    default:
-      GNUNET_assert (0);
-      break;
+    break;
+  case GNUNET_FS_DIRSCANNER_FINISHED:
+    if (verbose > 1)
+      FPRINTF (stdout, "%s", _("Meta data extraction has finished.\n"));
+    directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
+    ds = NULL;
+    GNUNET_FS_share_tree_trim (directory_scan_result);
+    directory_trim_complete ();
+    break;
+  case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
+    FPRINTF (stdout, "%s", _("Internal error scanning directory.\n"));
+    GNUNET_FS_directory_scan_abort (ds);
+    ds = NULL;
+    if (namespace != NULL)
+      GNUNET_FS_namespace_delete (namespace, GNUNET_NO);
+    GNUNET_FS_stop (ctx);
+    ret = 1;
+    break;
+  default:
+    GNUNET_assert (0);
+    break;
   }
   fflush (stdout);
 }
@@ -587,6 +604,12 @@
                                       disable_extractor, 
                                       ex, 
                                       &directory_scan_cb, NULL);
+  if (NULL == ds)
+  {
+    FPRINTF (stderr,
+            "%s", _("Failed to start meta directory scanner.  Is 
gnunet-helper-publish-fs installed?\n"));
+    return;
+  }
   kill_task =
       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, 
&do_stop_task,
                                     NULL);