[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[gnunet] branch master updated: namestore: allow for batch inserts from
|
From: |
gnunet |
|
Subject: |
[gnunet] branch master updated: namestore: allow for batch inserts from recordline stdin mode in CLI |
|
Date: |
Tue, 02 Jul 2024 15:10:13 +0200 |
This is an automated email from the git hooks/post-receive script.
martin-schanzenbach pushed a commit to branch master
in repository gnunet.
The following commit(s) were added to refs/heads/master by this push:
new 3d831433f namestore: allow for batch inserts from recordline stdin
mode in CLI
3d831433f is described below
commit 3d831433fce5d5e81059d099afa748a81ed06b34
Author: Martin Schanzenbach <schanzen@gnunet.org>
AuthorDate: Tue Jul 2 15:10:06 2024 +0200
namestore: allow for batch inserts from recordline stdin mode in CLI
---
src/cli/namestore/gnunet-namestore.c | 193 +++++++++++++++++++++++---
src/cli/namestore/test_namestore_put_stdin.sh | 31 ++++-
2 files changed, 195 insertions(+), 29 deletions(-)
diff --git a/src/cli/namestore/gnunet-namestore.c
b/src/cli/namestore/gnunet-namestore.c
index 6c0890a43..8ca4e028a 100644
--- a/src/cli/namestore/gnunet-namestore.c
+++ b/src/cli/namestore/gnunet-namestore.c
@@ -25,6 +25,7 @@
* TODO:
* - test
*/
+#include "gnunet_common.h"
#include "platform.h"
#include <gnunet_util_lib.h>
#include <gnunet_identity_service.h>
@@ -344,6 +345,19 @@ static struct RecordSetEntry *recordset;
*/
static struct GNUNET_SCHEDULER_Task *purge_task;
+/* FIXME: Make array and grow as needed */
+#define INITIAL_RI_BUFFER_SIZE 5000
+
+static unsigned int ri_count = 0;
+
+static struct GNUNET_NAMESTORE_RecordInfo *record_info;
+
+/** Maximum capacity of record_info array **/
+static unsigned int record_info_capacity = 0;
+
+/* How many records to put simulatneously */
+static unsigned int max_batch_size = 1000;
+
/**
* Parse expiration time.
*
@@ -400,7 +414,6 @@ parse_expiration (const char *expirationstring,
static int
parse_recordline (const char *line)
{
- struct RecordSetEntry **head = &recordset;
struct RecordSetEntry *r;
struct GNUNET_GNSRECORD_Data record;
char *cp;
@@ -475,30 +488,36 @@ parse_recordline (const char *line)
GNUNET_free (cp);
r = GNUNET_malloc (sizeof(struct RecordSetEntry) + record.data_size);
- r->next = *head;
+ r->next = recordset;
record.data = &r[1];
memcpy (&r[1], raw_data, record.data_size);
GNUNET_free (raw_data);
r->record = record;
- *head = r;
+ recordset = r;
return GNUNET_OK;
}
static void
-reset_handles (void)
+clear_recordset ()
{
- struct MarkedRecord *mrec;
- struct MarkedRecord *mrec_tmp;
struct RecordSetEntry *rs_entry;
- rs_entry = recordset;
while (NULL != (rs_entry = recordset))
{
recordset = recordset->next;
GNUNET_free (rs_entry);
}
recordset = NULL;
+}
+
+
+static void
+reset_handles (void)
+{
+ struct MarkedRecord *mrec;
+ struct MarkedRecord *mrec_tmp;
+ clear_recordset ();
if (NULL != ego_name)
{
GNUNET_free (ego_name);
@@ -608,6 +627,9 @@ do_shutdown (void *cls)
(void) cls;
reset_handles ();
+ if (NULL != record_info)
+ GNUNET_free (record_info);
+ record_info = NULL;
if (NULL != ns_qe)
{
GNUNET_NAMESTORE_cancel (ns_qe);
@@ -636,10 +658,25 @@ do_shutdown (void *cls)
static void
process_command_stdin ();
+static unsigned int ri_sent = 0;
+
+/**
+ * We have obtained the zone's private key, so now process
+ * the main commands using it.
+ *
+ * @param cfg configuration to use
+ */
+static void
+batch_insert_recordinfo (const struct GNUNET_CONFIGURATION_Handle *cfg);
static void
finish_command (void)
{
+ if (ri_sent < ri_count)
+ {
+ batch_insert_recordinfo (cfg);
+ return;
+ }
reset_handles ();
if (read_from_stdin)
{
@@ -663,6 +700,8 @@ add_continuation (void *cls, enum GNUNET_ErrorCode ec)
GNUNET_ErrorCode_get_hint (ec));
if (GNUNET_EC_NAMESTORE_RECORD_EXISTS != ec)
ret = 1;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
}
ret = 0;
finish_command ();
@@ -1260,7 +1299,8 @@ del_monitor (void *cls,
{
fprintf (stderr,
_ (
- "There are no records under label `%s' that could be
deleted.\n"),
+ "There are no records under label `%s' that could be
deleted.\n")
+ ,
label);
ret = 1;
finish_command ();
@@ -1336,6 +1376,41 @@ replace_cont (void *cls, enum GNUNET_ErrorCode ec)
}
+/**
+ * We have obtained the zone's private key, so now process
+ * the main commands using it.
+ *
+ * @param cfg configuration to use
+ */
+static void
+batch_insert_recordinfo (const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+ unsigned int sent_here;
+
+ GNUNET_assert (0 != ri_count);
+ set_qe = GNUNET_NAMESTORE_records_store (ns,
+ &zone_pkey,
+ ri_count - ri_sent,
+ record_info + ri_sent,
+ &sent_here,
+ &replace_cont,
+ NULL);
+ ri_sent += sent_here;
+ fprintf (stderr, "Sent %d/%d record infos\n", ri_sent, ri_count);
+ if (ri_sent == ri_count)
+ {
+ for (int i = 0; i < ri_count; i++)
+ {
+ GNUNET_free (record_info[i].a_rd);
+ record_info[i].a_rd = NULL;
+ }
+ ri_count = 0;
+ ri_sent = 0;
+ }
+ return;
+}
+
+
/**
* We have obtained the zone's private key, so now process
* the main commands using it.
@@ -1619,11 +1694,14 @@ run_with_zone_pkey (const struct
GNUNET_CONFIGURATION_Handle *cfg)
list_it = GNUNET_NAMESTORE_zone_iteration_start2 (ns,
(NULL == ego_name) ?
NULL : &zone_pkey,
-
&zone_iteration_error_cb,
+
&zone_iteration_error_cb
+ ,
NULL,
-
&display_record_iterator,
+
&display_record_iterator
+ ,
NULL,
-
&zone_iteration_finished,
+
&zone_iteration_finished
+ ,
NULL,
filter_flags);
}
@@ -1791,17 +1869,18 @@ process_command_stdin ()
static struct GNUNET_CRYPTO_PrivateKey next_zone_key;
static char next_name[GNUNET_DNSPARSER_MAX_NAME_LENGTH];
static int finished = GNUNET_NO;
- static int have_next_zonekey = GNUNET_NO;
+ static int have_next_recordline = GNUNET_NO;
int zonekey_set = GNUNET_NO;
char *tmp;
+ char *current_name = NULL;
- if (GNUNET_YES == have_next_zonekey)
+ if (GNUNET_YES == have_next_recordline)
{
zone_pkey = next_zone_key;
- if (NULL != name)
- GNUNET_free (name);
- name = GNUNET_strdup (next_name);
+ if (NULL != current_name)
+ GNUNET_free (current_name);
+ current_name = GNUNET_strdup (next_name);
zonekey_set = GNUNET_YES;
}
while (NULL != fgets (buf, sizeof (buf), stdin))
@@ -1815,6 +1894,7 @@ process_command_stdin ()
*/
if (buf[strlen (buf) - 1] == ':')
{
+ fprintf (stderr, "Switching to %s\n", buf);
memset (next_name, 0, sizeof (next_name));
strncpy (next_name, buf, strlen (buf) - 1);
tmp = strchr (next_name, '.');
@@ -1833,17 +1913,55 @@ process_command_stdin ()
return;
}
*tmp = '\0';
- have_next_zonekey = GNUNET_YES;
+ have_next_recordline = GNUNET_YES;
/* Run a command for the previous record set */
if (NULL != recordset)
{
- run_with_zone_pkey (cfg);
- return;
+ fprintf (stderr, "Recordset not empty\n");
+ if (ri_count >= record_info_capacity)
+ {
+ GNUNET_array_grow (record_info, record_info_capacity,
record_info_capacity + max_batch_size);
+ fprintf (stderr, "Recordinfo array grown to %u bytes!\n",
record_info_capacity);
+ }
+ record_info[ri_count].a_label = GNUNET_strdup (current_name);
+ int rd_count = 0;
+ for (struct RecordSetEntry *e = recordset; NULL != e; e = e->next)
+ rd_count++;
+ record_info[ri_count].a_rd = GNUNET_new_array (rd_count,
+ struct
+ GNUNET_GNSRECORD_Data);
+ rd_count = 0;
+ for (struct RecordSetEntry *e = recordset; NULL != e; e = e->next)
+ {
+ record_info[ri_count].a_rd[rd_count] = e->record;
+ record_info[ri_count].a_rd[rd_count].data = GNUNET_malloc (e->
+ record.
+
data_size);
+ record_info[ri_count].a_rd[rd_count].data_size = e->record.
+ data_size;
+ memcpy ((void*) record_info[ri_count].a_rd[rd_count].data,
+ e->record.data, e->record.data_size);
+ rd_count++;
+ }
+ record_info[ri_count].a_rd_count = rd_count;
+ ri_count++;
+ fprintf (stderr, "Added %d records to record info\n", rd_count);
+ clear_recordset ();
+ /* If the zone has changed, insert */
+ /* If we have reached batch size, insert */
+ if (0 != GNUNET_memcmp (&next_zone_key, &zone_pkey) ||
+ (ri_count >= max_batch_size))
+ {
+ fprintf (stderr, "Batch inserting %d RI\n", ri_count);
+ batch_insert_recordinfo (cfg);
+ return;
+ }
}
zone_pkey = next_zone_key;
- if (NULL != name)
- GNUNET_free (name);
- name = GNUNET_strdup (next_name);
+ fprintf (stderr, "Switching from %s to %s\n", current_name, next_name);
+ if (NULL != current_name)
+ GNUNET_free (current_name);
+ current_name = GNUNET_strdup (next_name);
zonekey_set = GNUNET_YES;
continue;
}
@@ -1852,6 +1970,7 @@ process_command_stdin ()
fprintf (stderr, "Warning, encountered recordline without zone\n");
continue;
}
+ fprintf (stderr, "Adding record line %s\n", buf);
parse_recordline (buf);
}
if (GNUNET_NO == finished)
@@ -1860,7 +1979,29 @@ process_command_stdin ()
{
if (GNUNET_YES == zonekey_set)
{
- run_with_zone_pkey (cfg); /** one last time **/
+ record_info[ri_count].a_label = GNUNET_strdup (current_name);
+ int rd_count = 0;
+ for (struct RecordSetEntry *e = recordset; NULL != e; e = e->next)
+ rd_count++;
+ record_info[ri_count].a_rd = GNUNET_new_array (rd_count,
+ struct
+ GNUNET_GNSRECORD_Data);
+ rd_count = 0;
+ for (struct RecordSetEntry *e = recordset; NULL != e; e = e->next)
+ {
+ record_info[ri_count].a_rd[rd_count] = e->record;
+ record_info[ri_count].a_rd[rd_count].data = GNUNET_malloc (e->record
+ .
+
data_size);
+ record_info[ri_count].a_rd[rd_count].data_size = e->record.data_size
+ ;
+ memcpy ((void*) record_info[ri_count].a_rd[rd_count].data,
+ e->record.data, e->record.data_size);
+ rd_count++;
+ }
+ record_info[ri_count].a_rd_count = rd_count;
+ ri_count++;
+ batch_insert_recordinfo (cfg); /** One last time **/
finished = GNUNET_YES;
return;
}
@@ -2066,6 +2207,12 @@ main (int argc, char *const *argv)
gettext_noop (
"delete all records in specified zone"),
&purge_zone),
+ GNUNET_GETOPT_option_uint ('B',
+ "batch-size",
+ "NUMBER",
+ gettext_noop (
+ "delete all records in specified zone"),
+ &max_batch_size),
GNUNET_GETOPT_option_flag (
's',
"shadow",
diff --git a/src/cli/namestore/test_namestore_put_stdin.sh
b/src/cli/namestore/test_namestore_put_stdin.sh
index deb7bc4cb..74f68f971 100755
--- a/src/cli/namestore/test_namestore_put_stdin.sh
+++ b/src/cli/namestore/test_namestore_put_stdin.sh
@@ -7,7 +7,7 @@ if [ -z $LOCATION ]
then
LOCATION="gnunet-config"
fi
-$LOCATION --version 1> /dev/null
+$LOCATION --section arm 1> /dev/null
if test $? != 0
then
echo "GNUnet command line tools cannot be found, check environmental
variables PATH and GNUNET_PREFIX"
@@ -40,14 +40,32 @@ function stop_peer
start_peer
# Create a public record
EGOKEY=`gnunet-identity -d | grep testego2 | cut -d' ' -f3`
-gnunet-namestore -a -c $CONFIGURATION -S <<EOF
-$TEST_RECORD_NAME.testego:
+gnunet-namestore -B 3 -a -c $CONFIGURATION -S <<EOF
+${TEST_RECORD_NAME}.testego:
+ A 3600000000 [pr] $TEST_IP
+ TXT 21438201833 [r] $TEST_IP2
+
+ TXT 21438201833 [r] aslkdj asdlkjaslkd 231!
+
+${TEST_RECORD_NAME}1.testego:
A 3600000000 [pr] $TEST_IP
TXT 21438201833 [r] $TEST_IP2
TXT 21438201833 [r] aslkdj asdlkjaslkd 232!
-$TEST_RECORD_NAME2.testego:
+${TEST_RECORD_NAME}2.testego:
+ A 3600000000 [pr] $TEST_IP
+ TXT 21438201833 [r] $TEST_IP2
+
+ TXT 21438201833 [r] aslkdj asdlkjaslkd 233!
+
+${TEST_RECORD_NAME}3.testego:
+ A 3600000000 [pr] $TEST_IP
+ TXT 21438201833 [r] $TEST_IP2
+
+ TXT 21438201833 [r] aslkdj asdlkjaslkd 234!
+
+${TEST_RECORD_NAME}4.testego:
AAAA 324241223 [prS] ::dead:beef
A 111324241223000000 [pC] 1.1.1.1
@@ -56,13 +74,14 @@ www7.$EGOKEY:
EOF
NAMESTORE_RES=$?
+gnunet-identity -d -c $CONFIGURATION
gnunet-namestore -D -r -c $CONFIGURATION
stop_peer
if [ $NAMESTORE_RES = 0 ]
then
- echo "PASS: Creating name in namestore"
+ echo "PASS: Recordline import in namestore"
else
- echo "FAIL: Creating name in namestore failed with $NAMESTORE_RES."
+ echo "FAIL: Recordline import in namestore failed with $NAMESTORE_RES."
exit 1
fi
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [gnunet] branch master updated: namestore: allow for batch inserts from recordline stdin mode in CLI,
gnunet <=