[gnunet] 121/164: Made SE vary sizes depending on new formula

[gnunet]

This is an automated email from the git hooks/post-receive script.

grothoff pushed a commit to branch master
in repository gnunet.

commit 5b67ff0f52dc150630bbd7dac2458a82cb83652e
Author: Elias Summermatter <elias.summermatter@seccom.ch>
AuthorDate: Sat May 29 17:19:52 2021 +0200

    Made SE vary sizes depending on new formula
---
 src/setu/gnunet-service-setu.c                  | 41 +++++++++++++++----------
 src/setu/gnunet-service-setu_protocol.h         |  8 +++++
 src/setu/gnunet-service-setu_strata_estimator.c | 20 +++++++-----
 src/setu/gnunet-service-setu_strata_estimator.h | 20 +++++++++++-
 src/setu/perf_setu_api.c                        |  2 +-
 5 files changed, 65 insertions(+), 26 deletions(-)

diff --git a/src/setu/gnunet-service-setu.c b/src/setu/gnunet-service-setu.c
index aa2a2eba0..534585588 100644
--- a/src/setu/gnunet-service-setu.c
+++ b/src/setu/gnunet-service-setu.c
@@ -1052,18 +1052,11 @@ update_message_control_flow(struct 
GNUNET_CONTAINER_MultiHashMap *hash_map,
 {
     struct message_control_flow_element *cfe = NULL;
     enum MESSAGE_CONTROL_FLOW_STATE *mcfs;
-    /**
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         "%u NEW_STATE %u\n", *hash_code->bits, new_mcfs); **/
 
     cfe = GNUNET_CONTAINER_multihashmap_get(hash_map, hash_code);
     if(NULL == cfe) {
         cfe = (struct message_control_flow_element*) 
GNUNET_malloc(sizeof(struct message_control_flow_element));
     }
-    /**
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         "ID: %u OFFER: %u DEMAND: %u ELEMEMT: %u %u\n", *hash_code->bits, 
cfe->offer, cfe->demand, cfe->element);
-    **/
     if ( OFFER_MESSAGE == mt) {
         mcfs = &cfe->offer;
     } else if ( DEMAND_MESSAGE == mt ) {
@@ -1082,19 +1075,12 @@ update_message_control_flow(struct 
GNUNET_CONTAINER_MultiHashMap *hash_map,
     } else {
         return GNUNET_SYSERR;
     }
-    /**
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         "%u VALUE %u < %u \n", *hash_code->bits , new_mcfs, *mcfs); **/
 
     if(new_mcfs <= *mcfs) {
         return GNUNET_NO;
     }
 
     *mcfs = new_mcfs;
-    /**
-    LOG (GNUNET_ERROR_TYPE_ERROR,
-         "ID: %u OFFER: %u DEMAND: %u ELEMEMT: %u\n", *hash_code->bits, 
cfe->offer, cfe->demand, cfe->element);
-    **/
     GNUNET_CONTAINER_multihashmap_put(hash_map, 
hash_code,cfe,GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
     return GNUNET_YES;
 }
@@ -2095,6 +2081,16 @@ handle_union_p2p_strata_estimator (void *cls,
       return;
   }
 
+  /** Only allow 1,2,4,8 SEs **/
+  if(msg->se_count > 8 || __builtin_popcount((int)msg->se_count) != 1) {
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "PROTOCOL VIOLATION: Invalid number of se transmitted by other 
peer %u\n",
+             msg->se_count);
+        GNUNET_break_op (0);
+        fail_union_operation (op);
+        return;
+  }
+
   is_compressed = (GNUNET_MESSAGE_TYPE_SETU_P2P_SEC == htons (
                      msg->header.type));
   GNUNET_STATISTICS_update (_GSS_statistics,
@@ -2123,10 +2119,11 @@ handle_union_p2p_strata_estimator (void *cls,
     fail_union_operation (op);
     return;
   }
-  if (GNUNET_OK !=
+    if (GNUNET_OK !=
       strata_estimator_read (&msg[1],
                              len,
                              is_compressed,
+                             msg->se_count,
                              SE_IBFS_TOTAL_SIZE,
                              remote_se))
   {
@@ -4859,9 +4856,20 @@ handle_client_accept (void *cls,
 
     /* kick off the operation */
     se = op->se;
-    buf = GNUNET_malloc (se->stratas[0]->strata_count * IBF_BUCKET_SIZE * 
SE_IBFS_TOTAL_SIZE);
+
+    uint8_t se_count = 1;
+      if(op->initial_size > 0) {
+        GNUNET_CONTAINER_multihashmap_iterate(op->set->content->elements,
+                                              
&determinate_avg_element_size_iterator,
+                                              op);
+        se_count = determine_strata_count(
+                op->total_elements_size_local / op->initial_size,
+                op->initial_size);
+    }
+    buf = GNUNET_malloc (se->stratas[0]->strata_count * IBF_BUCKET_SIZE * 
((SE_IBFS_TOTAL_SIZE / 8) * se_count));
     len = strata_estimator_write (se,
                                   SE_IBFS_TOTAL_SIZE,
+                                  se_count,
                                   buf);
     perf_rtt.se.sent += 1;
     perf_rtt.se.sent_var_bytes += len;
@@ -4880,6 +4888,7 @@ handle_client_accept (void *cls,
     strata_msg->set_size
       = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (
                          op->set->content->elements));
+    strata_msg->se_count = se_count;
     GNUNET_MQ_send (op->mq,
                     ev);
     op->phase = PHASE_EXPECT_IBF;
diff --git a/src/setu/gnunet-service-setu_protocol.h 
b/src/setu/gnunet-service-setu_protocol.h
index 58a04e945..8a6ba8d62 100644
--- a/src/setu/gnunet-service-setu_protocol.h
+++ b/src/setu/gnunet-service-setu_protocol.h
@@ -212,6 +212,14 @@ struct StrataEstimatorMessage
    */
   struct GNUNET_MessageHeader header;
 
+  /**
+   * The number of ses transmitted
+   */
+  uint8_t se_count;
+
+  /**
+   * Size of the local set
+   */
   uint64_t set_size;
 };
 
diff --git a/src/setu/gnunet-service-setu_strata_estimator.c 
b/src/setu/gnunet-service-setu_strata_estimator.c
index ad0ee24e8..c8cecebea 100644
--- a/src/setu/gnunet-service-setu_strata_estimator.c
+++ b/src/setu/gnunet-service-setu_strata_estimator.c
@@ -37,13 +37,15 @@
  * @return number of bytes written to @a buf
  */
 size_t
-strata_estimator_write (const struct MultiStrataEstimator *se,
+strata_estimator_write (struct MultiStrataEstimator *se,
                         uint16_t se_ibf_total_size,
+                        uint8_t number_se_send,
                         void *buf)
 {
   char *sbuf = buf;
   size_t osize;
   uint64_t sbuf_offset = 0;
+  se->size = number_se_send;
 
   GNUNET_assert (NULL != se);
   for (unsigned int i = 0; i < se->strata_count; i++)
@@ -90,6 +92,7 @@ int
 strata_estimator_read (const void *buf,
                        size_t buf_len,
                        int is_compressed,
+                       uint8_t number_se_received,
                        uint16_t se_ibf_total_size,
                        struct MultiStrataEstimator *se)
 {
@@ -99,7 +102,7 @@ strata_estimator_read (const void *buf,
   dbuf = NULL;
   if (GNUNET_YES == is_compressed)
   {
-    osize = se_ibf_total_size * IBF_BUCKET_SIZE * se->stratas[0]->strata_count;
+    osize = ((se_ibf_total_size/8) * number_se_received) * IBF_BUCKET_SIZE * 
se->stratas[0]->strata_count;
     dbuf = GNUNET_decompress (buf,
                               buf_len,
                               osize);
@@ -112,7 +115,7 @@ strata_estimator_read (const void *buf,
     buf_len = osize;
   }
 
-  if (buf_len != se->stratas[0]->strata_count * se_ibf_total_size * 
IBF_BUCKET_SIZE)
+  if (buf_len != se->stratas[0]->strata_count * ((se_ibf_total_size/8) * 
number_se_received) * IBF_BUCKET_SIZE)
   {
     GNUNET_break (0);  /* very odd error */
     GNUNET_free (dbuf);
@@ -124,6 +127,7 @@ strata_estimator_read (const void *buf,
     ibf_read_slice (buf, 0, se->ibf_size, se->strata[i], 8);
     buf += se->ibf_size * IBF_BUCKET_SIZE;
   }
+  se->size=number_se_received;
   GNUNET_free (dbuf);
   return GNUNET_OK;
 }
@@ -142,13 +146,13 @@ strata_estimator_insert (struct MultiStrataEstimator *se,
 
 
   /* count trailing '1'-bits of v */
-  for(int strata_ctr=0; strata_ctr < SE_SE_COUNT; strata_ctr++) {
+  for(int strata_ctr=0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++) {
       unsigned int i;
       uint64_t v;
 
       struct IBF_Key salted_key;
       salt_key (&key,
-                strata_ctr * (64 / SE_SE_COUNT),
+                strata_ctr * (64 / MULTI_SE_BASE_COUNT),
                 &salted_key);
       v = salted_key.key_val;
       for (i = 0; v & 1; v >>= 1, i++) {
@@ -172,7 +176,7 @@ strata_estimator_remove (struct MultiStrataEstimator *se,
 {
 
   /* count trailing '1'-bits of v */
-  for(int strata_ctr=0; strata_ctr < SE_SE_COUNT; strata_ctr++) {
+  for(int strata_ctr=0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++) {
       uint64_t v;
       unsigned int i;
       v = key.key_val;
@@ -279,8 +283,8 @@ strata_estimator_difference (const struct 
MultiStrataEstimator *se1,
   }
   //GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "TOTAL LOCAL: %u\n",avg_local_diff);
   //GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "TOTAL REMOTE: %u\n",avg_remote_diff);
-  se1->stratas[0]->strata[0]->local_decoded_count = avg_local_diff / 
SE_SE_COUNT;
-  se1->stratas[0]->strata[0]->remote_decoded_count = avg_remote_diff / 
SE_SE_COUNT;
+  se1->stratas[0]->strata[0]->local_decoded_count = avg_local_diff / 
number_of_estimators;
+  se1->stratas[0]->strata[0]->remote_decoded_count = avg_remote_diff / 
number_of_estimators;
 }
 
 
diff --git a/src/setu/gnunet-service-setu_strata_estimator.h 
b/src/setu/gnunet-service-setu_strata_estimator.h
index 401f6ed61..51d06a73c 100644
--- a/src/setu/gnunet-service-setu_strata_estimator.h
+++ b/src/setu/gnunet-service-setu_strata_estimator.h
@@ -68,8 +68,24 @@ struct MultiStrataEstimator
      */
     struct StrataEstimator **stratas;
 
+    /**
+     * Number of strata estimators in struct
+     */
+     uint8_t size;
+
 };
 
+/**
+ * Deteminate how many strata estimators in the message are necessary
+ * @param avg_element_size
+ * @param element_count
+ * @return number of strata's
+ */
+
+uint8_t
+determine_strata_count(uint64_t avg_element_size,
+                            uint64_t element_count);
+
 
 /**
  * Write the given strata estimator to the buffer.
@@ -79,8 +95,9 @@ struct MultiStrataEstimator
  * @return number of bytes written to @a buf
  */
 size_t
-strata_estimator_write (const struct MultiStrataEstimator *se,
+strata_estimator_write (struct MultiStrataEstimator *se,
                         uint16_t se_ibf_total_size,
+                        uint8_t number_se_send,
                         void *buf);
 
 
@@ -98,6 +115,7 @@ int
 strata_estimator_read (const void *buf,
                        size_t buf_len,
                        int is_compressed,
+                       uint8_t number_se_received,
                        uint16_t se_ibf_total_size,
                        struct MultiStrataEstimator *se);
 
diff --git a/src/setu/perf_setu_api.c b/src/setu/perf_setu_api.c
index a0b94ce83..22cac1257 100644
--- a/src/setu/perf_setu_api.c
+++ b/src/setu/perf_setu_api.c
@@ -404,7 +404,7 @@ run (void *cls,
                 "Running real set-reconciliation\n");
     //init_set1 ();
     // limit ~23800 element total
-    initRandomSets(0, 1,500,32);
+    initRandomSets(50, 500,500,32);
 }
 
 void perf_thread() {

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.