[GNUnet-SVN] [gnurl] 09/178: rate-limit: use three second window to better handle high speeds

[gnunet]

This is an automated email from the git hooks/post-receive script.

ng0 pushed a commit to branch master
in repository gnurl.

commit f5700ea88baa5baeb63b7cd42bd12fb03f4e0f4e
Author: Daniel Stenberg <address@hidden>
AuthorDate: Thu Mar 15 16:43:00 2018 +0100

    rate-limit: use three second window to better handle high speeds
    
    Due to very frequent updates of the rate limit "window", it could
    attempt to rate limit within the same milliseconds and that then made
    the calculations wrong, leading to it not behaving correctly on very
    fast transfers.
    
    This new logic updates the rate limit "window" to be no shorter than the
    last three seconds and only updating the timestamps for this when
    switching between the states TOOFAST/PERFORM.
    
    Reported-by: 刘佩东
    Fixes #2386
    Closes #2388
---
 lib/multi.c    | 31 +++++++++++++-----------
 lib/progress.c | 74 ++++++++++++++++++++++++++++++++++------------------------
 lib/progress.h |  1 +
 3 files changed, 62 insertions(+), 44 deletions(-)

diff --git a/lib/multi.c b/lib/multi.c
index 98e5fca2a..69df90288 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -1823,22 +1823,26 @@ static CURLMcode multi_runsingle(struct Curl_multi 
*multi,
       if(!result) {
         send_timeout_ms = 0;
         if(data->set.max_send_speed > 0)
-          send_timeout_ms = Curl_pgrsLimitWaitTime(data->progress.uploaded,
-                                data->progress.ul_limit_size,
-                                data->set.max_send_speed,
-                                data->progress.ul_limit_start,
-                                now);
+          send_timeout_ms =
+            Curl_pgrsLimitWaitTime(data->progress.uploaded,
+                                   data->progress.ul_limit_size,
+                                   data->set.max_send_speed,
+                                   data->progress.ul_limit_start,
+                                   now);
 
         recv_timeout_ms = 0;
         if(data->set.max_recv_speed > 0)
-          recv_timeout_ms = Curl_pgrsLimitWaitTime(data->progress.downloaded,
-                                data->progress.dl_limit_size,
-                                data->set.max_recv_speed,
-                                data->progress.dl_limit_start,
-                                now);
-
-        if(send_timeout_ms <= 0 && recv_timeout_ms <= 0)
+          recv_timeout_ms =
+            Curl_pgrsLimitWaitTime(data->progress.downloaded,
+                                   data->progress.dl_limit_size,
+                                   data->set.max_recv_speed,
+                                   data->progress.dl_limit_start,
+                                   now);
+
+        if(!send_timeout_ms && !recv_timeout_ms) {
           multistate(data, CURLM_STATE_PERFORM);
+          Curl_ratelimit(data, now);
+        }
         else if(send_timeout_ms >= recv_timeout_ms)
           Curl_expire(data, send_timeout_ms, EXPIRE_TOOFAST);
         else
@@ -1870,7 +1874,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
                                                  data->progress.dl_limit_start,
                                                  now);
 
-      if(send_timeout_ms > 0 || recv_timeout_ms > 0) {
+      if(send_timeout_ms || recv_timeout_ms) {
+        Curl_ratelimit(data, now);
         multistate(data, CURLM_STATE_TOOFAST);
         if(send_timeout_ms >= recv_timeout_ms)
           Curl_expire(data, send_timeout_ms, EXPIRE_TOOFAST);
diff --git a/lib/progress.c b/lib/progress.c
index ce8be7ffb..f59faa3d3 100644
--- a/lib/progress.c
+++ b/lib/progress.c
@@ -28,6 +28,9 @@
 #include "progress.h"
 #include "curl_printf.h"
 
+/* check rate limits within this many recent milliseconds, at minimum. */
+#define MIN_RATE_LIMIT_PERIOD 3000
+
 /* Provide a string that is 2 + 1 + 2 + 1 + 2 = 8 letters long (plus the zero
    byte) */
 static void time2str(char *r, curl_off_t seconds)
@@ -235,6 +238,7 @@ void Curl_pgrsStartNow(struct Curl_easy *data)
   data->progress.dl_limit_start.tv_usec = 0;
   /* clear all bits except HIDE and HEADERS_OUT */
   data->progress.flags &= PGRS_HIDE|PGRS_HEADERS_OUT;
+  Curl_ratelimit(data, data->progress.start);
 }
 
 /*
@@ -265,13 +269,13 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize,
   time_t minimum;
   time_t actual;
 
-  /* we don't have a starting point yet -- return 0 so it gets (re)set */
-  if(start.tv_sec == 0 && start.tv_usec == 0)
-    return 0;
-
-  if(!limit)
+  if(!limit || !size)
     return 0;
 
+  /*
+   * 'minimum' is the number of milliseconds 'size' should take to download to
+   * stay below 'limit'.
+   */
   if(size < CURL_OFF_T_MAX/1000)
     minimum = (time_t) (CURL_OFF_T_C(1000) * size / limit);
   else {
@@ -282,48 +286,56 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize,
       minimum = TIME_T_MAX;
   }
 
+  /*
+   * 'actual' is the time in milliseconds it took to actually download the
+   * last 'size' bytes.
+   */
   actual = Curl_timediff(now, start);
-
-  if(actual < minimum)
+  if(actual < minimum) {
+    /* if it downloaded the data faster than the limit, make it wait the
+       difference */
     return (minimum - actual);
+  }
 
   return 0;
 }
 
+/*
+ * Set the number of downloaded bytes so far.
+ */
 void Curl_pgrsSetDownloadCounter(struct Curl_easy *data, curl_off_t size)
 {
-  struct curltime now = Curl_now();
-
   data->progress.downloaded = size;
+}
 
-  /* download speed limit */
-  if((data->set.max_recv_speed > 0) &&
-     (Curl_pgrsLimitWaitTime(data->progress.downloaded,
-                             data->progress.dl_limit_size,
-                             data->set.max_recv_speed,
-                             data->progress.dl_limit_start,
-                             now) == 0)) {
-    data->progress.dl_limit_start = now;
-    data->progress.dl_limit_size = size;
+/*
+ * Update the timestamp and sizestamp to use for rate limit calculations.
+ */
+void Curl_ratelimit(struct Curl_easy *data, struct curltime now)
+{
+  /* don't set a new stamp unless the time since last update is long enough */
+  if(data->set.max_recv_speed > 0) {
+    if(Curl_timediff(now, data->progress.dl_limit_start) >=
+       MIN_RATE_LIMIT_PERIOD) {
+      data->progress.dl_limit_start = now;
+      data->progress.dl_limit_size = data->progress.downloaded;
+    }
+  }
+  if(data->set.max_send_speed > 0) {
+    if(Curl_timediff(now, data->progress.ul_limit_start) >=
+       MIN_RATE_LIMIT_PERIOD) {
+      data->progress.ul_limit_start = now;
+      data->progress.ul_limit_size = data->progress.uploaded;
+    }
   }
 }
 
+/*
+ * Set the number of uploaded bytes so far.
+ */
 void Curl_pgrsSetUploadCounter(struct Curl_easy *data, curl_off_t size)
 {
-  struct curltime now = Curl_now();
-
   data->progress.uploaded = size;
-
-  /* upload speed limit */
-  if((data->set.max_send_speed > 0) &&
-     (Curl_pgrsLimitWaitTime(data->progress.uploaded,
-                             data->progress.ul_limit_size,
-                             data->set.max_send_speed,
-                             data->progress.ul_limit_start,
-                             now) == 0)) {
-    data->progress.ul_limit_start = now;
-    data->progress.ul_limit_size = size;
-  }
 }
 
 void Curl_pgrsSetDownloadSize(struct Curl_easy *data, curl_off_t size)
diff --git a/lib/progress.h b/lib/progress.h
index 3c2231cb6..2baa925db 100644
--- a/lib/progress.h
+++ b/lib/progress.h
@@ -46,6 +46,7 @@ void Curl_pgrsSetDownloadSize(struct Curl_easy *data, 
curl_off_t size);
 void Curl_pgrsSetUploadSize(struct Curl_easy *data, curl_off_t size);
 void Curl_pgrsSetDownloadCounter(struct Curl_easy *data, curl_off_t size);
 void Curl_pgrsSetUploadCounter(struct Curl_easy *data, curl_off_t size);
+void Curl_ratelimit(struct Curl_easy *data, struct curltime now);
 int Curl_pgrsUpdate(struct connectdata *);
 void Curl_pgrsResetTransferSizes(struct Curl_easy *data);
 void Curl_pgrsTime(struct Curl_easy *data, timerid timer);

-- 
To stop receiving notification emails like this one, please contact
address@hidden