guile-commits
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Guile-commits] 02/02: http: Do not buffer HTTP chunks.


From: Ludovic Courtès
Subject: [Guile-commits] 02/02: http: Do not buffer HTTP chunks.
Date: Sun, 01 Mar 2015 22:43:51 +0000

civodul pushed a commit to branch stable-2.0
in repository guile.

commit 00d3ecf27408982a4c4ff6baad4f7c2592d7cb5a
Author: Ludovic Courtès <address@hidden>
Date:   Sun Mar 1 23:41:01 2015 +0100

    http: Do not buffer HTTP chunks.
    
    Fixes <http://bugs.gnu.org/19939>.
    
    * module/web/http.scm (read-chunk, read-chunk-body): Remove.
      (make-chunked-input-port)[next-chunk, buffer-, buffer-size,
      buffer-pointer]: Remove.
      [chunk-size, remaining]: New variables.
      [read!]: Rewrite to write directly to BV.
    * test-suite/tests/web-http.test ("chunked encoding")["reads chunks
      without buffering", "reads across chunk boundaries"]: New tests.
---
 module/web/http.scm            |   66 +++++++++++++++++++--------------------
 test-suite/tests/web-http.test |   54 +++++++++++++++++++++++++++++++-
 2 files changed, 84 insertions(+), 36 deletions(-)

diff --git a/module/web/http.scm b/module/web/http.scm
index aa75142..d8c9ae6 100644
--- a/module/web/http.scm
+++ b/module/web/http.scm
@@ -1,6 +1,6 @@
 ;;; HTTP messages
 
-;; Copyright (C)  2010, 2011, 2012, 2013, 2014 Free Software Foundation, Inc.
+;; Copyright (C)  2010-2015 Free Software Foundation, Inc.
 
 ;; This library is free software; you can redistribute it and/or
 ;; modify it under the terms of the GNU Lesser General Public
@@ -1907,6 +1907,7 @@ treated specially, and is just returned as a plain 
string."
 
 ;; Chunked Responses
 (define (read-chunk-header port)
+  "Read a chunk header and return the chunk size."
   (let* ((str (read-line port))
          (extension-start (string-index str (lambda (c) (or (char=? c #\;)
                                                        (char=? c #\return)))))
@@ -1916,53 +1917,50 @@ treated specially, and is just returned as a plain 
string."
                                16)))
     size))
 
-(define (read-chunk port)
-  (let ((size (read-chunk-header port)))
-    (read-chunk-body port size)))
-
-(define (read-chunk-body port size)
-  (let ((bv (get-bytevector-n port size)))
-    (get-u8 port)                       ; CR
-    (get-u8 port)                       ; LF
-    bv))
-
 (define* (make-chunked-input-port port #:key (keep-alive? #f))
   "Returns a new port which translates HTTP chunked transfer encoded
 data from PORT into a non-encoded format. Returns eof when it has
 read the final chunk from PORT. This does not necessarily mean
 that there is no more data on PORT. When the returned port is
 closed it will also close PORT, unless the KEEP-ALIVE? is true."
-  (define (next-chunk)
-    (read-chunk port))
-  (define finished? #f)
   (define (close)
     (unless keep-alive?
       (close-port port)))
-  (define buffer #vu8())
-  (define buffer-size 0)
-  (define buffer-pointer 0)
+
+  (define chunk-size 0)     ;size of the current chunk
+  (define remaining 0)      ;number of bytes left from the current chunk
+  (define finished? #f)     ;did we get all the chunks?
+
   (define (read! bv idx to-read)
     (define (loop to-read num-read)
       (cond ((or finished? (zero? to-read))
              num-read)
-            ((<= to-read (- buffer-size buffer-pointer))
-             (bytevector-copy! buffer buffer-pointer
-                               bv (+ idx num-read)
-                               to-read)
-             (set! buffer-pointer (+ buffer-pointer to-read))
-             (loop 0 (+ num-read to-read)))
-            (else
-             (let ((n (- buffer-size buffer-pointer)))
-               (bytevector-copy! buffer buffer-pointer
-                                 bv (+ idx num-read)
-                                 n)
-               (set! buffer (next-chunk))
-               (set! buffer-pointer 0)
-               (set! buffer-size (bytevector-length buffer))
-               (set! finished? (= buffer-size 0))
-               (loop (- to-read n)
-                     (+ num-read n))))))
+            ((zero? remaining)                    ;get a new chunk
+             (let ((size (read-chunk-header port)))
+               (set! chunk-size size)
+               (set! remaining size)
+               (if (zero? size)
+                   (begin
+                     (set! finished? #t)
+                     num-read)
+                   (loop to-read num-read))))
+            (else                           ;read from the current chunk
+             (let* ((ask-for (min to-read remaining))
+                    (read    (get-bytevector-n! port bv (+ idx num-read)
+                                                ask-for)))
+               (if (eof-object? read)
+                   (begin                         ;premature termination
+                     (set! finished? #t)
+                     num-read)
+                   (let ((left (- remaining read)))
+                     (set! remaining left)
+                     (when (zero? left)
+                       ;; We're done with this chunk; read CR and LF.
+                       (get-u8 port) (get-u8 port))
+                     (loop (- to-read read)
+                           (+ num-read read))))))))
     (loop to-read 0))
+
   (make-custom-binary-input-port "chunked input port" read! #f #f close))
 
 (define* (make-chunked-output-port port #:key (keep-alive? #f))
diff --git a/test-suite/tests/web-http.test b/test-suite/tests/web-http.test
index 7bba9ec..c59674f 100644
--- a/test-suite/tests/web-http.test
+++ b/test-suite/tests/web-http.test
@@ -1,4 +1,4 @@
-;;;; web-uri.test --- URI library          -*- mode: scheme; coding: utf-8; -*-
+;;;; web-http.test --- HTTP library        -*- mode: scheme; coding: utf-8; -*-
 ;;;;
 ;;;;   Copyright (C) 2010, 2011, 2014, 2015 Free Software Foundation, Inc.
 ;;;;
@@ -20,6 +20,7 @@
 (define-module (test-suite web-http)
   #:use-module (web uri)
   #:use-module (web http)
+  #:use-module (rnrs bytevectors)
   #:use-module (rnrs io ports)
   #:use-module (ice-9 regex)
   #:use-module (ice-9 control)
@@ -363,7 +364,56 @@
     (pass-if-equal
         "First line\n Second line"
         (get-string-all p))
-    (pass-if (port-eof? (make-chunked-input-port (open-input-string 
"0\r\n")))))
+    (pass-if (port-eof? (make-chunked-input-port (open-input-string "0\r\n"))))
+
+    (pass-if-equal "reads chunks without buffering"
+        ;; Make sure the chunked input port does not read more than what
+        ;; the client asked.  See <http://bugs.gnu.org/19939>
+        `("First " "chunk." "Second " "chunk."
+          (1 1 1 6 6 1 1
+           1 1 1 7 6 1 1))
+      (let* ((str      "C\r\nFirst chunk.\r\nD\r\nSecond chunk.\r\n")
+             (requests '())
+             (read!    (let ((port (open-input-string str)))
+                         (lambda (bv index count)
+                           (set! requests (cons count requests))
+                           (let ((n (get-bytevector-n! port bv index
+                                                       count)))
+                             (if (eof-object? n) 0 n)))))
+             (input    (make-custom-binary-input-port "chunky" read!
+                                                      #f #f #f))
+             (port     (make-chunked-input-port input)))
+        (setvbuf input _IONBF)
+        (setvbuf port _IONBF)
+        (list (utf8->string (get-bytevector-n port 6))
+              (utf8->string (get-bytevector-n port 6))
+              (utf8->string (get-bytevector-n port 7))
+              (utf8->string (get-bytevector-n port 6))
+              (reverse requests))))
+
+    (pass-if-equal "reads across chunk boundaries"
+        ;; Same, but read across chunk boundaries.
+        `("First " "chunk.Second " "chunk."
+          (1 1 1 6 6 1 1
+           1 1 1 7 6 1 1))
+      (let* ((str      "C\r\nFirst chunk.\r\nD\r\nSecond chunk.\r\n")
+             (requests '())
+             (read!    (let ((port (open-input-string str)))
+                         (lambda (bv index count)
+                           (set! requests (cons count requests))
+                           (let ((n (get-bytevector-n! port bv index
+                                                       count)))
+                             (if (eof-object? n) 0 n)))))
+             (input    (make-custom-binary-input-port "chunky" read!
+                                                      #f #f #f))
+             (port     (make-chunked-input-port input)))
+        (setvbuf input _IONBF)
+        (setvbuf port _IONBF)
+        (list (utf8->string (get-bytevector-n port 6))
+              (utf8->string (get-bytevector-n port 13))
+              (utf8->string (get-bytevector-n port 6))
+              (reverse requests)))))
+
   (pass-if-equal
       (call-with-output-string
        (lambda (out-raw)



reply via email to

[Prev in Thread] Current Thread [Next in Thread]