[nongnu] elpa/hyperdrive 063c2275d1 09/49: WIP: (-fill-version-ranges) Fill existent and nonexistent ranges

[ELPA Syncer]

branch: elpa/hyperdrive
commit 063c2275d151a086ebf248f50288bd46a49367f9
Author: Joseph Turner <joseph@ushin.org>
Commit: Joseph Turner <joseph@ushin.org>

    WIP: (-fill-version-ranges) Fill existent and nonexistent ranges
    
    Two codepaths:
    
    - fill existent entries in series
    - fill nonexistent entries in parallel
    
    Filling existent entries is logically recursive: fill entry, get its
    range-start, then fill the entry at the range-end of the prior range.
    
    Filling nonexistent entries is not recursive: In parallel, send
    a request to every version of entry between
    
    (hyperdrive-entry-version entry)
    and
    (- (hyperdrive-entry-version entry) hyperdrive-queue-size)
    
    If the earliest requested version is existent, go back to the
    "existent series" codepath. Otherwise, repeat the "nonexistent
    parallel" codepath.
    
    So good, so far.
    
    The problem is that the finally callback passed to
    `hyperdrive-fill-version-ranges' is not called in the correct place.
    We check the earliest requested version at the end of the "nonexistent
    parallel" codepath inside a separate queue finalizer for the codepath.
    Since the "nonexistent parallel" async requests are not made in the
    main queue used by "existent series" codepath, the main queue runs
    whenever the "existent series" codepath hands control over to the
    "nonexistent parallel" codepath.
    
    What about multiple queues per request, each with its own finalizer?
---
 hyperdrive-history.el |  1 +
 hyperdrive-lib.el     | 89 +++++++++++++++++++++++++++++++++++----------------
 2 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/hyperdrive-history.el b/hyperdrive-history.el
index 8ed090f8fd..b0fc207ca6 100644
--- a/hyperdrive-history.el
+++ b/hyperdrive-history.el
@@ -227,6 +227,7 @@ Universal prefix argument \\[universal-argument] forces
     (setf (hyperdrive-entry-version range-end-entry) range-end)
     (hyperdrive-fill-version-ranges range-end-entry
       :finally (lambda ()
+                 (message "FINALLY")
                  ;; TODO: Should we open the history buffer for entry
                  ;; or range-end-entry or...?
                  (hyperdrive-history entry)))))
diff --git a/hyperdrive-lib.el b/hyperdrive-lib.el
index faceaa7987..ecfc92201d 100644
--- a/hyperdrive-lib.el
+++ b/hyperdrive-lib.el
@@ -813,33 +813,68 @@ The QUEUE argument is used in recursive calls."
   (unless queue
     (setf queue (make-plz-queue :limit hyperdrive-queue-size
                                 :finally (when finally finally))))
-  (let ((copy-entry (hyperdrive-copy-tree entry t)))
-    (hyperdrive-fill copy-entry
-      :then (lambda (filled-entry)
-              (when (cl-plusp limit)
-                ;; Don't use `hyperdrive-entry-previous' here, since it makes 
a sync request
-                (pcase-let ((`(,range-start . ,_plist) 
(hyperdrive-entry-version-range filled-entry)))
-                  (setf (hyperdrive-entry-version filled-entry) (1- 
range-start))
-                  (when (eq 'unknown (hyperdrive-entry-exists-p filled-entry))
-                    ;; Recurse backward through history, filling unknown 
entries.
-                    ;; Stop recursing at known nonexistent entry or at the 
limit.
-                    (hyperdrive-fill-version-ranges filled-entry
-                      :limit (1- limit) :queue queue)))))
-      :else (lambda (err)
-              (pcase (plz-response-status (plz-error-response err))
-                ;; FIXME: If plz-error is a curl-error, this block will fail.
-                (404
-                 ;; ENTRY is known nonexistent: send LIMIT number of
-                 ;; parallel requests for entries before ENTRY's version.
-                 (cl-dotimes (i limit)
-                   (cl-decf (hyperdrive-entry-version copy-entry))
-                   (unless (and (cl-plusp (hyperdrive-entry-version 
copy-entry))
-                                (eq 'unknown (hyperdrive-entry-exists-p 
copy-entry)))
-                     (cl-return))
-                   (hyperdrive-fill-version-ranges copy-entry
-                     :limit 0 :queue queue)))
-                (_ (signal (car err) (cdr err)))))
-      :queue queue)))
+  (cl-labels ((fill-existent (entry limit)
+                ;; For existent entries, send requests in series.
+                (when (cl-plusp limit)
+                  ;; Don't use `hyperdrive-entry-previous' here, since it 
makes a sync request
+                  (pcase-let ((`(,range-start . ,_plist) 
(hyperdrive-entry-version-range entry)))
+                    (setf (hyperdrive-entry-version entry) (1- range-start))
+                    (when (eq 'unknown (hyperdrive-entry-exists-p entry))
+                      ;; Recurse backward through history.
+                      (hyperdrive-fill-version-ranges entry
+                        :limit (1- limit) :queue queue)))))
+              (fill-nonexistent (copy-entry limit)
+                (let ((nonexistent-queue (make-plz-queue
+                                          :limit hyperdrive-queue-size
+                                          :finally (lambda ()
+                                                     (let ((new-limit (- limit 
hyperdrive-queue-size))
+                                                           
(last-requested-entry (hyperdrive-copy-tree entry t)))
+                                                       (cl-incf 
(hyperdrive-entry-version last-requested-entry))
+                                                       ;; (message "ENTRY2: %s 
%s" (hyperdrive-entry-version entry) (hyperdrive-entry-exists-p 
last-requested-entry))
+                                                       (if 
(hyperdrive-entry-exists-p last-requested-entry)
+                                                           (fill-existent 
entry new-limit)
+                                                         (fill-nonexistent 
entry new-limit)))))))
+                  ;; For nonexistent entries, send requests in parallel.
+                  (cl-dotimes (i hyperdrive-queue-size)
+                    ;; Send the maximum number of simultaneous requests.
+                    (cl-decf (hyperdrive-entry-version entry))
+                    ;; (message "ENTRY0: %s %s %s %s" 
(hyperdrive-entry-version entry) (hyperdrive-entry-exists-p entry) limit i)
+                    (unless (and (cl-plusp (hyperdrive-entry-version entry))
+                                 (eq 'unknown (hyperdrive-entry-exists-p 
entry))
+                                 (> limit i))
+                      ;; Stop at the beginning of the history, at a known
+                      ;; existent/nonexistent entry, or at the limit.
+                      (cl-return))
+                    ;; (message "ENTRY1: %s %s" (hyperdrive-entry-version 
entry) (hyperdrive-entry-exists-p entry))
+
+                    (hyperdrive-fill (hyperdrive-copy-tree entry t)
+                      ;; `hyperdrive-fill' is only used to fill the version 
ranges;
+                      ;; the filled-entry is thrown away.
+                      :then (lambda (_filled-entry)
+                              ;; (message "KNOWN-EXISTENT: %s" 
(hyperdrive-entry-version filled-entry))
+                              (message "THEN")
+                              )
+                      :else (lambda (err)
+                              (message "KNOWN-NONEXISTENT: %s" 
(hyperdrive-entry-version entry))
+                              ;; TODO: Better error handling.
+                              (pcase (plz-response-status (plz-error-response 
err))
+                                ;; FIXME: If plz-error is a curl-error, this 
block will fail.
+                                (404 nil)
+                                (_ (signal (car err) (cdr err)))))
+                      :queue nonexistent-queue)))))
+    (let ((copy-entry (hyperdrive-copy-tree entry t)))
+      (hyperdrive-fill copy-entry
+        ;; `hyperdrive-fill' is only used to fill the version ranges;
+        ;; the filled-entry is thrown away.
+        :then (lambda (_filled-entry)
+                (fill-existent copy-entry limit))
+        :else (lambda (err)
+                (pcase (plz-response-status (plz-error-response err))
+                  ;; FIXME: If plz-error is a curl-error, this block will fail.
+                  (404
+                   (fill-nonexistent copy-entry limit))
+                  (_ (signal (car err) (cdr err)))))
+        :queue queue))))
 
 (defun hyperdrive-fill-metadata (hyperdrive)
   "Fill HYPERDRIVE's public metadata and return it.