[nongnu] elpa/bash-completion 0709a9803d 303/313: Support newlines in the output of complete -p.

[ELPA Syncer]

branch: elpa/bash-completion
commit 0709a9803d9226728c37cd9834dd9864e5ccca55
Author: Stephane Zermatten <szermatt@gmx.net>
Commit: Stephane Zermatten <szermatt@gmx.net>

    Support newlines in the output of complete -p.
    
    The configuration output by complete -p might contain multi-line
    strings, correctly quoted.
    
    Before this change, bash-completion-build-alist assumed that each
    command output by complete -p would span exactly one line, which isn't
    always true.
    
    With this change, bash-completion-build-alist tokenizes the output of
    complete-p as a series of bash commands separated by newlines. This
    allows a single command to span multiple lines as long as the newlines
    are inside of strings.
    
    For this to work, this change adapts the tokenizer to avoid
    unnecessarily making recursive calls, as recursion could then reach
    dangerous depths for large outputs.
---
 bash-completion.el           | 102 ++++++++++++++++++++++++-------------------
 test/bash-completion-test.el |  52 ++++++++++++++++------
 2 files changed, 96 insertions(+), 58 deletions(-)

diff --git a/bash-completion.el b/bash-completion.el
index e69c7148e3..5c391b9190 100644
--- a/bash-completion.el
+++ b/bash-completion.el
@@ -596,15 +596,11 @@ functions adds single quotes around it and return the 
result."
              "'"))))
 
 (defun bash-completion--parse (comp-start comp-pos wordbreaks 
bash-major-version)
-  "Process a command line split into TOKENS that end at POS.
+  "Process a command from COMP-START to COMP-POS.
 
 WORDBREAK is the value of COMP_WORDBREAKS to use for this completion,
 usually taken from the current process.
 
-This function takes a list of tokens built by
-`bash-completion-tokenize' and returns the variables compgen
-function expect in an association list.
-
 Returns a completion struct."
   (let* ((all-tokens (bash-completion-tokenize
                       comp-start comp-pos (if (>= bash-major-version 4)
@@ -673,7 +669,7 @@ Return a sublist of TOKENS."
        (dolist (token tokens)
          (let* ((string (bash-completion-tokenize-get-str token))
                 (is-terminal
-                 (and (member string '(";" "&" "|" "&&" "||"))
+                 (and (member string '(";" "&" "|" "&&" "||" "\n"))
                       (let ((range (bash-completion-tokenize-get-range token)))
                         (= (- (cdr range) (car range))
                            (length string))))))
@@ -729,7 +725,8 @@ set using `bash-completion-tokenize-set-end'.
 Tokens should always be accessed using the functions specified above,
 never directly as they're likely to change as this code evolves.
 The current format of a token is '(string . (start . end))."
-  (let ((bash-completion-wordbreaks
+  (let ((tokens nil)
+        (bash-completion-wordbreaks
          (mapconcat 'char-to-string
                     (delq nil (mapcar
                                (lambda (c)
@@ -738,26 +735,36 @@ The current format of a token is '(string . (start . 
end))."
                     "")))
     (save-excursion
       (goto-char start)
-      (nreverse (bash-completion-tokenize-new-element end nil)))))
+      (while (progn (skip-chars-forward " \t\r" end)
+                    (< (point) end))
+        (setq tokens
+              (bash-completion-tokenize-new-element end tokens)))
+      (nreverse tokens))))
 
-(defun bash-completion-tokenize-new-element (end tokens)
-  "Tokenize the rest of the line until END and complete TOKENS.
+(defun bash-completion-tokenize-new-element (limit tokens)
+  "Tokenize an element from point, up until LIMIT and complete TOKENS.
 
 This function is meant to be called exclusively from
 `bash-completion-tokenize' and `bash-completion-tokenize-0'.
 
-This function expect the point to be at the start of a new
-element to be added to the list of tokens.
-
-Return TOKENS with new tokens found between the current point and
-END prepended to it."
-  (skip-chars-forward " \t\n\r" end)
-  (if (< (point) end)
-      (bash-completion-tokenize-0 end tokens
-                                  (list
-                                   (cons 'str "")
-                                   (cons 'range (cons (point) nil))))
-    tokens))
+This function expects the point to be at the start of a new
+element to be added to the list of tokens. It parses the line
+until the limit of that element or until LIMIT.
+
+It leaves the point at the position where parsing should
+continue.
+
+Return TOKENS with new tokens prepended."
+  (skip-chars-forward " \t\r" limit)
+  (if (eq ?\n (char-after))
+      (progn
+        (goto-char (1+ (point)))
+        (cons `((str . "\n") (range ,(point) . ,(1+ (point)))) tokens))
+    (bash-completion-tokenize-0
+     limit tokens
+     (list
+      (cons 'str "")
+      (cons 'range (cons (point) nil))))))
 
 (defun bash-completion-tokenize-0 (end tokens token)
   "Tokenize the rest of the token until END and add it into TOKENS.
@@ -796,7 +803,8 @@ the token to the list of token and calls
 `bash-completion-tokenize-new-element' to look for the next
 token.
 
-END specifies the point at which tokenization should stop.
+END specifies the point at which tokenization should stop, even
+if the token is not complete.
 
 QUOTE specifies the current quote.  It should be nil, ?' or ?\"
 
@@ -804,7 +812,8 @@ TOKENS is the list of tokens built so far in reverse order.
 
 TOKEN is the token currently being built.
 
-Return TOKENS with new tokens prepended to it."
+Sets the point at the position of the next token. Returns TOKENS
+with new tokens prepended to it."
   ;; parse the token elements at the current position and
   ;; append them
   (let ((local-start (point)))
@@ -841,8 +850,7 @@ Return TOKENS with new tokens prepended to it."
     (bash-completion-tokenize-set-end token)
     (when quote
       (push (cons 'quote quote) token))
-    (push token tokens)
-    (bash-completion-tokenize-new-element end tokens))))
+    (push token tokens))))
 
 (defun bash-completion-nonsep (quote wordbreaks)
   "Return the set of non-breaking characters when QUOTE is the current quote.
@@ -1285,24 +1293,30 @@ The returned alist is a slightly parsed version of the 
output of
   (let ((alist (list)))
     (with-current-buffer buffer
       (save-excursion
-        (setq alist nil)
-        (goto-char (point-max))
-        (while (= 0 (forward-line -1))
-          (let ((words (bash-completion-strings-from-tokens
-                        (bash-completion-tokenize
-                         (line-beginning-position)
-                         (line-end-position)))))
-            (when (string= "complete" (car words))
-              (if (member "-D" (cdr words))
-                  ;; default completion
-                  (push (cons nil (delete "-D" (cdr words))) alist)
-                ;; normal completion
-                (let* ((reverse-wordsrest (nreverse (cdr words)))
-                       (command (car reverse-wordsrest))
-                       (options (nreverse (cdr reverse-wordsrest))) )
-                  (when (and command options)
-                    (push (cons command options) alist)))))))))
-    alist))
+        (goto-char (point-min))
+        (when (search-forward-regexp "^complete" nil 'noerror)
+          (let ((tokens (bash-completion-strings-from-tokens
+                         (bash-completion-tokenize
+                          (match-beginning 0) (point-max)))))
+            (while tokens
+              (let ((command tokens)
+                    (command-end (member "\n" tokens)))
+                (setq tokens (cdr command-end))
+                (when command-end
+                  (setcdr command-end nil))
+                (when (string= "complete" (car command))
+                  (setq command (nreverse (cdr command)))
+                  (when (equal "\n" (car command))
+                    (setq command (cdr command)))
+                  (if (member "-D" command)
+                      ;; default completion
+                      (push (cons nil (nreverse (delete "-D" command))) alist)
+                    ;; normal completion
+                    (let ((command-name (car command))
+                          (options (nreverse (cdr command))))
+                      (when (and command-name options)
+                        (push (cons command-name options) alist)))))))))))
+    (nreverse alist)))
 
 (defun bash-completion--customize (comp process &optional nodefault)
   (unless (eq 'command (bash-completion--type comp))
diff --git a/test/bash-completion-test.el b/test/bash-completion-test.el
index e96ffd936a..bbe7cd56f6 100644
--- a/test/bash-completion-test.el
+++ b/test/bash-completion-test.el
@@ -60,84 +60,91 @@ The return value is the one returned by BODY."
                 (bash-completion-test-with-buffer
                  "a hello world b c"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
-  ;; extra spaces
-  (should (equal '("a" "hello" "world" "b" "c")
+  ;; extra spaces and newline
+  (should (equal '("a" "hello" "\n" "world" "b" "c")
                 (bash-completion-test-with-buffer
                  "  a  hello \n world \t b \r c  "
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position 2))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; escaped spaces
   (should (equal '("a" "hello world" "b" "c")
                 (bash-completion-test-with-buffer
                  "a hello\\ world b c"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; double quotes
   (should (equal '("a" "hello world" "b" "c")
                 (bash-completion-test-with-buffer
                  "a \"hello world\" b c"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; escaped double quotes
   (should (equal '("a" "-\"hello world\"-" "b" "c")
                 (bash-completion-test-with-buffer
                  "a \"-\\\"hello world\\\"-\" b c"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; single quotes
   (should (equal '("a" "hello world" "b" "c")
                 (bash-completion-test-with-buffer
                  "a \"hello world\" b c"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
+
+  ;; quotes containing newline
+  (should (equal '("a" "hello\nworld" "b" "c")
+                (bash-completion-test-with-buffer
+                 "a \"hello\nworld\" b c"
+                 (bash-completion-strings-from-tokens
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; escaped single quotes
   (should (equal '("a" "-'hello world'-" "b" "c")
                 (bash-completion-test-with-buffer
                  "a '-\\'hello world\\'-' b c"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; complex quote mix
   (should (equal '("a" "hello world bc" "d")
                 (bash-completion-test-with-buffer
                  "a hel\"lo w\"o'rld b'c d"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; unescaped semicolon
   (should (equal '("to" "infinity" ";" "and beyond")
                 (bash-completion-test-with-buffer
                  "to infinity;and\\ beyond"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; unescaped &&"
   (should (equal '("to" "infinity" "&&" "and beyond")
                 (bash-completion-test-with-buffer
                  "to infinity&&and\\ beyond"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;;unescaped ||"
   (should (equal '("to" "infinity" "||" "and beyond")
                 (bash-completion-test-with-buffer
                  "to infinity||and\\ beyond"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position))))))
+                  (bash-completion-tokenize 1 (point-max))))))
 
   ;; quoted ;&|"
   (should (equal '("to" "infinity;&|and" "beyond")
                 (bash-completion-test-with-buffer
                  "to \"infinity;&|and\" beyond"
                  (bash-completion-strings-from-tokens
-                  (bash-completion-tokenize 1 (line-end-position)))))))
+                  (bash-completion-tokenize 1 (point-max)))))))
 
 (ert-deftest bash-completion--parse-test ()
   (let ((wordbreaks "@><=;|&(:'\""))
@@ -183,6 +190,20 @@ The return value is the one returned by BODY."
             "cd /var/tmp ; ZORG=t make -"
             (bash-completion--parse (point-min) 28 wordbreaks 3))))
 
+  ;; multiple commands on multiple lines
+  (should (equal
+           (bash-completion--make
+            :line "make -"
+            :cword 1
+            :words '("make" "-")
+            :stub-start 18
+            :stub "-"
+            :unparsed-stub "-"
+            :wordbreaks wordbreaks)
+           (bash-completion-test-with-buffer
+            "cd /var/tmp\nmake -"
+            (bash-completion--parse (point-min) (point-max) wordbreaks 3))))
+
   ;; pipe
   (should (equal
            (bash-completion--make
@@ -330,6 +351,7 @@ The return value is the one returned by BODY."
           '(("cdb" "-F" "_cdargs_aliases")
             ("project" "-F" "complete_projects")
             ("pro" "-F" "complete_projects")
+             ("scp" "-o" "default" "-W" "home\nhome.lan")
             ("cv" "-F" "_cdargs_aliases")
             ("cb" "-F" "_cdargs_aliases")
             (nil "-F" "_completion_loader"))
@@ -338,6 +360,8 @@ The return value is the one returned by BODY."
 complete -F _cdargs_aliases cdb
 complete -F complete_projects project
 complete -F complete_projects pro
+complete -o default -W 'home
+home.lan' scp
 complete -F _cdargs_aliases cv
 complete -F _cdargs_aliases cb
 complete -F _completion_loader -D