bug#22679: 25.0.91; ibuffer-do-shell-command-pipe truncate output

[Tino Calancha]

On Fri, 19 Aug 2016, Stefan Monnier wrote:

> > -   :modifier-p nil)
> > -  (shell-command-on-region
> > -   (point-min) (point-max) command))
> > +   :modifier-p nil
> > +   :opstring "Shell command executed on"
> > +   :modifier-p nil
> > +   :before (funcall #'ibuffer--before-shell-command)
> > +   :after (funcall #'ibuffer--after-shell-command))
> > +  (let ((out-buf (get-buffer "*Shell Command Output*")))
> > +    (with-current-buffer out-buf (goto-char (point-max)))
> > +    (call-process-region (point-min) (point-max) command nil 
out-buf)))
> I haven't looked at the rest of your patch but this part looks wrong:
> the docstring indicates that `command' is expected to be a shell command
> whereas call-process-region expects an executable.

I have corrected the call to `call-process-region': now it uses
'shell-file-name' as the executable.

I have also added one test for this bug that the new patch pass.
This test is just for documentation: i don't want to push it
to the master branch because it assumes the machine has an 'awk'
executable in 'exec-path'.

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
> From b3fe9fe795d317d1798a1ad2d116ff52131f6612 Mon Sep 17 00:00:00 2001
From: Tino Calancha <tino.calancha@gmail.com>
Date: Sat, 20 Aug 2016 15:46:27 +0900
Subject: [PATCH 1/2] Fix Bug#22679

* lisp/ibuf-macs.el (define-ibuffer-op): Added optional args
'before' and 'after'.
'before' is a form to evaluate before the operation.
'after' is a form to evaluate after the operation.
* lisp/ibuf-ext.el (ibuffer--after-shell-command-pos): New defvar;
store a buffer position where to set the point in the output
buffer after operation complete.
(ibuffer--before-shell-command): New defun; erase output buffer
if 'shell-command-not-erase-buffer' is nil and
set 'ibuffer--after-shell-command-pos'.
(ibuffer--after-shell-command): New defun; set point in the
output buffer after operation complete.
(ibuffer-do-shell-command-pipe, ibuffer-do-shell-command-file):
Bind 'shell-command-not-erase-buffer' to non-nil while processing
the buffers;  use 'ibuffer--after-shell-command' to set the point
in the output buffer.
---
 lisp/ibuf-ext.el  | 87 
+++++++++++++++++++++++++++++++++++++++++++------------
 lisp/ibuf-macs.el |  8 ++++-
 2 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/lisp/ibuf-ext.el b/lisp/ibuf-ext.el
index f93957e..0d79617 100644
--- a/lisp/ibuf-ext.el
+++ b/lisp/ibuf-ext.el
@@ -346,14 +346,60 @@ ibuffer-backward-filter-group
     (ibuffer-backward-filter-group 1))
   (ibuffer-forward-line 0))

+;; The value `beg-last-out' in `shell-command-not-erase-buffer'
+;; set the point at the beginning of the output of the first
+;; buffer processed.
+(defvar ibuffer--after-shell-command-pos)
+
+(defun ibuffer--before-shell-command ()
+  (let ((obuf (get-buffer-create "*Shell Command Output*"))
+        (sym shell-command-not-erase-buffer)
+        final-pos)
+    (when (buffer-live-p obuf)
+      (with-current-buffer obuf
+        (unless sym
+          (setq buffer-read-only nil)
+          (let ((inhibit-read-only t))
+            (erase-buffer)))
+        (setq final-pos
+              (cond ((or (not sym) (eq sym 'beg-last-out))
+                     (point-max))
+                    ((eq sym 'save-point)
+                     (point))))
+        (setq ibuffer--after-shell-command-pos
+              final-pos)))))
+
+(defun ibuffer--after-shell-command ()
+  (let* ((obuf (get-buffer-create "*Shell Command Output*"))
+         (pos  ibuffer--after-shell-command-pos)
+         (win  (car (get-buffer-window-list obuf))))
+    (setq ibuffer--after-shell-command-pos nil)
+    (with-current-buffer obuf
+      (unless pos (setq pos (point-max)))
+      (goto-char pos)
+      ;; Set point in the window displaying obuf, if any; otherwise
+      ;; display buf temporary in selected frame and set the point.
+      (if win
+          (set-window-point win pos)
+        (save-window-excursion
+          (let ((win (display-buffer obuf '(nil (inhibit-switch-frame . 
t)))))
+            (set-window-point win pos)))))))
+
 ;;;###autoload (autoload 'ibuffer-do-shell-command-pipe "ibuf-ext")
 (define-ibuffer-op shell-command-pipe (command)
   "Pipe the contents of each marked buffer to shell command COMMAND."
   (:interactive "sPipe to shell command: "
    :opstring "Shell command executed on"
-   :modifier-p nil)
-  (shell-command-on-region
-   (point-min) (point-max) command))
+   :modifier-p nil
+   :opstring "Shell command executed on"
+   :modifier-p nil
+   :before (funcall #'ibuffer--before-shell-command)
+   :after (funcall #'ibuffer--after-shell-command))
+  (let ((out-buf (get-buffer "*Shell Command Output*")))
+    (with-current-buffer out-buf (goto-char (point-max)))
+    (call-process-region (point-min) (point-max)
+                         shell-file-name nil out-buf nil
+                         shell-command-switch command)))

 ;;;###autoload (autoload 'ibuffer-do-shell-command-pipe-replace 
"ibuf-ext")
 (define-ibuffer-op shell-command-pipe-replace (command)
@@ -363,26 +409,31 @@ shell-command-pipe-replace
    :active-opstring "replace buffer contents in"
    :dangerous t
    :modifier-p t)
-  (with-current-buffer buf
-    (shell-command-on-region (point-min) (point-max)
-                            command nil t)))
+  (call-process-region (point-min) (point-max)
+                       shell-file-name t buf nil
+                       shell-command-switch command))

 ;;;###autoload (autoload 'ibuffer-do-shell-command-file "ibuf-ext")
 (define-ibuffer-op shell-command-file (command)
   "Run shell command COMMAND separately on files of marked buffers."
   (:interactive "sShell command on buffer's file: "
-   :opstring "Shell command executed on"
-   :modifier-p nil)
-  (shell-command (concat command " "
-                        (shell-quote-argument
-                         (or buffer-file-name
-                             (let ((file
-                                    (make-temp-file
-                                     (substring
-                                      (buffer-name) 0
-                                      (min 10 (length (buffer-name)))))))
-                               (write-region nil nil file nil 0)
-                               file))))))
+                :opstring "Shell command executed on"
+                :modifier-p nil
+                :before (funcall #'ibuffer--before-shell-command)
+                :after (funcall #'ibuffer--after-shell-command))
+  (let ((file (and (not (buffer-modified-p))
+                   buffer-file-name))
+        (out-buf (get-buffer "*Shell Command Output*")))
+    (when (or (null file) (not (file-exists-p file)))
+      (setq file
+            (make-temp-file
+             (substring
+              (buffer-name) 0
+              (min 10 (length (buffer-name))))))
+      (write-region nil nil file nil 0))
+    (with-current-buffer out-buf (goto-char (point-max)))
+    (call-process-shell-command (format "%s %s" command file)
+                                nil out-buf nil)))

 ;;;###autoload (autoload 'ibuffer-do-eval "ibuf-ext")
 (define-ibuffer-op eval (form)
diff --git a/lisp/ibuf-macs.el b/lisp/ibuf-macs.el
index 27e7af9..8bb05ec 100644
--- a/lisp/ibuf-macs.el
+++ b/lisp/ibuf-macs.el
@@ -169,6 +169,8 @@ ibuffer-save-marks
                                  dangerous
                                  (opstring "operated on")
                                  (active-opstring "Operate on")
+                                  before
+                                  after
                                  complex)
                                 &rest body)
   "Generate a function which operates on a buffer.
@@ -198,6 +200,8 @@ ibuffer-save-marks
 ACTIVE-OPSTRING is a string which will be displayed to the user in a
 confirmation message, in the form:
  \"Really ACTIVE-OPSTRING x buffers?\"
+BEFORE is a form to evaluate before start the operation.
+AFTER is a form to evaluate once the operation is complete.
 COMPLEX means this function is special; if COMPLEX is nil BODY
 evaluates once for each marked buffer, MBUF, with MBUF current
 and saving the point.  If COMPLEX is non-nil, BODY evaluates
@@ -206,7 +210,7 @@ ibuffer-save-marks
 marked buffer.  BODY is evaluated with `buf' bound to the
 buffer object.

-\(fn OP ARGS DOCUMENTATION (&key INTERACTIVE MARK MODIFIER-P DANGEROUS 
OPSTRING ACTIVE-OPSTRING COMPLEX) &rest BODY)"
+\(fn OP ARGS DOCUMENTATION (&key INTERACTIVE MARK MODIFIER-P DANGEROUS 
OPSTRING ACTIVE-OPSTRING BEFORE AFTER COMPLEX) &rest BODY)"
   (declare (indent 2) (doc-string 3))
   `(progn
      (defun ,(intern (concat (if (string-match "^ibuffer-do" (symbol-name 
op))
@@ -233,11 +237,13 @@ ibuffer-save-marks
                                 'ibuffer-deletion-char)
                                (_
                                 'ibuffer-marked-char))))
+         ,before ; pre-operation form.
         ,(let* ((finish (append
                          '(progn)
                          (if (eq modifier-p t)
                              '((setq ibuffer-did-modification t))
                            ())
+                          (and after `(,after)) ; post-operation form.
                          `((ibuffer-redisplay t)
 			    (message ,(concat "Operation finished; " 
opstring " %s buffers") count))))
                 (inner-body (if complex
--
2.8.1

> From 156c63d38a78555fbdd8e04038ad96898a904019 Mon Sep 17 00:00:00 2001
From: Tino Calancha <tino.calancha@gmail.com>
Date: Sat, 20 Aug 2016 18:47:04 +0900
Subject: [PATCH 2/2] Add test for Bug#22679

* test/lisp/ibuffer-tests.el (ibuffer-test-bug22679):
---
 test/lisp/ibuffer-tests.el | 58 
++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/test/lisp/ibuffer-tests.el b/test/lisp/ibuffer-tests.el
index de281c0..5442bd2 100644
--- a/test/lisp/ibuffer-tests.el
+++ b/test/lisp/ibuffer-tests.el
@@ -30,5 +30,63 @@
     (symbol-function
      'ibuffer-mark-unsaved-buffers))))

+(ert-deftest ibuffer-test-bug22679 ()
+  "Test for http://debbugs.gnu.org/22679 ."
+  :expected-result :failed
+  (let* ((nums (generate-new-buffer "nums"))
+         (letters (generate-new-buffer "letters"))
+         (nums-file  "/tmp/nums-file")
+         (letters-file "/tmp/letters-file")
+         (str-nums "1 2 3 4 5\n6 7 8\n")
+         (str-letters "a b c d e\nf g h\n")
+         (str1 "1 2 3\n6 7 8\n")
+         (str2 "a b c\nf g h\n")
+         (buffer (get-buffer-create "*Shell Command Output*"))
+         (command "awk '{print $1 FS $2 FS $3}'")
+         (check-input-buffers (lambda ()
+                                (with-current-buffer nums
+                                  (should (string= (buffer-string) 
str1)))
+                                (with-current-buffer letters
+                                  (should (string= (buffer-string) 
str2)))))
+         (check-output-buffer (lambda ()
+                                (with-current-buffer buffer
+                                  (should (or (string= (buffer-string) 
(concat str1 str2))
+                                              (string= (buffer-string) 
(concat str2 str1)))))))
+         ;; Erase output buffer before each test.
+         (shell-command-not-erase-buffer nil)
+         ;; Don't ask for confimation to replace buffer content.
+         (ibuffer-expert t))
+    (with-current-buffer nums    (insert str-nums))
+    (with-current-buffer letters (insert str-letters))
+    (with-temp-file nums-file    (insert str-nums))
+    (with-temp-file letters-file (insert str-letters))
+    (unwind-protect
+        (save-current-buffer
+          (mapc 'find-file (list nums-file letters-file))
+          (ibuffer)
+          ;; Test ibuffer-do-shell-command-pipe[-replace]
+          (mapc (lambda (x)
+                  (ibuffer-mark-by-name-regexp (format "\\`%s\\'" x)))
+                (mapcar 'buffer-name (list nums letters)))
+          (ibuffer-do-shell-command-pipe command)
+          (funcall check-output-buffer)
+          (ibuffer-do-shell-command-pipe-replace command)
+          (funcall check-input-buffers)
+          ;; Test ibuffer-do-shell-command-file
+          (ibuffer-unmark-all-marks)
+          (mapc (lambda (x)
+                  (ibuffer-mark-by-name-regexp (format "\\`%s\\'" x)))
+                (mapcar 'buffer-name (list (get-file-buffer nums-file)
+                                           (get-file-buffer 
letters-file))))
+          (ibuffer-do-shell-command-file command)
+          (funcall check-output-buffer))
+      ;; Clean up.
+      (switch-to-buffer (current-buffer))
+      (mapc 'kill-buffer (list nums
+                               letters
+                               (get-file-buffer nums-file)
+                               (get-file-buffer letters-file)))
+      (mapc 'delete-file (list nums-file letters-file)))))
+
 (provide 'ibuffer-tests)
 ;; ibuffer-tests.el ends here
--
2.8.1

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

In GNU Emacs 25.1.50.14 (x86_64-pc-linux-gnu, GTK+ Version 3.20.7)
 of 2016-08-20 built on calancha-pc
Repository revision: a4ba426d25bd6a5cbe11d81b82a789b8a2c948ed