[elpa] externals/javaimp 81bd14f2fa: Use progress reporter for reading jars / parsing sources

[Filipp Gunbin]

branch: externals/javaimp
commit 81bd14f2fa4ef5ff23fb89445e74b9934090fa69
Author: Filipp Gunbin <fgunbin@okko.tv>
Commit: Filipp Gunbin <fgunbin@okko.tv>

    Use progress reporter for reading jars / parsing sources
---
 javaimp-gradle.el |  21 ++++---
 javaimp-maven.el  |  13 +++--
 javaimp-util.el   |  26 +++++----
 javaimp.el        | 169 ++++++++++++++++++++++++++++++------------------------
 4 files changed, 131 insertions(+), 98 deletions(-)

diff --git a/javaimp-gradle.el b/javaimp-gradle.el
index 6fd35e6660..24e00761ab 100644
--- a/javaimp-gradle.el
+++ b/javaimp-gradle.el
@@ -33,14 +33,16 @@ gradlew (Gradle wrapper), it is used in preference."
 Passes specially crafted init file as -I argument to gradle and
 invokes task contained in it.  This task outputs all needed
 information."
-  (message "Visiting Gradle build file %s..." file)
+  (when javaimp-verbose
+    (message "Visiting Gradle build file %s..." file))
   (let* ((alists (javaimp-gradle--call file #'javaimp-gradle--handler))
          (modules (mapcar (lambda (alist)
                             (javaimp-gradle--module-from-alist alist file))
                           alists)))
     ;; first module is always root
-    (message "Building tree for root: %s"
-             (javaimp-print-id (javaimp-module-id (car modules))))
+    (when javaimp-verbose
+      (message "Building tree for root: %s"
+               (javaimp-print-id (javaimp-module-id (car modules)))))
     (list
      (javaimp-tree-build (car modules) modules
                         ;; more or less reliable way to find children
@@ -133,13 +135,15 @@ descriptor."
          ;; in build file directory.
          (default-directory (file-name-directory file))
          ;; Prefer local gradle wrapper
-         (local-gradlew (if (memq system-type '(cygwin windows-nt))
+         (local-gradlew (if (eq system-type '(windows-nt))
                             "gradlew.bat"
                           "gradlew"))
          (program (if (file-exists-p local-gradlew)
                       (concat default-directory local-gradlew)
-                    javaimp-gradle-program)))
-    (javaimp-call-build-tool
+                    javaimp-gradle-program))
+         (task (concat mod-path "javaimpTask")))
+    (message "Calling Gradle task %s on %s ..." task file)
+    (javaimp-call-java-program
      program
      handler
      "-q"
@@ -149,8 +153,7 @@ descriptor."
      "-Dorg.gradle.java.compile-classpath-packaging=true"
      "-I" (javaimp-cygpath-convert-file-name
            (expand-file-name "javaimp-init-script.gradle"
-                             (concat javaimp-basedir
-                                     (file-name-as-directory "support"))))
-     (concat mod-path "javaimpTask"))))
+                             (file-name-concat javaimp-basedir "support")))
+     task)))
 
 (provide 'javaimp-gradle)
diff --git a/javaimp-maven.el b/javaimp-maven.el
index 49b4388b3a..02a036c76f 100644
--- a/javaimp-maven.el
+++ b/javaimp-maven.el
@@ -35,7 +35,8 @@ mvnw (Maven wrapper), it is used in preference."
 reads project structure from the output and records which files
 belong to which modules and other module information.  Returns
 resulting module trees."
-  (message "Visiting Maven POM file %s..." file)
+  (when javaimp-verbose
+    (message "Visiting Maven POM file %s..." file))
   (let* ((xml-tree (javaimp-maven--call
                     file
                     #'javaimp-maven--effective-pom-handler
@@ -76,8 +77,9 @@ resulting module trees."
                                          modules)))
                        (cdr modules)))))
       (mapcar (lambda (root)
-                (message "Building tree for root: %s"
-                         (javaimp-print-id (javaimp-module-id root)))
+                (when javaimp-verbose
+                  (message "Building tree for root: %s"
+                           (javaimp-print-id (javaimp-module-id root))))
                 (javaimp-tree-build
                  root modules
                 ;; more or less reliable way to find children is to
@@ -210,13 +212,14 @@ are somewhat arbitrary."
 (defun javaimp-maven--call (file handler task &optional dir)
   (let* ((default-directory (or dir (file-name-directory file)))
          ;; Prefer local mvn wrapper
-         (local-mvnw (if (memq system-type '(cygwin windows-nt))
+         (local-mvnw (if (memq system-type '(windows-nt))
                          "mvnw.cmd"
                        "mvnw"))
          (program (if (file-exists-p local-mvnw)
                       (concat default-directory local-mvnw)
                     javaimp-mvn-program)))
-    (javaimp-call-build-tool
+    (message "Calling Maven task %s on %s ..." task file)
+    (javaimp-call-java-program
      program
      handler
      "-f" (javaimp-cygpath-convert-file-name file)
diff --git a/javaimp-util.el b/javaimp-util.el
index a8850487d2..d6d8249f91 100644
--- a/javaimp-util.el
+++ b/javaimp-util.el
@@ -43,9 +43,15 @@ it is initialized from the JAVA_HOME environment variable."
   :type 'string
   :group 'javaimp)
 
-(defvar javaimp-tool-output-buf-name "*javaimp-tool-output*"
-  "Name of the buffer to which `javaimp-call-build-tool' copies
-build tool output.  Can be let-bound to nil to suppress copying.")
+(defcustom javaimp-verbose nil
+  "If non-nil, be verbose."
+  :type 'boolean
+  :group 'javaimp)
+
+(defvar javaimp-output-buf-name "*javaimp-output*"
+  "Name of the buffer to which `javaimp-call-java-program' copies
+invoked programs's output.  Can be let-bound to nil to suppress
+copying.")
 
 
 
@@ -223,10 +229,9 @@ the FILENAME unchanged."
          (car (apply #'process-lines javaimp-cygpath-program args))))
     filename))
 
-(defun javaimp-call-build-tool (program handler &rest args)
+(defun javaimp-call-java-program (program handler &rest args)
   "Run PROGRAM with ARGS, then call HANDLER in the temporary buffer
 with point set to eob and return its result."
-  (message "Calling: %s %s" program (string-join args " "))
   (with-temp-buffer
     (let ((status
            (let ((coding-system-for-read
@@ -235,19 +240,20 @@ with point set to eob and return its result."
                   (if javaimp-java-home
                       (cons (format "JAVA_HOME=%s" javaimp-java-home)
                             process-environment)
-                    process-environment)))
+                    process-environment))
+                 (process-connection-type nil))
              (apply #'process-file program nil t nil args)))
          (buf (current-buffer)))
-      (when javaimp-tool-output-buf-name
+      (when javaimp-output-buf-name
         (with-current-buffer (get-buffer-create
-                              javaimp-tool-output-buf-name)
+                              javaimp-output-buf-name)
           (setq buffer-read-only nil)
          (erase-buffer)
          (insert-buffer-substring buf)
           (setq buffer-read-only t)))
       (unless (and (numberp status) (= status 0))
-        (when javaimp-tool-output-buf-name
-          (display-buffer javaimp-tool-output-buf-name))
+        (when javaimp-output-buf-name
+          (display-buffer javaimp-output-buf-name))
        (error "%s exit status: %s" program status))
       (goto-char (point-min))
       (funcall handler))))
diff --git a/javaimp.el b/javaimp.el
index 0638536627..cd872d580c 100644
--- a/javaimp.el
+++ b/javaimp.el
@@ -61,7 +61,7 @@
 ;; build tool, see `javaimp-maven-visit' and `javaimp-gradle-visit',
 ;; and the `javaimp-handler-regexp-alist' variable.  The output from
 ;; the build tool can be inspected in buffer named by
-;; `javaimp-tool-output-buf-name' variable.  If there exists
+;; `javaimp-output-buf-name' variable.  If there exists
 ;; Maven/Gradle wrapper in the project directory, as it is popular
 ;; these days, it will be used in preference to `javaimp-mvn-program'
 ;; / `javaimp-gradle-program'.
@@ -102,6 +102,9 @@
 ;; presented in a nested fashion, instead of a flat list (the
 ;; default).
 ;;
+;; - `javaimp-verbose' - set to non-nil to get more messages about
+;; what Javaimp is doing.
+;;
 ;; See other defcustoms via 'M-x customize-group javaimp'.
 ;;
 ;;
@@ -198,11 +201,6 @@ in per-directory locals)."
 class)."
   :type 'boolean)
 
-(defcustom javaimp-verbose nil
-  "If non-nil, be verbose."
-  :type 'boolean)
-
-
 (defcustom javaimp-jar-program "jar"
   "Path to the `jar' program used to read contents of jar files."
   :type 'string)
@@ -314,7 +312,7 @@ any module's source file."
 ;; Dependencies
 
 (defsubst javaimp--get-file-ts (file)
-  (nth 5 (file-attributes file)))
+  (file-attribute-modification-time (file-attributes file)))
 
 (defun javaimp--update-module-maybe (node)
   (let ((module (javaimp-node-contents node))
@@ -349,25 +347,19 @@ any module's source file."
       (setf (javaimp-module-load-ts module)
             (current-time)))))
 
-(defun javaimp--get-jar-classes (file)
-  (javaimp--get-file-classes-cached
-   file
-   'javaimp-jar-file-cache
-   #'javaimp--read-jar-classes))
-
 (defun javaimp--read-jar-classes (file)
   "Read FILE which should be a .jar or a .jmod and return classes
 contained in it as a list."
   (let ((ext (downcase (file-name-extension file))))
     (unless (member ext '("jar" "jmod"))
       (error "Unexpected file name: %s" file))
-    (let ((javaimp-tool-output-buf-name nil)) ;don't log
-      (javaimp-call-build-tool
+    (let ((javaimp-output-buf-name nil))
+      (javaimp-call-java-program
        (symbol-value (intern (format "javaimp-%s-program" ext)))
        #'javaimp--read-jar-classes-handler
        (if (equal ext "jar") "tf" "list")
-       ;; On cygwin, "jar/jmod" is a windows program, so file path
-       ;; needs to be converted appropriately.
+       ;; On cygwin, "jar/jmod" is a native windows program, so file
+       ;; path needs to be converted appropriately.
        (javaimp-cygpath-convert-file-name file 'windows)))))
 
 (defun javaimp--read-jar-classes-handler ()
@@ -398,7 +390,9 @@ should be an alist with elements of the form (FILE
 . CACHED-FILE).  If not found in cache, or the cache is outdated,
 then classes are read using CLASS-READER, which should be a
 function of one argument, a FILE.  If that function throws an
-error, the cache for FILE is cleared."
+error, the cache for FILE is cleared.  CLASS-READER may also be
+nil, in which case the symbol t is returned for a cache miss, and
+cache not updated."
   (condition-case err
       (let ((cached-file
              (alist-get file (symbol-value cache-sym) nil nil #'string=)))
@@ -407,13 +401,17 @@ error, the cache for FILE is cleared."
                   ;; time, and thus condition always true
                   (> (float-time (javaimp--get-file-ts file))
                     (float-time (javaimp-cached-file-read-ts cached-file))))
-          (setq cached-file (make-javaimp-cached-file
-                            :file file
-                            :read-ts (javaimp--get-file-ts file)
-                            :classes (funcall class-reader file))))
-        (setf (alist-get file (symbol-value cache-sym) nil 'remove #'string=)
-              cached-file)
-        (javaimp-cached-file-classes cached-file))
+          (setq cached-file (if class-reader
+                                (make-javaimp-cached-file
+                                :file file
+                                :read-ts (javaimp--get-file-ts file)
+                                :classes (funcall class-reader file))
+                              t)))
+        (if (eq cached-file t)
+            t
+          (setf (alist-get file (symbol-value cache-sym) nil 'remove #'string=)
+                cached-file)
+          (javaimp-cached-file-classes cached-file)))
     (t
      ;; Clear on any error
      (setf (alist-get file (symbol-value cache-sym) nil 'remove #'string=) nil)
@@ -476,10 +474,13 @@ current module or source tree, see
                     ;; jdk
                     (when javaimp-java-home
                       (javaimp--get-jdk-classes javaimp-java-home))
-                    ;; module dependencies
+                    ;; Module dependencies.  We build the list each
+                    ;; time because jars may change.
                     (when module
-                      (javaimp--get-module-deps-classes module))
-                    ;; current module or source tree
+                      (javaimp--collect-classes 'javaimp-jar-file-cache
+                                                #'javaimp--read-jar-classes
+                                                (javaimp-module-dep-jars 
module)))
+                    ;; Current module or source tree
                     (when javaimp-parse-current-module
                       (seq-mapcat #'javaimp--get-directory-classes
                                   (javaimp--get-current-source-dirs module)))))
@@ -499,36 +500,51 @@ current module or source tree, see
 JAVA-HOME (earlier Java versions), read all .jar files in it."
   (let ((dir (concat (file-name-as-directory java-home) "jmods")))
     (if (file-directory-p dir)
-        (seq-mapcat #'javaimp--get-jar-classes
-                    (directory-files dir t "\\.jmod\\'"))
+        (javaimp--collect-classes
+         'javaimp-jar-file-cache #'javaimp--read-jar-classes
+         (directory-files dir t "\\.jmod\\'"))
       (setq dir (mapconcat #'file-name-as-directory
                            `(,java-home "jre" "lib") nil))
       (if (file-directory-p dir)
-          (seq-mapcat #'javaimp--get-jar-classes
-                      (directory-files dir t "\\.jar\\'"))
+          (javaimp--collect-classes
+           'javaimp-jar-file-cache #'javaimp--read-jar-classes
+           (directory-files dir t "\\.jar\\'"))
         (user-error "Could not load JDK classes")))))
 
-(defun javaimp--get-module-deps-classes (module)
-  ;; We're not caching full list of classes coming from
-  ;; module dependencies because jars may change
-  (let (jar-errors)
-    (prog1
-        (seq-mapcat
-         (lambda (jar)
-           (condition-case err
-               (javaimp--get-jar-classes jar)
-             (t
-              (push (concat jar ": " (error-message-string err))
-                    jar-errors)
-              nil)))
-         (javaimp-module-dep-jars module))
-      (when jar-errors
-        (with-output-to-temp-buffer "*Javaimp Jar errors*"
-          (princ javaimp--jar-error-header)
-          (terpri)
-          (dolist (err (nreverse jar-errors))
-            (princ err)
-            (terpri)))))))
+(defun javaimp--collect-classes (cache-sym fun files)
+  (let (tmp unread res errors)
+    ;; Collect from cache hits
+    (dolist (file files)
+      (setq tmp (javaimp--get-file-classes-cached file cache-sym nil))
+      (if (eq tmp t)
+          (push file unread)
+        (setq res (nconc res (copy-sequence tmp)))))
+    ;; Now read all cache misses
+    (when unread
+      (let ((reporter (make-progress-reporter
+                       (format "Reading %d files (%d taken from cache) ..."
+                               (length unread) (- (length files) (length 
unread)))
+                       0 (length unread)))
+            (i 0))
+        (dolist (file unread)
+          (setq tmp (condition-case err
+                        (javaimp--get-file-classes-cached file cache-sym fun)
+                      (t
+                       (push (concat file ": " (error-message-string err))
+                             errors)
+                       nil)))
+          (setq res (nconc res (copy-sequence tmp)))
+          (setq i (1+ i))
+          (progress-reporter-update reporter i file))
+        (progress-reporter-done reporter)))
+    (when errors
+      (with-output-to-temp-buffer "*Javaimp collect classes errors*"
+        (princ javaimp--jar-error-header)
+        (terpri)
+        (dolist (err (nreverse errors))
+          (princ err)
+          (terpri))))
+    res))
 
 (defun javaimp--get-current-source-dirs (module)
   "Return list of source directories for inspection for Java
@@ -557,25 +573,29 @@ If there's no such directive, then the last resort is just
 
 (defun javaimp--get-directory-classes (dir)
   (when (file-accessible-directory-p dir)
-    (when javaimp-verbose
-      (message "Parsing files in %s..." dir))
-    (seq-mapcat #'javaimp--get-file-classes
-                (seq-filter (lambda (file)
-                              (not (file-symlink-p file)))
-                            (directory-files-recursively dir "\\.java\\'")))))
-
-(defun javaimp--get-file-classes (file)
-  (if-let ((buf (get-file-buffer file)))
-      ;; Don't use cache, just collect what we have in buffer
-      (with-current-buffer buf
-        (save-excursion
-          (save-restriction
-            (widen)
-            (javaimp--get-buffer-classes))))
-    (javaimp--get-file-classes-cached
-     file
-     'javaimp-source-file-cache
-     #'javaimp--read-source-classes)))
+    (let ((sources (seq-filter (lambda (f)
+                                 (not (file-symlink-p f)))
+                               (directory-files-recursively dir "\\.java\\'")))
+          files buffers)
+      (dolist (s sources)
+        (if-let ((buf (get-file-buffer s)))
+            (push buf buffers)
+          (push s files)))
+      (append
+       ;; Read files
+       (javaimp--collect-classes
+        'javaimp-source-file-cache #'javaimp--read-source-classes files)
+       ;; Read buffers.  Don't use cache, just collect what we have in
+       ;; buffer.
+       (with-delayed-message
+           (1 (format "Reading %d buffers..." (length buffers)))
+         (seq-mapcat (lambda (buf)
+                       (with-current-buffer buf
+                         (save-excursion
+                           (save-restriction
+                             (widen)
+                             (javaimp--get-buffer-classes)))))
+                     buffers))))))
 
 (defun javaimp--read-source-classes (file)
   (with-temp-buffer
@@ -762,8 +782,9 @@ in a major mode hook."
                             (javaimp-parse-get-interface-abstract-methods))))
     (mapcar
      (lambda (top-class)
-       (message "Building tree for top-level class-like scope: %s"
-                (javaimp-scope-name top-class))
+       (when javaimp-verbose
+         (message "Building tree for top-level class-like scope: %s"
+                  (javaimp-scope-name top-class)))
        (javaimp-tree-build top-class
                            (append methods
                                    classes