feature/tree-sitter af288d813b 6/8: Rework tree-sitter font-lock

[Yuan Fu]

branch: feature/tree-sitter
commit af288d813b3c31a528ddcbdce08f497855a56e94
Author: Yuan Fu <casouri@gmail.com>
Commit: Yuan Fu <casouri@gmail.com>

    Rework tree-sitter font-lock
    
    Remove :toggle and :level, add :feature.
    
    * lisp/progmodes/js.el (js--treesit-settings): Add :feature.
    (js--treesit-enable): Set treesit-font-lock-feature-list.
    * lisp/progmodes/python.el: Replace :level with :feature.
    (python-mode): Set treesit-font-lock-feature-list.
    * lisp/treesit.el (treesit-font-lock-feature-list): New variable.
    (treesit-font-lock-settings): Change format.
    (treesit-font-lock-rules): Remove :toggle and :level, add :feature.
    (treesit-font-lock-recompute-features): New function.
    (treesit-font-lock-fontify-region): Change to work with the new
    format.
    (treesit-font-lock-enable): Add call to
    treesit-font-lock-recompute-features.  And improve the font-lock-mode
    workaround.
---
 lisp/progmodes/js.el     |   4 +-
 lisp/progmodes/python.el |  10 +--
 lisp/treesit.el          | 165 ++++++++++++++++++++++++-----------------------
 3 files changed, 93 insertions(+), 86 deletions(-)

diff --git a/lisp/progmodes/js.el b/lisp/progmodes/js.el
index 1b82e204b9..18499a466a 100644
--- a/lisp/progmodes/js.el
+++ b/lisp/progmodes/js.el
@@ -3459,6 +3459,7 @@ indentation, which-function and movement functions."
 (defvar js--treesit-settings
   (treesit-font-lock-rules
    :language 'javascript
+   :feature 'basic
    :override t
    `(;; Everything overrides template string.
      (template_string) @font-lock-string-face
@@ -3637,8 +3638,9 @@ ARG is the same as in `end-of-defun."
   (setq-local beginning-of-defun-function #'js--treesit-beginning-of-defun)
   (setq-local end-of-defun-function #'js--treesit-end-of-defun)
 
-  (setq-local font-lock-defaults '(nil t))
+  (setq-local font-lock-keywords-only t)
   (setq-local treesit-font-lock-settings js--treesit-settings)
+  (setq-local treesit-font-lock-feature-list '((basic)))
 
   (add-hook 'which-func-functions #'js-treesit-current-defun nil t)
 
diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index a0466707e6..c48f9e95e9 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -1032,7 +1032,7 @@ Do not fontify the initial f for f-strings."
 (defvar python--treesit-settings
   (treesit-font-lock-rules
    :language 'python
-   :level 1
+   :feature 'basic
    '(;; Queries for def and class.
      (function_definition
       name: (identifier) @font-lock-function-name-face)
@@ -1048,7 +1048,7 @@ Do not fontify the initial f for f-strings."
       (:match "^\"\"\"" @font-lock-doc-face))
      (interpolation (identifier) @font-lock-variable-name-face))
    :language 'python
-   :level 2
+   :feature 'moderate
    :override t
    `(;; Keywords, builtins, and constants.
      [,@python--treesit-keywords] @font-lock-keyword-face
@@ -1066,7 +1066,7 @@ Do not fontify the initial f for f-strings."
 
      [(true) (false) (none)] @font-lock-constant-face)
    :language 'python
-   :level 3
+   :feature 'elaborate
    :override t
    `(;; Variable names.
      (assignment left: (identifier)
@@ -6401,7 +6401,9 @@ Add import for undefined name `%s' (empty to skip): "
   (if (and python-use-tree-sitter
            (treesit-can-enable-p))
       (progn
-        (setq-local font-lock-defaults '(nil t))
+        (setq-local font-lock-keywords-only t)
+        (setq-local treesit-font-lock-feature-list
+                    '((basic) (moderate) (elaborate)))
         (setq-local treesit-font-lock-settings
                     python--treesit-settings)
         (treesit-font-lock-enable))
diff --git a/lisp/treesit.el b/lisp/treesit.el
index 1e911df1d9..c8ba3e84e6 100644
--- a/lisp/treesit.el
+++ b/lisp/treesit.el
@@ -284,6 +284,28 @@ in-order.  START and END are passed to each range 
function."
               "Generic tree-sitter font-lock error"
               'treesit-error)
 
+(defvar-local treesit-font-lock-feature-list nil
+  "A list of lists of feature symbols.
+
+Each sublist represents a decoration level.
+`font-lock-maximum-decoration' controls which levels are
+activated.
+
+Inside each sublist are feature symbols, which corresponds to the
+:feature value of a query defined in `treesit-font-lock-rules'.
+Removing a feature symbol from this list disables the
+corresponding query during font-lock.
+
+Common feature names (for general programming language) include
+function-name, type, variable-name (LHS of assignments), builtin,
+constant, keyword, string-interpolation, comment, doc, string,
+operator, preprocessor, escape-sequence, key (in key-value
+pairs).  Major modes are free to subdivide or extend on these
+common features.
+
+For changes to this variable to take effect, run
+`treesit-font-lock-recompute-features'.")
+
 (defvar-local treesit-font-lock-settings nil
   "A list of SETTINGs for treesit-based fontification.
 
@@ -292,30 +314,21 @@ should always use `treesit-font-lock-rules' to set this 
variable.
 
 Each SETTING is of form
 
-    (LANGUAGE QUERY OVERRIDE TOGGLE LEVEL)
+    (QUERY ENABLE FEATURE OVERRIDE)
 
-Each SETTING controls one parser (often of different language).
-LANGUAGE is the language symbol.  See Info node `(elisp)Language
-Definitions'.
+QUERY must be a compiled query.  See Info node `(elisp)Pattern
+Matching' for how to write a query and compile it.
 
-QUERY is either a string query, a sexp query, or a compiled
-query.  See Info node `(elisp)Pattern Matching' for how to write
-a query in either string or s-expression form.  When using
-repeatedly, a compiled query is much faster than a string or sexp
-one, so it is recommend to compile your queries if it will be
-used over and over.
+For SETTING to be activated for font-lock, ENABLE must be t.  To
+disable this SETTING, set ENABLE to nil.
+
+FEATURE is the \"feature name\" of the query, users can control
+which features are enabled with `font-lock-maximum-decoration'
+and `treesit-font-lock-feature-list'.
 
 OVERRIDE is the override flag for this query.  Its value can be
 t, nil, append, prepend, keep.  See more in
-`treesit-font-lock-rules'.
-
-TOGGLE should be a variable (symbol) or nil.  The variable's
-value (nil/non-nil) controls whether to activate the query during
-fontification.  If TOGGLE is nil, the query is always activated.
-
-LEVEL is the decoration level of this query or nil.  Decoration
-level is controlled by `font-lock-maximum-decoration'.  If LEVEL
-is nil, the query is always activated.")
+`treesit-font-lock-rules'.")
 
 (defun treesit-font-lock-rules (&rest args)
   "Return a value suitable for `treesit-font-lock-settings'.
@@ -331,14 +344,18 @@ configure the query (and only that query).  For example,
     (treesit-font-lock-rules
      :language \\='javascript
      :override t
-     :enable 'html-fontify-script
+     :feature\\='constant
      \\='((true) @font-lock-constant-face
        (false) @font-lock-constant-face)
      :language \\='html
+     :feature \\='script
      \"(script_element) @font-lock-builtin-face\")
 
-For each QUERY, a :language keyword is required.  Other keywords
-include:
+For each QUERY, a :language keyword and a :feature keyword is
+required. Each query's :feature is a symbol summarizing what does
+the query fontify.  It is used to allow users to enable/disable
+certain features.  See `treesit-font-lock-kind-list' for more.
+Other keywords include:
 
   KEYWORD    VALUE    DESCRIPTION
   :override  nil      If the region already has a face,
@@ -347,19 +364,6 @@ include:
              append   Append the new face to existing ones
              prepend  Prepend the new face to existing ones
              keep     Fill-in regions without an existing face
-  :toggle    <symbol> If non-nil, the value should be a variable.
-                      The value of that variable (non-nil/nil)
-                      activates/deactivates the query during
-                      fontification.
-             nil      Always activate this query.
-  :level     <integer>If non-nil, the value is the decoration
-                      level of this query.
-                      (See `font-lock-maximum-decoration'.)
-             nil      Always activate this query.
-
-Note that a query is applied only when both :toggle and :level
-permit it.  :level is used for global, coarse-grained control,
-whereas :toggle is for local, fine-grained control.
 
 Capture names in QUERY should be face names like
 `font-lock-keyword-face'.  The captured node will be fontified
@@ -375,7 +379,7 @@ ignored.
   (let (;; Tracks the current :language/:override/:toggle/:level value
         ;; that following queries will apply to.
         current-language current-override
-        current-toggle current-level
+        current-feature
         ;; The list this function returns.
         (result nil))
     (while args
@@ -399,21 +403,14 @@ ignored.
                        `((or t nil append prepend keep)
                          ,flag)))
              (setq current-override flag)))
-          (:toggle
+          (:feature
            (let ((var (pop args)))
              (when (or (not (symbolp var))
                        (memq var '(t nil)))
                (signal 'treesit-font-lock-error
-                       `("Value of :toggle should be a variable name"
+                       `("Value of :feature should be a symbol"
                          ,var)))
-             (setq current-toggle var)))
-          (:level
-           (let ((level (pop args)))
-             (when (not (and (integerp level) (> level 0)))
-               (signal 'treesit-font-lock-error
-                       `("Value of :level should be a positive integer"
-                         ,level)))
-             (setq current-level level)))
+             (setq current-feature var)))
           ;; (2) Process query.
           ((pred treesit-query-p)
            (when (null current-language)
@@ -421,21 +418,40 @@ ignored.
                      `("Language unspecified, use :language keyword to specify 
a language for this query" ,token)))
            (if (treesit-compiled-query-p token)
                (push `(,current-language token) result)
-             (push `(,current-language
-                     ,(treesit-query-compile current-language token)
-                     ,current-override
-                     ,current-toggle
-                     ,current-level)
+             (push `(,(treesit-query-compile current-language token)
+                     t
+                     ,current-feature
+                     ,current-override)
                    result))
            ;; Clears any configurations set for this query.
            (setq current-language nil
                  current-override nil
-                 current-toggle nil
-                 current-level nil))
+                 current-feature nil))
           (_ (signal 'treesit-font-lock-error
-                     `("Unexpected value" token))))))
+                     `("Unexpected value" ,token))))))
     (nreverse result)))
 
+(defun treesit-font-lock-recompute-features ()
+  "Enable/disable font-lock settings according to decoration level.
+Sets the ENABLE flag for each setting in
+`treesit-font-lock-settings', according to
+`treesit-font-lock-feature-list' and
+`font-lock-maximum-decoration'."
+  (let* ((level (font-lock-value-in-major-mode
+                 font-lock-maximum-decoration))
+         (features (cl-loop
+                    for idx = 0 then (1+ idx)
+                    for features in treesit-font-lock-feature-list
+                    if (or (eq level t)
+                           (>= level (1+ idx)))
+                    append features)))
+    (cl-loop for idx = 0 then (1+ idx)
+             for setting in treesit-font-lock-settings
+             for feature = (nth 2 setting)
+             ;; Set the ENABLE flag for the setting.
+             do (setf (nth 1 (nth idx treesit-font-lock-settings))
+                      (if (memq feature features) t nil)))))
+
 (defun treesit-font-lock-fontify-region
     (start end &optional loudly)
   "Fontify the region between START and END.
@@ -443,27 +459,16 @@ If LOUDLY is non-nil, message some debugging information."
   (treesit-update-ranges start end)
   (font-lock-unfontify-region start end)
   (dolist (setting treesit-font-lock-settings)
-    (let* ((language (nth 0 setting))
-           (match-pattern (nth 1 setting))
-           (override (nth 2 setting))
-           (toggle (nth 3 setting))
-           (level (nth 4 setting))
-           (parser (treesit-parser-create language)))
-      (when-let ((node (treesit-node-on start end parser))
-                 ;; Only activate this query if both :toggle and
-                 ;; :level permit it.
-                 (activate
-                  (and (or (null toggle)
-                           (symbol-value toggle))
-                       (or (null level)
-                           (pcase (font-lock-value-in-major-mode
-                                   font-lock-maximum-decoration)
-                             ('t t)
-                             ('nil (eq level 1))
-                             (lvl (<= level lvl)))))))
+    (let* ((query (nth 0 setting))
+           (enable (nth 1 setting))
+           (override (nth 3 setting))
+           (language (treesit-query-language query)))
+      (when-let ((node (treesit-node-on start end language))
+                 ;; Only activate if ENABLE flag is t.
+                 (activate (eq t enable)))
         (ignore activate)
         (let ((captures (treesit-query-capture
-                         node match-pattern
+                         node query
                          ;; Specifying the range is important. More
                          ;; often than not, NODE will be the root
                          ;; node, and if we don't specify the range,
@@ -508,17 +513,15 @@ If LOUDLY is non-nil, message some debugging information."
 
 (defun treesit-font-lock-enable ()
   "Enable tree-sitter font-locking for the current buffer."
+  (treesit-font-lock-recompute-features)
   (setq-local font-lock-fontify-region-function
               #'treesit-font-lock-fontify-region)
   ;; If we don't set `font-lock-defaults' to some non-nil value,
-  ;; font-lock doesn't enable properly (the font-lock-mode-internal
-  ;; doesn't run).  See `font-lock-add-keywords'.
-  (when (and font-lock-mode
-             (null font-lock-keywords)
-             (null font-lock-defaults))
-    (font-lock-mode -1)
-    (setq-local font-lock-defaults '(nil t))
-    (font-lock-mode 1)))
+  ;; font-lock doesn't enable properly (`font-lock-mode-internal'
+  ;; doesn't run).  See `font-lock-specified-p'.
+  (when (null font-lock-defaults)
+    (setq font-lock-defaults '(nil)))
+  (font-lock-mode 1))
 
 ;;; Indent