[elpa] externals/wisi 60b8ef1 15/35: Update ada-mode, wisi

[Stefan Monnier]

branch: externals/wisi
commit 60b8ef1190edd4727f8935ef16b477292a417e1b
Author: Stephen Leake <stephen_leake@stephe-leake.org>
Commit: Stephen Leake <stephen_leake@stephe-leake.org>

    Update ada-mode, wisi
    
    * packages/ada-mode: Version 5.1.9.
    * packages/wisi: Version 1.1.2.
---
 NEWS            |  13 ++++
 README          |   2 +-
 wisi-compile.el | 201 +++++++++++++++++++++++++++++---------------------------
 wisi-parse.el   |  43 +++++++-----
 wisi.el         | 135 ++++++++++++++++++++++---------------
 5 files changed, 225 insertions(+), 169 deletions(-)

diff --git a/NEWS b/NEWS
index 17d084a..9e8ad77 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,19 @@ Please send wisi bug reports to bug-gnu-emacs@gnu.org, with
 'wisi' in the subject. If possible, use M-x report-emacs-bug.
 
 
+* wisi 1.1.2
+20 Jan 2016
+
+** wisi-compile no longer requires semantic.
+
+** wisi-parse slightly faster
+
+** minor bug fixes
+
+** wisi-extend-action now takes two args (first last)
+
+** wisi-face-action-1 optional arg no-override is replaced by 
override-no-error.
+
 * wisi 1.1.1
 10 Apr 2015
 
diff --git a/README b/README
index f363c12..098b476 100644
--- a/README
+++ b/README
@@ -1,4 +1,4 @@
-Emacs wisi package 1.1.1
+Emacs wisi package 1.1.2
 
 The wisi package provides utilities for using generalized LALR parsers
 to do indentation and navigation. See ada-mode for an example of its
diff --git a/wisi-compile.el b/wisi-compile.el
index 463fb01..0251133 100644
--- a/wisi-compile.el
+++ b/wisi-compile.el
@@ -1,6 +1,6 @@
-;;; Grammar compiler for the wisent LALR parser, integrating Wisi OpenToken 
output.  -*- lexical-binding:t -*-
+;; wisi-compile.el --- Grammar compiler for the wisi parser, integrating Wisi 
OpenToken output.  -*- lexical-binding:t -*-
 ;;
-;; Copyright (C) 2012, 2013, 2015 Free Software Foundation, Inc.
+;; Copyright (C) 2012, 2013, 2015, 2016 Free Software Foundation, Inc.
 ;;
 ;; Author: Stephen Leake <stephen_leake@member.fsf.org>
 ;;
@@ -41,33 +41,40 @@
 ;; produces corresponding elisp source code, similar to that
 ;; produced by semantic-grammar-create-package.
 ;;
-;; wisi-compile-grammar (provided here) generate the automaton
-;; structure required by wisi-parse, using functions from
-;; wisent/comp.el
+;; wisi-compile-grammar (provided here) generates the automaton
+;; structure required by wisi-parse
 ;;
 ;;;;
 
-(require 'semantic/wisent/comp)
+(defun wisi-compose-action (value symbol-obarray nonterms)
+  (let* ((nonterm (car value))
+       (index   (cdr value))
+       (symbol (intern-soft (format "%s:%d" nonterm index) symbol-obarray))
+       (rhs (car (nth index (cdr (assoc nonterm nonterms))))))
+    (list nonterm symbol (length rhs))
+    ))
 
-(defun wisi-compose-action (value symbol-array nonterms)
-  (let ((symbol (intern-soft (format "%s:%d" (car value) (cdr value)) 
symbol-array))
-       (prod (car (nth (cdr value) (cdr (assoc (car value) nonterms))))))
-    (if symbol
-       (list (car value) symbol (length prod))
-      (error "%s not in symbol-array" symbol))))
-
-(defun wisi-replace-actions (action symbol-array nonterms)
+(defun wisi-replace-actions (action symbol-obarray nonterms)
   "Replace semantic action symbol names in ACTION with list as defined in 
`wisi-compile-grammar'.
-ACTION is the alist for one state from the grammar; NONTERMS is from the 
grammar.
-Return the new alist."
-  ;; result is (nonterm index action-symbol token-count)
+ACTION is the alist for one state from the grammar, with the form:
+  ((default . error) ITEM ... )
+ITEM is one of:
+reduction  (TOKEN . (NONTERM . INDEX)) where NONTERM . INDEX gives the action 
symbol name.
+shift (TOKEN . STATE)
+shift/reduce conflict (STATE (NONTERM . INDEX))
+reduce/shift conflict ((NONTERM . INDEX) (NONTERM . INDEX))
+
+SYMBOL-OBARRAY contains the action symbols.
+NONTERMS is from the grammar.
+Return the new action alist."
+  ;; result is list of (nonterm index action-symbol token-count)
   (let (result item)
     (while action
      (setq item (pop action))
      (cond
       ((or
        (memq (cdr item) '(error accept))
-       (numberp (cdr item)))
+       (numberp (cdr item))) ;; shift
        (push item result))
 
       ((listp (cdr item))
@@ -76,27 +83,20 @@ Return the new alist."
          ((symbolp (car value))
           ;; reduction
           (push (cons (car item)
-                      (wisi-compose-action value symbol-array nonterms))
+                      (wisi-compose-action value symbol-obarray nonterms))
                 result))
 
          ((integerp (car value))
           ;; shift/reduce conflict
           (push (cons (car item)
                       (list (car value)
-                            (wisi-compose-action (cadr value) symbol-array 
nonterms)))
-                result))
-
-         ((integerp (cadr value))
-          ;; reduce/shift conflict
-          (push (cons (car item)
-                      (list (wisi-compose-action (car value) symbol-array 
nonterms)
-                            (cadr value)))
+                            (wisi-compose-action (cadr value) symbol-obarray 
nonterms)))
                 result))
 
          (t ;; reduce/reduce conflict
           (push (cons (car item)
-                      (list (wisi-compose-action (car value) symbol-array 
nonterms)
-                            (wisi-compose-action (cadr value) symbol-array 
nonterms)))
+                      (list (wisi-compose-action (car value) symbol-obarray 
nonterms)
+                            (wisi-compose-action (cadr value) symbol-obarray 
nonterms)))
                 result))
          )))
 
@@ -106,49 +106,36 @@ Return the new alist."
 
    (reverse result)))
 
-(defun wisi-semantic-action (r rcode tags rlhs)
-  "Define an Elisp function for semantic action at rule R.
-On entry RCODE[R] contains a vector [BODY N (NTERM I)] where BODY
-is the body of the semantic action, N is the number of tokens in
-the production, NTERM is the nonterminal the semantic action
-belongs to, and I is the index of the production and associated
-semantic action in the NTERM rule.  Returns the semantic action
-symbol, which is interned in RCODE[0].
-
-The semantic action function accepts one argument, the list of
-tokens to be reduced. It returns nil; it is called for the user
-side-effects only."
+(defun wisi-semantic-action (form nonterm iactn symbol-obarray)
+  "Define an Elisp semantic action function for a production, interned in 
SYMBOL-OBARRAY.
+FORM is the body of the semantic action.
+NONTERM is the nonterminal left hand side.
+IACTN is the index of the production in the NTERM rule.
+
+The semantic action function accepts two arguments;
+- $nterm      : the nonterminal
+- wisi-tokens : the list of tokens to be reduced.
+
+It returns nil; it is called for the semantic side-effects only."
   ;; based on comp.el wisent-semantic-action
-  (let* ((actn (aref rcode r))
-        (n    (aref actn 1))         ; number of tokens in production
-        (name (apply 'format "%s:%d" (aref actn 2)))
-        (form (aref actn 0))
-        (action-symbol (intern name (aref rcode 0))))
+  (let* ((name (format "%s:%d" nonterm iactn))
+        (action-symbol (intern name symbol-obarray)))
 
     (fset action-symbol
-         `(lambda (wisi-tokens)
-            (let* (($nterm ',(aref tags (aref rlhs r)))
-                   ($1 nil));; wisent-parse-nonterminals defines a default 
body of $1 for empty actions
-              ,form
-              nil)))
-
-    (list (car (aref actn 2)) action-symbol n)))
+         `(lambda ($nterm wisi-tokens)
+            ,form
+            nil))))
 
 (defun wisi-compile-grammar (grammar)
-  ;; FIXME: This docstring is full of ambiguities making it unclear whether
-  ;; we're talking for example about data that includes the symbol `nonterm' as
-  ;; opposed to data that includes some non terminal object we denote
-  ;; with the meta-variable "nonterm".
-  ;; The convention in Elisp's docstrings is to use all-caps for metavariables
-  ;; (and `...' quoting as opposed to the '... quoting used below in a few
-  ;; spots).
   "Compile the LALR(1) GRAMMAR; return the automaton for wisi-parse.
 GRAMMAR is a list TERMINALS NONTERMS ACTIONS GOTOS, where:
 
 TERMINALS is a list of terminal token symbols.
 
 NONTERMS is a list of productions; each production is a
-list (nonterm (tokens action) ...) where `action' is any lisp form.
+list (nonterm (tokens semantic-action) ...) where `semantic-action' is
+any lisp form. The set of (tokens semantic-action) are the right hand
+sides; nonterm is the left hand side.
 
 ACTIONS is an array indexed by parser state, of alists indexed by
 terminal tokens. The value of each item in the alists is one of:
@@ -162,7 +149,6 @@ integer - shift; gives new state
 '(nonterm . index) - reduce by nonterm production index.
 
 '(integer (nonterm . index)) - a shift/reduce conflict
-'((nonterm . index) integer) - a reduce/shift conflict
 '((nonterm . index) (nonterm . index)) - a reduce/reduce conflict
 
 The first item in the alist must have the key 'default (not a
@@ -173,54 +159,77 @@ GOTOS is an array indexed by parser state, of alists 
giving the
 new state after a reduce for each nonterminal legal in that
 state.
 
-The automaton is an array with 3 elements:
+The automaton is an array [parser-actions gotos symbol-obarray]:
 
-parser-actions is a copy of the input ACTIONS, with reduction
-actions replaced by a list (NONTERM ACTION-SYMBOL TOKEN-COUNT),
-where NONTERM is a symbol from NONTERMS, and is the
-non-terminal to reduce to, TOKEN-COUNT is the number of tokens in
-the reduction, ACTION-SYMBOL is nil if there is no user action,
-or a symbol from semantic-actions (below).
+- parser-actions is a copy of the input ACTIONS, with semantic
+actions replaced by a list (nonterm action-symbol token-count),
+where:
 
-gotos is a copy of GOTOS.
+-- nonterm is a symbol from NONTERMS, and is the non-terminal to
+reduce to
 
-semantic-actions is an obarray containing functions that
-implement the user action for each nonterminal; the function
-names have the format nonterm:index."
-  (defvar nrules) (defvar ptable) (defvar rcode) (defvar rlhs) (defvar tags)
-  (defvar token-list) (defvar var-list)
-  (let (nrules ptable rcode rlhs tags token-list var-list)
-    (wisent-parse-grammar;; set global vars used by wisent-semantic-action
-     (cons
-      (nth 0 grammar);; TOKENS
-      (cons nil ;; ASSOCS
-           (nth 1 grammar));; NONTERMS
-      ))
-
-    (aset rcode 0 (make-vector 13 0));; obarray for semantic actions
-
-    ;; create semantic action functions, interned in rcode[0]
-    (let* ((i 1))
-      (while (<= i nrules)
-       (wisi-semantic-action i rcode tags rlhs)
-       (setq i (1+ i)))
-      )
+-- token-count is the number of tokens in the reduction,
 
-    ;; replace semantic actions in ACTIONS with symbols from symbol-array
+-- action-symbol is nil if there is no semantic action, or a
+symbol interned in symbol-obarray
+
+- gotos is a copy of GOTOS.
+
+- symbol-obarray is an obarray containing functions that
+implement the semantic action for each nonterminal; the function
+names have the format nonterm:index."
+  ;; We store named symbols for semantic actions, not just lambda
+  ;; functions, so we have a name for debug trace.
+  ;;
+  ;; FIXME: TERMINALS is not used. Eliminating it requires decoupling
+  ;; from OpenToken; we'll do that in the move to FastToken.
+  ;;
+  ;; FIXME: eliminate use of semantic-lex-* in *-wy.el. Similarly
+  ;; requires decoupling from OpenToken
+  ;;
+  ;; FIXME: can eliminate obarray? We don't need the obarray to
+  ;; avoid garbage collection of the symbols; they are all referenced in the 
compiled grammar.
+  ;; But each semantic action function has to be defined (and byte-compiled?) 
somewhere?
+  ;;     currently actions are _not_ byte-compiled; wisi-compile-grammar is 
run at load time
+  ;;     need 'eval-when-compile' to byte-compile them?
+  ;;     can't byte-compile obarray?
+
+  (let ((defs (nth 1 grammar))
+       (symbol-obarray (make-vector 13 0));; for parse actions
+       def nonterm rhs-list rule
+       semantic-action index)
+
+    (while defs
+      (setq def      (car defs)
+            defs     (cdr defs)
+            nonterm  (car def)
+            rhs-list (cdr def)
+            index    0)
+      (while rhs-list
+        (setq rule            (car rhs-list)
+              rhs-list        (cdr rhs-list)
+              semantic-action (cadr rule))
+
+       (when semantic-action
+         (wisi-semantic-action semantic-action nonterm index symbol-obarray))
+
+       (setq index (1+ index))
+       ))
+
+    ;; replace semantic actions in ACTIONS with symbols from symbol-obarray
     (let ((nactions (length (nth 2 grammar)))
          (actions (nth 2 grammar))
-         (symbol-array (aref rcode 0))
          (i 0))
       (while (< i nactions)
        (aset actions i
-             (wisi-replace-actions (aref actions i) symbol-array (nth 1 
grammar)))
+             (wisi-replace-actions (aref actions i) symbol-obarray (nth 1 
grammar)))
        (setq i (1+ i)))
       (vector
        actions
        (nth 3 grammar)
-       symbol-array)
+       symbol-obarray)
       )))
 
 (provide 'wisi-compile)
 
-;;; wisi-compile.el ends here
+;;;; end of file
diff --git a/wisi-parse.el b/wisi-parse.el
index 4fa300e..b9da937 100644
--- a/wisi-parse.el
+++ b/wisi-parse.el
@@ -111,6 +111,9 @@ point at which that max was spawned.")
   list (symbol text start . end), where `symbol' is the terminal
   symbol, `text' is the token string, `start . end' is the range
   in the buffer."
+
+  ;; FIXME: (aref automaton 3) is the obarray storing the semantic actions;
+  ;; not used here (see related FIXME in wisi-compile)
   (let* ((actions (aref automaton 0))
         (gotos   (aref automaton 1))
         (parser-states ;; vector of parallel parser states
@@ -148,8 +151,8 @@ point at which that max was spawned.")
                        (let ((state (aref (wisi-parser-state-stack 
parser-state)
                                           (wisi-parser-state-sp 
parser-state))))
                          (wisi-error-msg (concat "too many parallel parsers 
required in grammar state %d;"
-                                                 " simplify grammar, or 
increase `wisi-parse-max-parallel'"
-                                                 state)))))
+                                                 " simplify grammar, or 
increase `wisi-parse-max-parallel'")
+                                                 state))))
 
              (let ((j (wisi-free-parser parser-states)))
                (cond
@@ -381,7 +384,7 @@ nil, 'shift, or 'accept."
     result)
   )
 
-(defun wisi-parse-exec-action (func tokens)
+(defun wisi-parse-exec-action (func nonterm tokens)
   "Execute action if all tokens past wisi-cache-max."
   ;; We don't execute actions if all tokens are before wisi-cache-max,
   ;; because later actions can update existing caches, and if the
@@ -392,7 +395,7 @@ nil, 'shift, or 'accept."
   (if (< 0 (length tokens))
       (if (>= (wisi-parse-max-pos tokens) wisi-cache-max)
 
-         (funcall func tokens)
+         (funcall func nonterm tokens)
 
        (when (> wisi-debug 1)
          (message "... action skipped; before wisi-cache-max %d" 
wisi-cache-max)))
@@ -407,7 +410,7 @@ nil, 'shift, or 'accept."
     (when (> wisi-debug 1) (message "%s" (car pending)))
 
     (let ((func-args (pop pending)))
-      (wisi-parse-exec-action (car func-args) (cadr func-args)))
+      (wisi-parse-exec-action (nth 0 func-args) (nth 1 func-args) (cl-caddr 
func-args)))
     ))
 
 (defun wisi-parse-1 (token parser-state pendingp actions gotos)
@@ -508,7 +511,7 @@ the first and last tokens of the nonterminal."
   "Reduce PARSER-STATE.stack, and execute or pend ACTION."
   (let* ((stack (wisi-parser-state-stack parser-state)); reference
         (sp (wisi-parser-state-sp parser-state)); copy
-        (token-count (or (nth 2 action) 0))
+        (token-count (nth 2 action))
         (nonterm (nth 0 action))
         (nonterm-region (when (> token-count 0)
                           (wisi-nonterm-bounds stack (- sp (* 2 (1- 
token-count)) 1) (1- sp))))
@@ -519,25 +522,29 @@ the first and last tokens of the nonterminal."
     (when (not new-state)
       (error "no goto for %s %d" nonterm post-reduce-state))
 
-    (dotimes (i token-count)
-      (aset tokens (- token-count i 1) (aref stack (- sp (* 2 i) 1))))
+    (when (nth 1 action)
+      ;; don't need wisi-tokens for a null user action
+      (dotimes (i token-count)
+       (aset tokens (- token-count i 1) (aref stack (- sp (* 2 i) 1)))))
 
     (setq sp (+ 2 (- sp (* 2 token-count))))
     (aset stack (1- sp) (cons nonterm nonterm-region))
     (aset stack sp new-state)
     (setf (wisi-parser-state-sp parser-state) sp)
 
-    (if pendingp
-       (if (wisi-parser-state-pending parser-state)
+    (when (nth 1 action)
+      ;; nothing to do for a null user action
+      (if pendingp
+         (if (wisi-parser-state-pending parser-state)
+             (setf (wisi-parser-state-pending parser-state)
+                   (append (wisi-parser-state-pending parser-state)
+                           (list (list (nth 1 action) nonterm tokens))))
            (setf (wisi-parser-state-pending parser-state)
-                 (append (wisi-parser-state-pending parser-state)
-                         (list (list (nth 1 action) tokens))))
-         (setf (wisi-parser-state-pending parser-state)
-               (list (list (nth 1 action) tokens))))
-
-      ;; Not pending.
-      (wisi-parse-exec-action (nth 1 action) tokens)
-      )
+                 (list (list (nth 1 action) nonterm tokens))))
+
+       ;; Not pending.
+       (wisi-parse-exec-action (nth 1 action) nonterm tokens)
+       ))
     ))
 
 (provide 'wisi-parse)
diff --git a/wisi.el b/wisi.el
index a748f62..0b29b3e 100644
--- a/wisi.el
+++ b/wisi.el
@@ -1,13 +1,13 @@
 ;;; wisi.el --- Utilities for implementing an indentation/navigation engine 
using a generalized LALR parser -*- lexical-binding:t -*-
 ;;
-;; Copyright (C) 2012 - 2015  Free Software Foundation, Inc.
+;; Copyright (C) 2012 - 2016  Free Software Foundation, Inc.
 ;;
 ;; Author: Stephen Leake <stephen_leake@member.fsf.org>
 ;; Maintainer: Stephen Leake <stephen_leake@member.fsf.org>
 ;; Keywords: parser
 ;;  indentation
 ;;  navigation
-;; Version: 1.1.1
+;; Version: 1.1.2
 ;; package-requires: ((cl-lib "0.4") (emacs "24.2"))
 ;; URL: http://stephe-leake.org/emacs/ada-mode/emacs-ada-mode.html
 ;;
@@ -158,6 +158,7 @@
 ;; an appropriate end-point for syntax-propertize, other than
 ;; point-max. So we call (syntax-propertize point-max) in wisi-setup,
 ;; and also call syntax-propertize in wisi-after-change.
+;; FIXME: no longer needed in Emacs 25? (email from Stefan Monnier)
 ;;
 ;;;; code style
 ;;
@@ -289,10 +290,10 @@ If at end of buffer, returns `wisent-eoi-term'."
          (scan-error
           ;; Something screwed up; we should not get here if
           ;; syntax-propertize works properly.
-          (error "wisi-forward-token: forward-sexp failed %s" err)
+          (signal 'wisi-parse-error (format "wisi-forward-token: forward-sexp 
failed %s" err))
           ))))
 
-     (t ;; assuming word or symbol syntax
+     (t ;; assuming word or symbol syntax; includes numbers
       (skip-syntax-forward "w_'")
       (setq token-text (buffer-substring-no-properties start (point)))
       (setq token-id
@@ -442,6 +443,12 @@ Used in before/after change functions.")
   (wisi-delete-cache after)
   )
 
+;; To see the effects of wisi-before-change, wisi-after-change, you need:
+;; (global-font-lock-mode 0)
+;; (setq jit-lock-functions nil)
+;;
+;; otherwise jit-lock runs and overrides them
+
 (defun wisi-before-change (begin end)
   "For `before-change-functions'."
   ;; begin . end is range of text being deleted
@@ -562,6 +569,8 @@ Used in before/after change functions.")
        (setq begin-state (syntax-ppss begin))
        (setq end-state (syntax-ppss end))
        ;; syntax-ppss has moved point to "end".
+
+       ;; extend fontification over new text,
        (skip-syntax-forward "w_")
        (setq word-end (point))
        (goto-char begin)
@@ -592,7 +601,7 @@ Used in before/after change functions.")
          ;; no easy way to tell if there is intervening non-string
          (setq need-invalidate nil))
 
-        ((or
+        ((and
           (nth 4 begin-state)
           (nth 4 end-state)); in comment
          ;; no easy way to detect intervening code
@@ -644,7 +653,8 @@ If accessing cache at a marker for a token as set by 
`wisi-cache-tokens', POS mu
 
 (defun wisi-goto-error ()
   "Move point to position in last error message (if any)."
-  (when (string-match ":\\([0-9]+\\):\\([0-9]+\\):" wisi-parse-error-msg)
+  (when (and wisi-parse-error-msg
+            (string-match ":\\([0-9]+\\):\\([0-9]+\\):" wisi-parse-error-msg))
     (let ((line (string-to-number (match-string 1 wisi-parse-error-msg)))
          (col (string-to-number (match-string 2 wisi-parse-error-msg))))
       (push-mark)
@@ -679,7 +689,9 @@ If accessing cache at a marker for a token as set by 
`wisi-cache-tokens', POS mu
       (when (> wisi-debug 0)
        (message msg))
 
+      ;; Don't keep retrying failed parse until text changes again.
       (setq wisi-parse-try nil)
+
       (setq wisi-parse-error-msg nil)
       (setq wisi-end-caches nil)
 
@@ -762,8 +774,10 @@ delete from `wisi-end-caches'."
       )))
 
 (defvar wisi-tokens nil)
-;; keep byte-compiler happy; `wisi-tokens' is bound in action created
-;; by wisi-semantic-action
+(defvar $nterm nil)
+;; keep byte-compiler happy; `wisi-tokens' and `$nterm' are bound in
+;; action created by wisi-semantic-action, and in module parser.
+;; FIXME: $nterm should have wisi- prefix
 
 (defun wisi-statement-action (pairs)
   "Cache information in text properties of tokens.
@@ -842,7 +856,7 @@ grammar action as:
                     (1+ (car region))
                     'wisi-cache
                     (wisi-cache-create
-                     :nonterm    $nterm;; $nterm defined in 
wisi-semantic-action
+                     :nonterm    $nterm
                      :token      token
                      :last       (- (cdr region) (car region))
                      :class      (or override-start class)
@@ -856,8 +870,7 @@ grammar action as:
                (when first-item
                  (setq first-item nil)
                  (when (or override-start
-                           ;; FIXME: why block-middle here?
-                           (memq class '(block-middle block-start 
statement-start)))
+                           (memq class '(block-start statement-start)))
                    (setq override-start nil)
                    (setq first-keyword-mark mark)))
 
@@ -1009,57 +1022,50 @@ vector [number class token_id class token_id ...]:
          ))
       )))
 
-(defun wisi-extend-action (number)
-  "Extend text of cache at token NUMBER to cover all of token NUMBER.
-Also override token with new token."
-  (let* ((token-region (aref wisi-tokens (1- number)));; wisi-tokens is 
let-bound in wisi-parse-reduce
-        (token (car token-region))
-        (region (cdr token-region))
+(defun wisi-extend-action (first last)
+  "Extend text of cache at token FIRST to cover all tokens thru LAST."
+  (let* ((first-region (cdr (aref wisi-tokens (1- first))));; wisi-tokens is 
let-bound in wisi-parse-reduce
+        (last-region (cdr (aref wisi-tokens (1- last))))
        cache)
 
-    (when region
-      (setq cache (wisi-get-cache (car region)))
-      (setf (wisi-cache-last cache) (- (cdr region) (car region)))
-      (setf (wisi-cache-token cache) token)
+    (when first-region
+      (setq cache (wisi-get-cache (car first-region)))
+      (setf (wisi-cache-last cache) (- (cdr last-region) (car first-region)))
       )
     ))
 
-(defun wisi-face-action-1 (face region &optional no-override)
-  "Apply FACE to REGION. If NO-OVERRIDE is non-nil, don't override existing 
face."
+(defun wisi-face-action-1 (face region &optional override-no-error)
+  "Apply FACE to REGION.
+If OVERRIDE-NO-ERROR is non-nil, don't report an error for overriding an 
existing face."
   (when region
     ;; We allow overriding a face property, because we don't want to
     ;; delete them in wisi-invalidate (see comments there). On the
     ;; other hand, it can be an error, so keep this debug
-    ;; code. However, note that font-lock-face properties must be
-    ;; removed first, or the buffer must be fresh (never parsed).
+    ;; code. However, to validly report errors, note that
+    ;; font-lock-face properties must be removed first, or the buffer
+    ;; must be fresh (never parsed), and wisi-debug must be > 1.
     ;;
-    ;; Grammar sets no-override when a higher-level production might
-    ;; override a face in a lower-level production; that's not an
-    ;; error.
-    (let (cur-face
-         (do-set t))
-      (when (or no-override
-               (> wisi-debug 1))
-       (setq cur-face (get-text-property (car region) 'font-lock-face))
-       (if cur-face
-           (if no-override
-               (setq do-set nil)
-             (message "%s:%d overriding face %s with %s on '%s'"
+    ;; Grammar sets override-no-error when a higher-level production might
+    ;; override a face in a lower-level production.
+    (when (> wisi-debug 1)
+      (let ((cur-face (get-text-property (car region) 'font-lock-face)))
+       (when cur-face
+         (unless override-no-error
+           (message "%s:%d overriding face %s with %s on '%s'"
                     (buffer-file-name)
                     (line-number-at-pos (car region))
                     face
                     cur-face
                     (buffer-substring-no-properties (car region) (cdr 
region))))
 
-         ))
-      (when do-set
-       (with-silent-modifications
-         (add-text-properties
-          (car region) (cdr region)
-          (list
-           'font-lock-face face
-           'fontified t))))
-    )))
+         )))
+    (with-silent-modifications
+      (add-text-properties
+       (car region) (cdr region)
+       (list
+       'font-lock-face face
+       'fontified t)))
+    ))
 
 (defun wisi-face-action (pairs &optional no-override)
   "Cache face information in text properties of tokens.
@@ -1361,14 +1367,28 @@ Return start cache."
 (defun wisi-comment-indent ()
   "For `comment-indent-function'. Indent single line comment to
 the comment on the previous line."
-  ;; This should only be called by comment-indent-new-line or
-  ;; fill-comment-paragraph, so there will be a preceding comment line
-  ;; that we can trust.
-  (save-excursion
-    (forward-comment -1)
-    (if (looking-at comment-start)
-       (current-column)
-      (error "wisi-comment-indent called after non-comment"))))
+  (or
+   (save-excursion
+     (forward-comment -1)
+     (when (looking-at comment-start)
+       ;; There is a preceding comment line.
+       (current-column)))
+
+   ;; Probably called from `comment-indent'; either to insert a new
+   ;; comment, or to indent the first line of an existing one.  In
+   ;; either case, the comment may be after code on the same line.
+   (save-excursion
+     (let ((start-col (current-column)))
+       (back-to-indentation)
+       (if (looking-at comment-start)
+          ;; An existing comment alone on a line. Return nil, so
+          ;; `comment-indent' will call `indent-according-to-mode'
+          nil
+
+        ;; A comment after code on the same line; point was at the
+        ;; comment start, so assume it is already correct.
+        start-col)))
+   ))
 
 (defun wisi-indent-current (offset)
   "Return indentation OFFSET relative to indentation of current line."
@@ -1475,6 +1495,13 @@ correct. Must leave point at indentation of current 
line.")
   (wisi-invalidate-cache)
   (wisi-validate-cache (point-max)))
 
+(defun wisi-lex-buffer ()
+  (interactive)
+  (syntax-propertize (point-max))
+  (goto-char (point-min))
+  (while (not (eq wisent-eoi-term (car (wisi-forward-token)))))
+  )
+
 (defun wisi-show-cache ()
   "Show cache at point."
   (interactive)