[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[elpa] externals/phps-mode f9e5d6233f 19/29: Incremental lexer working s
|
From: |
Christian Johansson |
|
Subject: |
[elpa] externals/phps-mode f9e5d6233f 19/29: Incremental lexer working somewhat again |
|
Date: |
Thu, 15 Feb 2024 03:14:37 -0500 (EST) |
branch: externals/phps-mode
commit f9e5d6233f09b59432b2e112e046ff074b502526
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>
Incremental lexer working somewhat again
---
phps-mode-lex-analyzer.el | 470 +++++++------------------------------
phps-mode-lexer.el | 39 ++-
phps-mode-test.el | 71 +++---
test/phps-mode-test-ast.el | 1 +
test/phps-mode-test-integration.el | 1 -
test/phps-mode-test-parser.el | 1 +
6 files changed, 154 insertions(+), 429 deletions(-)
diff --git a/phps-mode-lex-analyzer.el b/phps-mode-lex-analyzer.el
index a664e06368..0ec26fa5e0 100644
--- a/phps-mode-lex-analyzer.el
+++ b/phps-mode-lex-analyzer.el
@@ -70,23 +70,8 @@
(defvar-local phps-mode-lex-analyzer--tokens nil
"Latest tokens.")
-(defvar-local phps-mode-lex-analyzer--state nil
- "Latest state.")
-
-(defvar-local phps-mode-lex-analyzer--state-stack nil
- "Latest state-stack.")
-
-(defvar-local phps-mode-lex-analyzer--states nil
- "History of state, heredoc-label, stack-stack and heredoc label stack.")
-
-(defvar-local phps-mode-lex-analyzer--heredoc-label nil
- "Latest Heredoc label.")
-
-(defvar-local phps-mode-lex-analyzer--heredoc-label-stack nil
- "Latest Heredoc label-stack.")
-
-(defvar-local phps-mode-lex-analyzer--nest-location-stack nil
- "Nest location stack.")
+(defvar-local phps-mode-lex-analyzer--cache nil
+ "Cache of last lex.")
(defvar-local phps-mode-lex-analyzer--parse-trail nil
"Valid parse trail or nil.")
@@ -106,18 +91,13 @@
(setq phps-mode-lex-analyzer--allow-after-change-p t)
(setq phps-mode-lex-analyzer--ast nil)
(setq phps-mode-lex-analyzer--bookkeeping nil)
+ (setq phps-mode-lex-analyzer--cache nil)
(setq phps-mode-lex-analyzer--change-min nil)
- (setq phps-mode-lex-analyzer--heredoc-label nil)
- (setq phps-mode-lex-analyzer--heredoc-label-stack nil)
(setq phps-mode-lex-analyzer--idle-timer nil)
(setq phps-mode-lex-analyzer--imenu nil)
- (setq phps-mode-lex-analyzer--nest-location-stack nil)
(setq phps-mode-lex-analyzer--parse-error nil)
(setq phps-mode-lex-analyzer--parse-trail nil)
(setq phps-mode-lex-analyzer--processed-buffer-p nil)
- (setq phps-mode-lex-analyzer--state nil)
- (setq phps-mode-lex-analyzer--state-stack nil)
- (setq phps-mode-lex-analyzer--states nil)
(setq phps-mode-lex-analyzer--tokens nil)
(when clear-existing
(phps-mode-serial--kill-active
@@ -229,13 +209,6 @@ ALLOW-CACHE-READ and ALLOW-CACHE-WRITE."
buffer-contents
nil
nil
- nil
- nil
- nil
- nil
- nil
- nil
- nil
buffer-file-name
allow-cache-read
allow-cache-write))
@@ -247,20 +220,15 @@ ALLOW-CACHE-READ and ALLOW-CACHE-WRITE."
;; Move variables into this buffers local variables
(setq phps-mode-lex-analyzer--tokens (nth 0 lex-result))
- (setq phps-mode-lex-analyzer--states (nth 1 lex-result))
- (setq phps-mode-lex-analyzer--state (nth 2 lex-result))
- (setq phps-mode-lex-analyzer--state-stack (nth 3 lex-result))
- (setq phps-mode-lex-analyzer--heredoc-label (nth 4 lex-result))
- (setq phps-mode-lex-analyzer--heredoc-label-stack (nth 5
lex-result))
- (setq phps-mode-lex-analyzer--nest-location-stack (nth 6
lex-result))
- (setq phps-mode-lex-analyzer--parse-trail (nth 7 lex-result))
- (setq phps-mode-lex-analyzer--parse-error (nth 8 lex-result))
- (setq phps-mode-lex-analyzer--ast (nth 9 lex-result))
- (setq phps-mode-lex-analyzer--bookkeeping (nth 10 lex-result))
- (setq phps-mode-lex-analyzer--imenu (nth 11 lex-result))
- (setq phps-mode-lex-analyzer--symbol-table (nth 12 lex-result))
- (setq timer-elapsed-lexer (nth 13 lex-result))
- (setq timer-elapsed-parser (nth 14 lex-result))
+ (setq phps-mode-lex-analyzer--cache (nth 1 lex-result))
+ (setq phps-mode-lex-analyzer--parse-trail (nth 2 lex-result))
+ (setq phps-mode-lex-analyzer--parse-error (nth 3 lex-result))
+ (setq phps-mode-lex-analyzer--ast (nth 4 lex-result))
+ (setq phps-mode-lex-analyzer--bookkeeping (nth 5 lex-result))
+ (setq phps-mode-lex-analyzer--imenu (nth 6 lex-result))
+ (setq phps-mode-lex-analyzer--symbol-table (nth 7 lex-result))
+ (setq timer-elapsed-lexer (nth 8 lex-result))
+ (setq timer-elapsed-parser (nth 9 lex-result))
(setq phps-mode-lex-analyzer--processed-buffer-p t)
(phps-mode-lex-analyzer--reset-imenu)
@@ -417,24 +385,20 @@ ALLOW-CACHE-READ and ALLOW-CACHE-WRITE."
(defun phps-mode-lex-analyzer--incremental-lex-string
(buffer-name
buffer-contents
+ cache
incremental-start-new-buffer
- point-max
- head-states
- incremental-state
- incremental-state-stack
- incremental-heredoc-label
- incremental-heredoc-label-stack
- incremental-nest-location-stack
- head-tokens
&optional
force-synchronous
filename
allow-cache-write)
"Incremental lex region."
- (let* ((async (and (boundp 'phps-mode-async-process)
- phps-mode-async-process))
- (async-by-process (and (boundp 'phps-mode-async-process-using-async-el)
- phps-mode-async-process-using-async-el))
+ (let* ((async
+ (and
+ (boundp 'phps-mode-async-process)
+ phps-mode-async-process))
+ (async-by-process
+ (and (boundp 'phps-mode-async-process-using-async-el)
+ phps-mode-async-process-using-async-el))
(timer-start)
(timer-elapsed-lexer)
(timer-elapsed-parser)
@@ -462,14 +426,7 @@ ALLOW-CACHE-READ and ALLOW-CACHE-WRITE."
(phps-mode-lex-analyzer--lex-string
buffer-contents
incremental-start-new-buffer
- point-max
- head-states
- incremental-state
- incremental-state-stack
- incremental-heredoc-label
- incremental-heredoc-label-stack
- incremental-nest-location-stack
- head-tokens
+ cache
filename
nil
allow-cache-write))
@@ -481,20 +438,15 @@ ALLOW-CACHE-READ and ALLOW-CACHE-WRITE."
(let ((buffer-point-max (point-max)))
(setq phps-mode-lex-analyzer--tokens (nth 0 lex-result))
- (setq phps-mode-lex-analyzer--states (nth 1 lex-result))
- (setq phps-mode-lex-analyzer--state (nth 2 lex-result))
- (setq phps-mode-lex-analyzer--state-stack (nth 3 lex-result))
- (setq phps-mode-lex-analyzer--heredoc-label (nth 4 lex-result))
- (setq phps-mode-lex-analyzer--heredoc-label-stack (nth 5
lex-result))
- (setq phps-mode-lex-analyzer--nest-location-stack (nth 6
lex-result))
- (setq phps-mode-lex-analyzer--parse-trail (nth 7 lex-result))
- (setq phps-mode-lex-analyzer--parse-error (nth 8 lex-result))
- (setq phps-mode-lex-analyzer--ast (nth 9 lex-result))
- (setq phps-mode-lex-analyzer--bookkeeping (nth 10 lex-result))
- (setq phps-mode-lex-analyzer--imenu (nth 11 lex-result))
- (setq phps-mode-lex-analyzer--symbol-table (nth 12 lex-result))
- (setq timer-elapsed-lexer (nth 13 lex-result))
- (setq timer-elapsed-parser (nth 14 lex-result))
+ (setq phps-mode-lex-analyzer--cache (nth 1 lex-result))
+ (setq phps-mode-lex-analyzer--parse-trail (nth 2 lex-result))
+ (setq phps-mode-lex-analyzer--parse-error (nth 3 lex-result))
+ (setq phps-mode-lex-analyzer--ast (nth 4 lex-result))
+ (setq phps-mode-lex-analyzer--bookkeeping (nth 5 lex-result))
+ (setq phps-mode-lex-analyzer--imenu (nth 6 lex-result))
+ (setq phps-mode-lex-analyzer--symbol-table (nth 7 lex-result))
+ (setq timer-elapsed-lexer (nth 8 lex-result))
+ (setq timer-elapsed-parser (nth 9 lex-result))
(phps-mode-debug-message
(message
@@ -658,92 +610,6 @@ ALLOW-CACHE-READ and ALLOW-CACHE-WRITE."
async
async-by-process)))
-(defun phps-mode-lex-analyzer--move-states (start diff)
- "Move lexer states after (or equal to) START with modification DIFF."
- (when phps-mode-lex-analyzer--states
- (setq phps-mode-lex-analyzer--states
- (phps-mode-lex-analyzer--get-moved-states
- phps-mode-lex-analyzer--states
- start
- diff))))
-
-(defun phps-mode-lex-analyzer--get-moved-states (states start diff)
- "Return moved lexer STATES after (or equal to) START with modification DIFF."
- (let ((old-states states)
- (new-states '()))
- (when old-states
-
- ;; Iterate through states add states before start start unchanged and
the others modified with diff
- (dolist (state-object (nreverse old-states))
- (let ((state-start (nth 0 state-object))
- (state-end (nth 1 state-object))
- (state-symbol (nth 2 state-object))
- (state-stack (nth 3 state-object))
- (heredoc-label (nth 4 state-object))
- (heredoc-label-stack (nth 5 state-object))
- (nest-location-stack (nth 6 state-object)))
- (if (>= state-start start)
- (let ((new-state-start (+ state-start diff))
- (new-state-end (+ state-end diff)))
- (push
- (list
- new-state-start
- new-state-end
- state-symbol
- state-stack
- heredoc-label
- heredoc-label-stack
- nest-location-stack)
- new-states))
- (if (> state-end start)
- (let ((new-state-end (+ state-end diff)))
- (push
- (list
- state-start
- new-state-end
- state-symbol
- state-stack
- heredoc-label
- heredoc-label-stack
- nest-location-stack)
- new-states))
- (push
- state-object
- new-states))))))
- new-states))
-
-(defun phps-mode-lex-analyzer--move-tokens (start diff)
- "Update tokens with moved lexer tokens after
-or equal to START with modification DIFF."
- (when phps-mode-lex-analyzer--tokens
- (setq
- phps-mode-lex-analyzer--tokens
- (phps-mode-lex-analyzer--get-moved-tokens
- phps-mode-lex-analyzer--tokens
- start
- diff))))
-
-(defun phps-mode-lex-analyzer--get-moved-tokens (old-tokens start diff)
- "Return moved lexer OLD-TOKENS positions after
-(or equal to) START with DIFF points."
- (let ((new-tokens '()))
- (when old-tokens
-
- ;; Iterate over all tokens, add those that are to be left unchanged and
add modified ones that should be changed.
- (dolist (token (nreverse old-tokens))
- (let ((token-symbol (car token))
- (token-start (car (cdr token)))
- (token-end (cdr (cdr token))))
- (if (>= token-start start)
- (let ((new-token-start (+ token-start diff))
- (new-token-end (+ token-end diff)))
- (push `(,token-symbol ,new-token-start . ,new-token-end)
new-tokens))
- (if (> token-end start)
- (let ((new-token-end (+ token-end diff)))
- (push `(,token-symbol ,token-start . ,new-token-end)
new-tokens))
- (push token new-tokens))))))
- new-tokens))
-
(defun phps-mode-lex-analyzer--reset-changes ()
"Reset change."
(setq phps-mode-lex-analyzer--change-min nil))
@@ -759,152 +625,49 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
(with-current-buffer buffer
(phps-mode-lex-analyzer--reset-imenu)
(let ((run-full-lexer nil)
- (old-tokens phps-mode-lex-analyzer--tokens)
- (old-states phps-mode-lex-analyzer--states)
- (log '()))
+ (log '())
+ (incremental-start-new-buffer
+ phps-mode-lex-analyzer--change-min))
+
+ ;; Reset idle timer
+ (phps-mode-lex-analyzer--cancel-idle-timer)
(if phps-mode-lex-analyzer--change-min
(progn
(phps-mode-debug-message
- (message "Processing change point minimum: %s"
phps-mode-lex-analyzer--change-min))
- (let ((incremental-state nil)
- (incremental-state-stack nil)
- (incremental-heredoc-label nil)
- (incremental-heredoc-label-stack nil)
- (incremental-nest-location-stack nil)
- (head-states '())
- (head-tokens '())
- (change-start phps-mode-lex-analyzer--change-min)
- (incremental-start-new-buffer
phps-mode-lex-analyzer--change-min))
-
- ;; Reset idle timer
- (phps-mode-lex-analyzer--cancel-idle-timer)
-
- ;; NOTE Starts are inclusive while ends are exclusive buffer
locations
-
- ;; Some tokens have dynamic length and if a change occurs at
token-end
- ;; we must start the incremental process at previous token
start
-
- ;; Build list of tokens from old buffer before start of
changes (head-tokens)
-
- (catch 'quit
- (dolist (token old-tokens)
- (let ((token-type (car token))
- (start (car (cdr token)))
- (end (cdr (cdr token))))
- (if (< end change-start)
- (push token head-tokens)
- (when (< start change-start)
- (when (equal token-type 'T_END_HEREDOC)
- ;; When incremental start is on a T_END_HEREDOC
token
- ;; rewind another token to allow expansion of
- ;; T_ENCAPSED_AND_WHITESPACE
- (phps-mode-debug-message
- (message
- "Rewinding incremental start due to
'T_END_HEREDOC token"))
- (let ((previous-token (pop head-tokens)))
- (setq
- start
- (car (cdr previous-token)))))
-
- (phps-mode-debug-message
- (message
- "New incremental-start-new-buffer: %s"
- start))
- (setq
- incremental-start-new-buffer
- start))
- (throw 'quit "break")))))
-
- (setq head-tokens (nreverse head-tokens))
- (phps-mode-debug-message
- (message "Head tokens: %s" head-tokens)
- (message "Incremental-start-new-buffer: %s"
incremental-start-new-buffer))
-
- ;; Did we find a start for the incremental process?
- (if head-tokens
- (progn
- (phps-mode-debug-message
- (message "Found head tokens"))
-
- ;; In old buffer:
- ;; 1. Determine state (incremental-state) and
state-stack (incremental-state-stack) heredoc label (incremental-heredoc-label)
heredoc-label-stack (heredoc-label-stack) before incremental start
- ;; 2. Build list of states before incremental start
(head-states)
- (catch 'quit
- (dolist (state-object (nreverse old-states))
- (let ((end (nth 1 state-object)))
- (if (<= end incremental-start-new-buffer)
- (progn
- (setq incremental-state (nth 2 state-object))
- (setq incremental-state-stack (nth 3
state-object))
- (setq incremental-heredoc-label (nth 4
state-object))
- (setq incremental-heredoc-label-stack (nth 5
state-object))
- (setq incremental-nest-location-stack (nth 6
state-object))
- (push state-object head-states))
- (throw 'quit "break")))))
-
- (phps-mode-debug-message
- (message "Head states: %s" head-states)
- (message "Incremental state: %s" incremental-state)
- (message "State stack: %s" incremental-state-stack)
- (message "Incremental heredoc-label: %s"
incremental-heredoc-label)
- (message "Incremental heredoc-label-stack: %s"
incremental-heredoc-label-stack)
- (message "Incremental nest-location-stack: %s"
incremental-nest-location-stack))
-
- (if (and
- head-states
- incremental-state)
- (progn
- (phps-mode-debug-message
- (message "Found head states"))
-
- (push (list 'INCREMENTAL-LEX
incremental-start-new-buffer) log)
-
- ;; Do partial lex from previous-token-end to
change-stop
-
- (let ((buffer-contents)
- (buffer-max))
- (save-restriction
- (widen)
- (setq
- buffer-contents
- (buffer-substring-no-properties
- (point-min)
- (point-max)))
- (setq
- buffer-max
- (point-max)))
- (phps-mode-lex-analyzer--incremental-lex-string
- (buffer-name)
- buffer-contents
- incremental-start-new-buffer
- buffer-max
- head-states
- incremental-state
- incremental-state-stack
- incremental-heredoc-label
- incremental-heredoc-label-stack
- incremental-nest-location-stack
- head-tokens
- force-synchronous
- (if (buffer-modified-p) nil buffer-file-name)
- (not (buffer-modified-p)))))
-
- (push (list 'FOUND-NO-HEAD-STATES
incremental-start-new-buffer) log)
- (phps-mode-debug-message
- (message "Found no head states"))
-
- (setq run-full-lexer t)))
-
- (push (list 'FOUND-NO-HEAD-TOKENS
incremental-start-new-buffer) log)
- (phps-mode-debug-message
- (message "Found no head tokens"))
+ (message
+ "Processing change point minimum: %s"
+ phps-mode-lex-analyzer--change-min))
+
+ (push (list 'INCREMENTAL-LEX incremental-start-new-buffer) log)
+
+ ;; Do partial lex from previous-token-end to change-stop
+
+ (let ((buffer-contents)
+ (buffer-max))
+ (save-restriction
+ (widen)
+ (setq
+ buffer-contents
+ (buffer-substring-no-properties
+ (point-min)
+ (point-max)))
+ (setq
+ buffer-max
+ (point-max)))
+
+ (phps-mode-lex-analyzer--incremental-lex-string
+ (buffer-name)
+ buffer-contents
+ phps-mode-lex-analyzer--cache
+ incremental-start-new-buffer
+ force-synchronous
+ (if (buffer-modified-p) nil buffer-file-name)
+ (not (buffer-modified-p)))))
- (setq run-full-lexer t))))
(push (list 'FOUND-NO-CHANGE-POINT-MINIMUM) log)
(phps-mode-debug-message
(message "Found no change point minimum"))
-
(setq run-full-lexer t))
(when run-full-lexer
@@ -918,41 +681,6 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
log))))
-(defun phps-mode-lex-analyzer--get-moved-lines-indent (old-lines-indents
start-line-number diff)
- "Move OLD-LINES-INDENTS from START-LINE-NUMBER with DIFF points."
- (let ((lines-indents (make-hash-table :test 'equal))
- (line-number 1))
- (when old-lines-indents
- (let ((line-indent (gethash line-number old-lines-indents))
- (new-line-number))
- (while line-indent
-
- (when (< line-number start-line-number)
- ;; (message "Added new indent 3 %s from %s to %s" line-indent
line-number line-number)
- (puthash line-number line-indent lines-indents))
-
- (when (and (> diff 0)
- (>= line-number start-line-number)
- (< line-number (+ start-line-number diff)))
- ;; (message "Added new indent 2 %s from %s to %s" line-indent
line-number line-number)
- (puthash line-number (gethash start-line-number old-lines-indents)
lines-indents))
-
- (when (>= line-number start-line-number)
- (setq new-line-number (+ line-number diff))
- ;; (message "Added new indent %s from %s to %s" line-indent
line-number new-line-number)
- (puthash new-line-number line-indent lines-indents))
-
- (setq line-number (1+ line-number))
- (setq line-indent (gethash line-number old-lines-indents))))
- lines-indents)))
-
-(defun phps-mode-lex-analyzer--move-imenu-index (start diff)
- "Moved imenu from START by DIFF points."
- (when phps-mode-lex-analyzer--imenu
- (setq phps-mode-lex-analyzer--imenu
- (phps-mode-lex-analyzer--get-moved-imenu
phps-mode-lex-analyzer--imenu start diff))
- (phps-mode-lex-analyzer--reset-imenu)))
-
(defun phps-mode-lex-analyzer--get-bookkeeping ()
"Return bookkeeping, process buffer if not done already."
phps-mode-lex-analyzer--bookkeeping)
@@ -961,28 +689,6 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
"Return Imenu, process buffer if not done already."
phps-mode-lex-analyzer--imenu)
-(defun phps-mode-lex-analyzer--get-moved-imenu (old-index start diff)
- "Move imenu-index OLD-INDEX beginning from START with DIFF."
- (let ((new-index '()))
-
- (when old-index
- (if (and (listp old-index)
- (listp (car old-index)))
- (dolist (item old-index)
- (let ((sub-item (phps-mode-lex-analyzer--get-moved-imenu item
start diff)))
- (push (car sub-item) new-index)))
- (let ((item old-index))
- (let ((item-label (car item)))
- (if (listp (cdr item))
- (let ((sub-item (phps-mode-lex-analyzer--get-moved-imenu (cdr
item) start diff)))
- (push `(,item-label . ,sub-item) new-index))
- (let ((item-start (cdr item)))
- (when (>= item-start start)
- (setq item-start (+ item-start diff)))
- (push `(,item-label . ,item-start) new-index)))))))
-
- (nreverse new-index)))
-
(defun phps-mode-lex-analyzer--cancel-idle-timer ()
"Cancel idle timer."
(phps-mode-debug-message (message "Cancelled idle timer"))
@@ -1238,22 +944,28 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
"Do not un-comment comment ending at %s"
token-end))))))))))))
-(defun phps-mode-lex-analyzer--setup (start end)
- "Just prepare other lexers for lexing region START to END."
- (require 'phps-mode-macros)
- (phps-mode-debug-message (message "Lexer setup %s - %s" start end))
- (unless phps-mode-lex-analyzer--state
- (setq phps-mode-lex-analyzer--state 'ST_INITIAL)))
-
-;; TODO Is it possible to have a incremental parser?
-(defun phps-mode-lex-analyzer--lex-string (contents &optional start end states
state state-stack heredoc-label heredoc-label-stack nest-location-stack tokens
filename allow-cache-read allow-cache-write)
+(defun phps-mode-lex-analyzer--lex-string
+ (
+ contents
+ &optional
+ incremental-start
+ memory-cache
+ filename
+ allow-cache-read
+ allow-cache-write
+ )
"Run lexer on CONTENTS."
;; Create a separate buffer, run lexer inside of it, catch errors and return
them
;; to enable nice presentation
(require 'phps-mode-macros)
- ;; TODO Should use pre-populated cache here for incremental lexing
- (setq phps-mode-lexer--cached nil)
+ ;; Setup memory cache
+ (if incremental-start
+ (setq phps-mode-lexer--cached-point incremental-start)
+ (setq phps-mode-lexer--cached-point nil))
+ (if memory-cache
+ (setq phps-mode-lexer--cached memory-cache)
+ (setq phps-mode-lexer--cached nil))
(let* ((loaded-from-cache)
(cache-key)
@@ -1283,6 +995,7 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
(if loaded-from-cache
loaded-from-cache
(let* ((buffer (generate-new-buffer "*PHPs Parser*"))
+ (cache)
(parse-error)
(parse-trail)
(ast-tree)
@@ -1318,13 +1031,8 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
;; Need to copy buffer-local values before killing buffer
;; Copy variables outside of buffer
- (setq state phps-mode-lexer--state)
- (setq state-stack phps-mode-lexer--state-stack)
- (setq states phps-mode-lexer--states)
+ (setq cache phps-mode-lexer--cached)
(setq tokens (nreverse phps-mode-lexer--generated-tokens))
- (setq heredoc-label phps-mode-lexer--heredoc-label)
- (setq heredoc-label-stack phps-mode-lexer--heredoc-label-stack)
- (setq nest-location-stack phps-mode-lexer--nest-location-stack)
(setq parse-trail phps-mode-ast--parse-trail)
(setq ast-tree phps-mode-ast--tree)
(setq bookkeeping phps-mode-parser-sdt-bookkeeping)
@@ -1350,12 +1058,7 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
(let ((data
(list
tokens
- states
- state
- state-stack
- heredoc-label
- heredoc-label-stack
- nest-location-stack
+ cache
parse-trail
parse-error
ast-tree
@@ -1370,7 +1073,8 @@ of performed operations. Optionally do it
FORCE-SYNCHRONOUS."
phps-mode-cache--use-p
allow-cache-write
cache-key)
- ;; (message "Saved to cache")
+ (phps-mode-debug-message
+ (message "Saved to cache"))
(phps-mode-cache-save
data
cache-key))
diff --git a/phps-mode-lexer.el b/phps-mode-lexer.el
index 265225745e..dfd2b3a48d 100644
--- a/phps-mode-lexer.el
+++ b/phps-mode-lexer.el
@@ -41,9 +41,12 @@
(eval-when-compile (phps-mode-lexer-generator--lambdas))
"Hash-table of lex-analyzer rules organized by state.")
-(defvar phps-mode-lexer--cached nil
+(defvar-local phps-mode-lexer--cached nil
"Hash-table of performed lexes to enable incremental lexing for the parser.")
+(defvar-local phps-mode-lexer--cached-point nil
+ "The point up to where the cache should be used.")
+
;; LEXER FUNCTIONS BELOW
@@ -54,17 +57,30 @@
(defun phps-mode-lexer--re2c (index old-state)
"Elisp port of original Zend re2c lexer."
- (let ((cache-key index))
+ (let ((cache-key index)
+ (cache))
+ ;; So if we have a cached lex, and we should use the cache for this point
in the stream
+ ;; make sure that it does not use a move operation or that the move
operation is below
+ ;; the valid point of the cache
+ (when (and
+ phps-mode-lexer--cached-point
+ (<= index phps-mode-lexer--cached-point)
+ phps-mode-lexer--cached)
+ (setq
+ cache
+ (gethash cache-key phps-mode-lexer--cached)))
(if (and
- phps-mode-lexer--cached
- (gethash cache-key phps-mode-lexer--cached))
+ cache
+ (or
+ (equal (nth 1 cache) nil)
+ (<= (nth 1 cache) phps-mode-lexer--cached-point)))
(progn
(phps-mode-debug-message
(message
- "Returning cached lex for key %S: %S"
+ "\nReturning cached lex for key %S: %S"
cache-key
- (gethash cache-key phps-mode-lexer--cached)))
- (gethash cache-key phps-mode-lexer--cached))
+ cache))
+ cache)
(let ((eof (>= index (point-max))))
(if eof
@@ -126,9 +142,9 @@
start
end))
(message
- "\nRunning lexer from point %s, state: %s, lookahead: '%s'.."
+ "\nRunning lexer from point %s, state: %S, lookahead: '%s'.."
(point)
- phps-mode-lexer--state
+ old-state
lookahead)))
(setq phps-mode-lexer--move-flag nil)
(setq phps-mode-lexer--restart-flag nil)
@@ -228,6 +244,11 @@
(let ((lexer-response
(list tokens move-to-index new-state)))
+ (phps-mode-debug-message
+ (message
+ "\nStored cached lex for key %S: %S"
+ cache-key
+ lexer-response))
(puthash
cache-key
lexer-response
diff --git a/phps-mode-test.el b/phps-mode-test.el
index c77a1d6c1c..15b3b95337 100644
--- a/phps-mode-test.el
+++ b/phps-mode-test.el
@@ -20,60 +20,57 @@
(defmacro phps-mode-test--incremental-vs-intial-buffer (source &optional title
&rest change)
"Set up test buffer with SOURCE, TITLE, apply CHANGE and
compare incremental values with initial values."
- `(let ((test-buffer-incremental (generate-new-buffer "test-incremental"))
- (incremental-state)
- (incremental-state-stack)
- (incremental-states)
- (incremental-heredoc-label)
- (incremental-heredoc-label-stack)
+ `(let ((test-buffer-incremental
+ (generate-new-buffer "test-incremental"))
+ (incremental-cache)
(incremental-parse-trail)
(incremental-parse-error)
(incremental-tokens)
(incremental-imenu)
(incremental-buffer)
(incremental-bookkeeping)
- (incremental-nest-location-stack)
- (test-buffer-initial (generate-new-buffer "test-initial"))
- (initial-state)
- (initial-state-stack)
- (initial-states)
- (initial-heredoc-label)
- (initial-heredoc-label-stack)
+ (test-buffer-initial
+ (generate-new-buffer "test-initial"))
+ (initial-cache)
(initial-parse-trail)
(initial-parse-error)
(initial-tokens)
(initial-imenu)
(initial-buffer)
- (initial-bookkeeping)
- (initial-nest-location-stack))
+ (initial-bookkeeping))
;; Setup incremental buffer
(switch-to-buffer test-buffer-incremental)
(insert ,source)
(goto-char 0)
+
(phps-mode-debug-message
(message "\nTesting incremental buffer '%s':\n'%s'\n" ,title ,source))
+
(phps-mode)
(phps-mode-debug-message
(message "\nPerforming changes\n"))
-
+
+ (switch-to-buffer test-buffer-incremental)
,@change
- (phps-mode-lex-analyzer--process-changes test-buffer-incremental)
- (setq incremental-state phps-mode-lex-analyzer--state)
- (setq incremental-state-stack phps-mode-lex-analyzer--state-stack)
- (setq incremental-heredoc-label phps-mode-lex-analyzer--heredoc-label)
- (setq incremental-heredoc-label-stack
phps-mode-lex-analyzer--heredoc-label-stack)
+
+ (phps-mode-lex-analyzer--process-changes
+ test-buffer-incremental)
+
+ (setq
+ incremental-cache
+ (phps-mode-test--hash-to-list
+ phps-mode-lex-analyzer--cache
+ t))
(setq incremental-parse-trail phps-mode-lex-analyzer--parse-trail)
(setq incremental-parse-error phps-mode-lex-analyzer--parse-error)
- (setq incremental-states phps-mode-lex-analyzer--states)
(setq incremental-tokens phps-mode-lex-analyzer--tokens)
(setq incremental-imenu phps-mode-lex-analyzer--imenu)
(save-restriction
(widen)
(setq incremental-buffer (buffer-substring (point-min) (point-max))))
(setq incremental-bookkeeping (phps-mode-test--hash-to-list
phps-mode-lex-analyzer--bookkeeping t))
- (setq incremental-nest-location-stack
phps-mode-lex-analyzer--nest-location-stack)
;; Setup incremental buffer
(switch-to-buffer test-buffer-initial)
@@ -81,35 +78,37 @@ compare incremental values with initial values."
(goto-char 0)
(phps-mode-debug-message
(message "\nTesting initial buffer '%s':\n'%s'\n" ,title
incremental-buffer))
+
(phps-mode)
- (setq initial-state phps-mode-lex-analyzer--state)
- (setq initial-state-stack phps-mode-lex-analyzer--state-stack)
- (setq initial-states phps-mode-lex-analyzer--states)
- (setq initial-heredoc-label phps-mode-lex-analyzer--heredoc-label)
- (setq initial-heredoc-label-stack
phps-mode-lex-analyzer--heredoc-label-stack)
+
+ (setq
+ initial-cache
+ (phps-mode-test--hash-to-list
+ phps-mode-lex-analyzer--cache
+ t))
(setq initial-parse-trail phps-mode-lex-analyzer--parse-trail)
(setq initial-parse-error phps-mode-lex-analyzer--parse-error)
(setq initial-tokens phps-mode-lex-analyzer--tokens)
(setq initial-imenu phps-mode-lex-analyzer--imenu)
(setq initial-buffer (buffer-substring (point-min) (point-max)))
- (setq initial-bookkeeping (phps-mode-test--hash-to-list
phps-mode-lex-analyzer--bookkeeping t))
- (setq initial-nest-location-stack
phps-mode-lex-analyzer--nest-location-stack)
+ (setq
+ initial-bookkeeping
+ (phps-mode-test--hash-to-list
+ phps-mode-lex-analyzer--bookkeeping
+ t))
;; Run tests
(phps-mode-debug-message
(message "\nComparing tokens, bookkeeping and imenu between
buffer:\n\n'%s'\n\nand:\n\n'%s'\n" initial-buffer incremental-buffer))
(should (equal initial-buffer incremental-buffer))
+
;; (message "Initial tokens: %s\n" initial-tokens)
;; (message "Incremental tokens: %s\n" incremental-tokens)
+
+ (should (equal initial-cache incremental-cache))
(should (equal initial-tokens incremental-tokens))
- (should (equal initial-state incremental-state))
- (should (equal initial-state-stack incremental-state-stack))
- (should (equal initial-states incremental-states))
- (should (equal initial-heredoc-label incremental-heredoc-label))
- (should (equal initial-heredoc-label-stack
incremental-heredoc-label-stack))
(should (equal initial-parse-trail incremental-parse-trail))
(should (equal initial-parse-error incremental-parse-error))
- (should (equal initial-nest-location-stack
incremental-nest-location-stack))
(should (equal initial-imenu incremental-imenu))
(should (equal initial-bookkeeping incremental-bookkeeping))
diff --git a/test/phps-mode-test-ast.el b/test/phps-mode-test-ast.el
index 59584ee92b..a011f9ac92 100644
--- a/test/phps-mode-test-ast.el
+++ b/test/phps-mode-test-ast.el
@@ -24,6 +24,7 @@
name
(lambda()
(setq phps-mode-lexer--cached nil)
+ (setq phps-mode-lexer--cached-point nil)
(let ((parse (phps-mode-parser-parse)))
(phps-mode-test--output-parse-productions parse))
(phps-mode-ast--generate)
diff --git a/test/phps-mode-test-integration.el
b/test/phps-mode-test-integration.el
index 4f56f254ea..804ee01e6b 100644
--- a/test/phps-mode-test-integration.el
+++ b/test/phps-mode-test-integration.el
@@ -150,7 +150,6 @@
"<?php\n$str = <<<EOD\nExample of string\nspanning multiple lines\nusing
heredoc syntax.\nEOD;\n\n/* More complex example, with variables. */\nclass
foo\n{\n var $foo;\n var $bar;\n\n function __construct()\n {\n
$this->foo = 'Foo';\n $this->bar = array('Bar1', 'Bar2', 'Bar3');\n
}\n}\n\n$foo = new foo();\n$name = 'MyName';\n\necho <<<EOT\nMy name is
\"$name\". I am printing some $foo->foo.\nNow, I am printing some
{$foo->bar[1]}.\nThis should print a capi [...]
"Integration-test 14 complex HEREDOC with removed heredoc delimiter
semicolon"
;; (message "\nTokens: %s" phps-mode-lex-analyzer--tokens)
- ;; (message "States: %s\n" phps-mode-lex-analyzer--states)
(goto-char 88)
(delete-char 1))
diff --git a/test/phps-mode-test-parser.el b/test/phps-mode-test-parser.el
index dcfd893f55..c440c7442a 100644
--- a/test/phps-mode-test-parser.el
+++ b/test/phps-mode-test-parser.el
@@ -23,6 +23,7 @@
name
(buffer-substring-no-properties (point-min) (point-max)))
(setq phps-mode-lexer--cached nil)
+ (setq phps-mode-lexer--cached-point nil)
;; Run test
(funcall logic)
- [elpa] externals/phps-mode updated (70559d0374 -> e7e5cdacf0), Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode b0cc0715d9 01/29: Updated parser to 8.3, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode b7c4ff5df0 03/29: Improved lexing code, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 27ed8d5c5c 04/29: Improved comments, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode f481c6708b 09/29: Improved debugging, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode d3a3c28f48 02/29: Passing first parse test for PHP 8.3, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 2cdeaf00e3 07/29: Added new parser unit tests for PHP 8.3, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 2f9ba6792c 08/29: Started on integrating new parser with major-mode, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode c51076e329 12/29: Passing unit tests for AST, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode fe91b2a9f3 11/29: Passing more tests for PHP 8.3, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode f9e5d6233f 19/29: Incremental lexer working somewhat again,
Christian Johansson <=
- [elpa] externals/phps-mode f3ad4ab698 15/29: Started on refactor of parser-lexer relationship, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode cf399d7697 17/29: Fixed some byte compilation issues, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 23ce2baf9e 16/29: Passing all unit tests again, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 7e9a744775 24/29: Added todo items, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode e7e23cce60 26/29: Fix for incremental syntax coloring issue, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 673a801311 25/29: Resolved issue of parser buffer not closing, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 676cfe11e6 28/29: Updated version and change-log, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode e7e5cdacf0 29/29: Fixed bye-compilation warning and missing error declaration, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode ad00c6eac5 27/29: Updated parser generator information and cleaned up code, Christian Johansson, 2024/02/15
- [elpa] externals/phps-mode 0262e707ca 05/29: Passing more parsing tests, Christian Johansson, 2024/02/15