[elpa] externals/ess 76c24f4c22 06/12: Port ELT tests for R tokenification to etest

[ELPA Syncer]

branch: externals/ess
commit 76c24f4c22361c1444590f0591f69a430f969455
Author: Lionel Henry <lionel.hry@gmail.com>
Commit: Lionel Henry <lionel.hry@gmail.com>

    Port ELT tests for R tokenification to etest
---
 test/ess-test-r-syntax.el |   2 +-
 test/ess-test-r-token.el  | 420 +++++++++++++++++++++++++++++++++++++
 test/ess-test-r-utils.el  |  19 +-
 test/literate/tokens.R    | 522 ----------------------------------------------
 test/literate/tokens.el   |   8 -
 test/run-tests            |   5 +-
 6 files changed, 442 insertions(+), 534 deletions(-)

diff --git a/test/ess-test-r-syntax.el b/test/ess-test-r-syntax.el
index db8cd2abc1..94f9b8e44b 100644
--- a/test/ess-test-r-syntax.el
+++ b/test/ess-test-r-syntax.el
@@ -1,4 +1,4 @@
-;;; ess-test-r-tokens.el --- ESS tests for R tokens  -*- lexical-binding: t; 
-*-
+;;; ess-test-r-syntax.el --- ESS tests for R syntax  -*- lexical-binding: t; 
-*-
 ;;
 ;; This file is free software; you can redistribute it and/or modify
 ;; it under the terms of the GNU General Public License as published by
diff --git a/test/ess-test-r-token.el b/test/ess-test-r-token.el
new file mode 100644
index 0000000000..7988d7e496
--- /dev/null
+++ b/test/ess-test-r-token.el
@@ -0,0 +1,420 @@
+;;; ess-test-r-token.el --- ESS tests for R tokens  -*- lexical-binding: t; -*-
+;;
+;; This file is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 2, or (at your option)
+;; any later version.
+;;
+;; This file is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+;;
+;; A copy of the GNU General Public License is available at
+;; https://www.r-project.org/Licenses/
+;;
+;;; Commentary:
+;;
+
+(require 'ert)
+(require 'etest "etest/etest")
+(require 'ess-r-mode)
+(require 'ess-test-r-utils)
+
+(etest-deftest ess-test-r-token-jump-strings-symbols-test ()
+  :case "¶`a\"a\"a` \"a`a`a\""
+
+  (ess-test-should-token= "identifier" "`a\"a\"a`")
+  :result "`a\"a\"a`¶ \"a`a`a\""
+
+  (ess-test-should-token= "string" "\"a`a`a\"")
+  :result "`a\"a\"a` \"a`a`a\"¶")
+
+(etest-deftest ess-test-r-token-jump-identifiers-test ()
+  :case "¶.a_a a10"
+
+  (ess-test-should-token= "identifier" ".a_a")
+  :result ".a_a¶ a10"
+
+  (ess-test-should-token= "identifier" "a10")
+  :result ".a_a a10¶")
+
+(etest-deftest ess-test-r-token-jump-numbers-test ()
+  :case "¶100 1E10 1e10 1.10"
+
+  (ess-test-should-token= "number" "100")
+  :result "100¶ 1E10 1e10 1.10"
+
+  (ess-test-should-token= "number" "1E10")
+  :result "100 1E10¶ 1e10 1.10"
+
+  (ess-test-should-token= "number" "1e10")
+  :result "100 1E10 1e10¶ 1.10"
+
+  (ess-test-should-token= "number" "1.10")
+  :result "100 1E10 1e10 1.10¶")
+
+(etest-deftest ess-test-r-token-jump-delimiters-test ()
+  :case "¶() a[[[]]] {}"
+
+  (ess-test-should-token= "(")
+  :result "(¶) a[[[]]] {}"
+
+  (ess-test-should-token= ")")
+  :result "()¶ a[[[]]] {}"
+
+  (ess-jump-token)
+  (ess-test-should-token= "[[")
+  :result "() a[[¶[]]] {}"
+
+  (should (ess-jump-token "["))
+  (should (ess-token-before= "[["))
+  :result "() a[[[¶]]] {}"
+
+  (ess-test-should-token= "]]")
+  :result "() a[[[]]¶] {}"
+
+  (should (ess-jump-token "]"))
+  (should (ess-token-before= "]]"))
+  :result "() a[[[]]]¶ {}"
+
+  (ess-test-should-token= "{")
+  :result "() a[[[]]] {¶}"
+
+  (ess-test-should-token= "}")
+  :result "() a[[[]]] {}¶")
+
+(etest-deftest ess-test-r-token-buffer-boundaries-test ()
+  :case "¶"
+
+  (should (ess-token-before= "buffer-start"))
+  (should (ess-token-after= "buffer-end"))
+  :result "¶")
+
+(etest-deftest ess-test-r-token-jump-punctuation-test ()
+  :case "¶.; .,"
+
+  (ess-jump-token)
+  (ess-test-should-token= ";")
+  :result ".;¶ .,"
+
+  (ess-jump-token)
+  (ess-test-should-token= ",")
+  :result ".; .,¶")
+
+(etest-deftest ess-test-r-token-jump-keywords-test ()
+  :case "¶if if_else else function while for"
+
+  (ess-test-should-token= "if")
+  (ess-test-should-token= "identifier" "if_else")
+  (ess-test-should-token= "else")
+  (ess-test-should-token= "function")
+  (ess-test-should-token= "while")
+  (ess-test-should-token= "for")
+
+  :result "if if_else else function while for¶")
+
+(etest-deftest ess-test-r-token-jump-logical-operators-test ()
+  :case "¶a & a && a &&& a | a || a ||| a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "&")
+
+  :result "a &¶ a && a &&& a | a || a ||| a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "&&")
+
+  :result "a & a &&¶ a &&& a | a || a ||| a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "&&")
+
+  :result "a & a && a &&¶& a | a || a ||| a"
+
+  (should (ess-jump-token "&"))
+  (should (ess-token-before= "&&"))
+
+  :result "a & a && a &&&¶ a | a || a ||| a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "|")
+
+  :result "a & a && a &&& a |¶ a || a ||| a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "||")
+
+  :result "a & a && a &&& a | a ||¶ a ||| a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "||")
+
+  :result "a & a && a &&& a | a || a ||¶| a"
+
+  (should (ess-jump-token "|"))
+  (should (ess-token-before= "||"))
+
+  :result "a & a && a &&& a | a || a |||¶ a")
+
+(etest-deftest ess-test-r-token-jump-comparison-operators-test ()
+  :case "¶a = a := a == a === a :== a != a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "=")
+  :result "a =¶ a := a == a === a :== a != a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= ":=")
+  :result "a = a :=¶ a == a === a :== a != a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "==")
+  :result "a = a := a ==¶ a === a :== a != a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "==")
+  :result "a = a := a == a ==¶= a :== a != a :!= a"
+
+  (should (ess-jump-token "="))
+  (should (ess-token-before= "=="))
+  :result "a = a := a == a ===¶ a :== a != a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= ":=")
+  :result "a = a := a == a === a :=¶= a != a :!= a"
+
+  (should (ess-jump-token "="))
+  (should (ess-token-before= "=="))
+  :result "a = a := a == a === a :==¶ a != a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "!=")
+  :result "a = a := a == a === a :== a !=¶ a :!= a"
+
+  (ess-jump-token)
+  (ess-test-should-token= ":")
+  :result "a = a := a == a === a :== a != a :¶!= a"
+
+  (ess-test-should-token= "!=")
+  :result "a = a := a == a === a :== a != a :!=¶ a")
+
+(etest-deftest ess-test-r-token-jump-%%-operators-test ()
+  :case "¶a %>% a %a`a`a\"a\"a$a@a% a %% a %%% a % a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "%infix%" "%>%")
+  :result "a %>%¶ a %a`a`a\"a\"a$a@a% a %% a %%% a % a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "%infix%" "%a`a`a\"a\"a$a@a%")
+  :result "a %>% a %a`a`a\"a\"a$a@a%¶ a %% a %%% a % a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "%%")
+  :result "a %>% a %a`a`a\"a\"a$a@a% a %%¶ a %%% a % a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "%%")
+  :result "a %>% a %a`a`a\"a\"a$a@a% a %% a %%¶% a % a"
+
+  (ess-test-should-token= "%infix%" "% a %")
+  :result "a %>% a %a`a`a\"a\"a$a@a% a %% a %%% a %¶ a")
+
+(etest-deftest ess-test-r-token-jump-arithmetic-operators-test ()
+  :case "¶a + a - a - -a * a ** a ^ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "+")
+  :result "a +¶ a - a - -a * a ** a ^ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "-")
+  :result "a + a -¶ a - -a * a ** a ^ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "-")
+  :result "a + a - a -¶ -a * a ** a ^ a ^ ++a"
+
+  (ess-test-should-token= "-")
+  :result "a + a - a - -¶a * a ** a ^ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "*")
+  :result "a + a - a - -a *¶ a ** a ^ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "**")
+  :result "a + a - a - -a * a **¶ a ^ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "^")
+  :result "a + a - a - -a * a ** a ^¶ a ^ ++a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "^")
+  :result "a + a - a - -a * a ** a ^ a ^¶ ++a"
+
+  (ess-test-should-token= "+")
+  :result "a + a - a - -a * a ** a ^ a ^ +¶+a"
+
+  (ess-test-should-token= "+")
+  :result "a + a - a - -a * a ** a ^ a ^ ++¶a")
+
+(etest-deftest ess-test-r-token-jump-:-operators-test ()
+  :case "¶a:  a::  a:::  a::::  a:::="
+
+  (ess-jump-token)
+  (ess-test-should-token= ":")
+  :result "a:¶  a::  a:::  a::::  a:::="
+
+  (ess-jump-token)
+  (ess-test-should-token= "::")
+  :result "a:  a::¶  a:::  a::::  a:::="
+
+  (ess-jump-token)
+  (ess-test-should-token= ":::")
+  :result "a:  a::  a:::¶  a::::  a:::="
+
+  (ess-jump-token)
+  (ess-test-should-token= ":::")
+  :result "a:  a::  a:::  a:::¶:  a:::="
+
+  (should (ess-jump-token ":"))
+  (should (ess-token-before= ":::"))
+  :result "a:  a::  a:::  a::::¶  a:::="
+
+  (ess-jump-token)
+  (ess-test-should-token= ":::")
+  :result "a:  a::  a:::  a::::  a:::¶=")
+
+(etest-deftest ess-test-r-token-jump-assignment-operators-test ()
+  :case "¶a <-  a <<-  a -> >  a ->> a >> a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "<-")
+  :result "a <-¶  a <<-  a -> >  a ->> a >> a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "<<-")
+  :result "a <-  a <<-¶  a -> >  a ->> a >> a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "->")
+  :result "a <-  a <<-  a ->¶ >  a ->> a >> a"
+
+  (ess-test-should-token= ">")
+  :result "a <-  a <<-  a -> >¶  a ->> a >> a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "->>")
+  :result "a <-  a <<-  a -> >  a ->>¶ a >> a"
+
+  (ess-jump-token)
+  (ess-test-should-token= ">")
+  (ess-test-should-token= ">")
+  :result "a <-  a <<-  a -> >  a ->> a >>¶ a")
+
+(etest-deftest ess-test-r-token-jump-inequality-operators-test ()
+  :case "¶a < >  a >=  a > =  a <="
+
+  (ess-jump-token)
+  (ess-test-should-token= "<")
+  :result "a <¶ >  a >=  a > =  a <="
+
+  (ess-test-should-token= ">")
+  :result "a < >¶  a >=  a > =  a <="
+
+  (ess-jump-token)
+  (ess-test-should-token= ">=")
+  :result "a < >  a >=¶  a > =  a <="
+
+  (ess-jump-token)
+  (ess-test-should-token= ">")
+  :result "a < >  a >=  a >¶ =  a <="
+
+  (ess-test-should-token= "=")
+  :result "a < >  a >=  a > =¶  a <="
+
+  (ess-jump-token)
+  (ess-test-should-token= "<=")
+  :result "a < >  a >=  a > =  a <=¶")
+
+(etest-deftest ess-test-r-token-jump-special-operators-test ()
+  :case "¶~a~~a"
+
+  (ess-test-should-token= "~")
+  :result "~¶a~~a"
+
+  (ess-jump-token)
+  (ess-test-should-token= "~")
+  (ess-test-should-token= "~")
+  :result "~a~~¶a")
+
+(etest-deftest ess-test-r-token-refine-param-assignment-test ()
+  :case "call(param ¶= NULL)"
+
+  (ess-test-should-token= "=")
+  (should (ess-refined-token= (ess-token-before) "param-assign"))
+  :result "call(param =¶ NULL)")
+
+(etest-deftest ess-test-r-token-refine-quoted-param-names-test ()
+  :case "call(¶\"param\" = NULL)"
+
+  (ess-test-should-token= "string" "\"param\"")
+  (should (ess-refined-token= (ess-token-before) "identifier"))
+  :result "call(\"param\"¶ = NULL)")
+
+(etest-deftest ess-test-r-token-refine-quoted-call-names-test ()
+  :case "¶\"call\"()"
+
+  (ess-test-should-token= "string" "\"call\"")
+  (should (ess-refined-token= (ess-token-before) "identifier"))
+  :result "\"call\"¶()")
+
+(etest-deftest ess-test-r-token-skip-blanks-test ()
+  :case "
+text¶
+text"
+
+  (should (ess-skip-blanks-forward t))
+  :result "
+text
+¶text"
+
+  (should (not (ess-skip-blanks-backward)))
+  :result "
+text
+¶text"
+
+  (should (ess-skip-blanks-backward t))
+  :result "
+text¶
+text")
+
+(etest-deftest ess-test-r-token-skip-comments-test ()
+  :case "text¶ # comment"
+
+  (should (ess-skip-blanks-forward t))
+  :result "text ¶# comment"
+
+  (should (not (ess-skip-blanks-forward t)))
+  :result "text ¶# comment")
+
+(etest-deftest ess-test-r-token-skip-form-feed-test ()
+  :case "
+text
+
+
+¶text"
+
+  (should (ess-skip-blanks-backward t))
+  :result "
+text¶
+
+
+text")
+
+;; Local Variables:
+;; etest-local-config: etest-r-config
+;; End:
diff --git a/test/ess-test-r-utils.el b/test/ess-test-r-utils.el
index c77ee48663..ebe79e7b96 100644
--- a/test/ess-test-r-utils.el
+++ b/test/ess-test-r-utils.el
@@ -388,13 +388,30 @@ Throws an error if unsuccesful."
       (should (ess--essr-check-if-in-essrenv)))
     (kill-buffer)))
 
-(defun ess-test-token= (type &optional value)       ;; FIXME: `ess-' prefix?
+(defun ess-test-token= (type &optional value)
   "Check that the next token conforms to TYPE and VALUE.
 This checks it back and forth and moves the point after the
 token."
   (and (ess-jump-token type value)
        (ess-token-before= type value)))
 
+;; The macro has better failure reporting
+(defmacro ess-test-should-token= (type &optional value)
+  "Check that the next token conforms to TYPE and VALUE.
+This checks it back and forth and moves the point after the
+token."
+  `(let* ((-type ,type)
+          (-value ,value)
+          (tok (should (ess-jump-token))))
+     (let ((-type (if (listp -type) -type (list -type)))
+           (-value (if (listp -value) -value (list -value))))
+       (when -type
+         (should (ess-token-type tok))
+         (should (member (ess-token-type tok) -type)))
+       (when -value
+         (should (ess-token-value tok))
+         (should (member (ess-token-value tok) -value))))))
+
 (defmacro ess-with-toggled-font-lock-keyword (enable keywords &rest body)
   (declare (indent 2)
            (debug (&rest form)))
diff --git a/test/literate/tokens.R b/test/literate/tokens.R
deleted file mode 100644
index e84bb2ef77..0000000000
--- a/test/literate/tokens.R
+++ /dev/null
@@ -1,522 +0,0 @@
-
-##### Jumping/Climbing tokens
-
-### 1 Strings and backquoted names -----------------------------------
-
-¶`a"a"a` "a`a`a"
-
-##! (should (ess-test-token= "identifier" "`a\"a\"a`"))
-
-`a"a"a`¶ "a`a`a"
-
-##> (should (ess-test-token= "string" "\"a`a`a\""))
-
-`a"a"a` "a`a`a"¶
-
-
-### 2 Identifiers ----------------------------------------------------
-
-¶.a_a a10
-
-##! (should (ess-test-token= "identifier" ".a_a"))
-
-.a_a¶ a10
-
-##> (should (ess-test-token= "identifier" "a10"))
-
-.a_a a10¶
-
-
-### 3 Numbers --------------------------------------------------------
-
-¶100 1E10 1e10 1.10
-
-##! (should (ess-test-token= "number" "100"))
-
-100¶ 1E10 1e10 1.10
-
-##> (should (ess-test-token= "number" "1E10"))
-
-100 1E10¶ 1e10 1.10
-
-##> (should (ess-test-token= "number" "1e10"))
-
-100 1E10 1e10¶ 1.10
-
-##> (should (ess-test-token= "number" "1.10"))
-
-100 1E10 1e10 1.10¶
-
-
-### 4 Delimiters -----------------------------------------------------
-
-¶() a[[[]]] {}
-
-##! (should (ess-test-token= "("))
-
-(¶) a[[[]]] {}
-
-##> (should (ess-test-token= ")"))
-
-()¶ a[[[]]] {}
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "[["))
-
-() a[[¶[]]] {}
-
-##> (should (ess-jump-token "["))
-##> (should (ess-token-before= "[["))
-
-() a[[[¶]]] {}
-
-##> (should (ess-test-token= "]]"))
-
-() a[[[]]¶] {}
-
-##> (should (ess-jump-token "]"))
-##> (should (ess-token-before= "]]"))
-
-() a[[[]]]¶ {}
-
-##> (should (ess-test-token= "{"))
-
-() a[[[]]] {¶}
-
-##> (should (ess-test-token= "}"))
-
-() a[[[]]] {}¶
-
-
-### 5 Buffer boundaries
-
-¶
-
-##! (should (ess-token-before= "buffer-start"))
-##! (should (ess-token-after= "buffer-end"))
-
-¶
-
-
-### 6 Punctuation
-
-¶.; .,
-
-##! (ess-jump-token)
-##! (should (ess-test-token= ";"))
-
-.;¶ .,
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ","))
-
-.; .,¶
-
-
-### 7 Keywords
-
-¶if if_else else function while for
-
-##! (should (ess-test-token= "if"))
-##! (should (ess-test-token= "identifier" "if_else"))
-##! (should (ess-test-token= "else"))
-##! (should (ess-test-token= "function"))
-##! (should (ess-test-token= "while"))
-##! (should (ess-test-token= "for"))
-
-if if_else else function while for¶
-
-
-
-##### Jumping/Climbing operator tokens
-
-### 1 Logical operators ----------------------------------------------
-
-¶a & a && a &&& a | a || a ||| a
-
-##! (ess-jump-token)
-##! (should (ess-test-token= "&"))
-
-a &¶ a && a &&& a | a || a ||| a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "&&"))
-
-a & a &&¶ a &&& a | a || a ||| a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "&&"))
-
-a & a && a &&¶& a | a || a ||| a
-
-##> (should (ess-jump-token "&"))
-##> (should (ess-token-before= "&&"))
-
-a & a && a &&&¶ a | a || a ||| a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "|"))
-
-a & a && a &&& a |¶ a || a ||| a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "||"))
-
-a & a && a &&& a | a ||¶ a ||| a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "||"))
-
-a & a && a &&& a | a || a ||¶| a
-
-##> (should (ess-jump-token "|"))
-##> (should (ess-token-before= "||"))
-
-a & a && a &&& a | a || a |||¶ a
-
-
-### 2 Equality / equal operators -------------------------------------
-
-¶a = a := a == a === a :== a != a :!= a
-
-##! (ess-jump-token)
-##! (should (ess-test-token= "="))
-
-a =¶ a := a == a === a :== a != a :!= a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ":="))
-
-a = a :=¶ a == a === a :== a != a :!= a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "=="))
-
-a = a := a ==¶ a === a :== a != a :!= a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "=="))
-
-a = a := a == a ==¶= a :== a != a :!= a
-
-##> (should (ess-jump-token "="))
-##> (should (ess-token-before= "=="))
-
-a = a := a == a ===¶ a :== a != a :!= a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ":="))
-
-a = a := a == a === a :=¶= a != a :!= a
-
-##> (should (ess-jump-token "="))
-##> (should (ess-token-before= "=="))
-
-a = a := a == a === a :==¶ a != a :!= a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "!="))
-
-a = a := a == a === a :== a !=¶ a :!= a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ":"))
-
-a = a := a == a === a :== a != a :¶!= a
-
-##> (should (ess-test-token= "!="))
-
-a = a := a == a === a :== a != a :!=¶ a
-
-
-### 3 %% operators ---------------------------------------------------
-
-¶a %>% a %a`a`a"a"a$a@a% a %% a %%% a % a
-
-##! (ess-jump-token)
-##! (should (ess-test-token= "%infix%" "%>%"))
-
-a %>%¶ a %a`a`a"a"a$a@a% a %% a %%% a % a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "%infix%" "%a`a`a\"a\"a$a@a%"))
-
-a %>% a %a`a`a"a"a$a@a%¶ a %% a %%% a % a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "%%"))
-
-a %>% a %a`a`a"a"a$a@a% a %%¶ a %%% a % a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "%%"))
-
-a %>% a %a`a`a"a"a$a@a% a %% a %%¶% a % a
-
-##> (should (ess-test-token= "%infix%" "% a %"))
-
-a %>% a %a`a`a"a"a$a@a% a %% a %%% a %¶ a
-
-
-### 4 Arithmetic operators -------------------------------------------
-
-¶a + a - a - -a * a ** a ^ a ^ ++a
-
-##! (ess-jump-token)
-##! (should (ess-test-token= "+"))
-
-a +¶ a - a - -a * a ** a ^ a ^ ++a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "-"))
-
-a + a -¶ a - -a * a ** a ^ a ^ ++a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "-"))
-
-a + a - a -¶ -a * a ** a ^ a ^ ++a
-
-##> (should (ess-test-token= "-"))
-
-a + a - a - -¶a * a ** a ^ a ^ ++a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "*"))
-
-a + a - a - -a *¶ a ** a ^ a ^ ++a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "**"))
-
-a + a - a - -a * a **¶ a ^ a ^ ++a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "^"))
-
-a + a - a - -a * a ** a ^¶ a ^ ++a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "^"))
-
-a + a - a - -a * a ** a ^ a ^¶ ++a
-
-##> (should (ess-test-token= "+"))
-
-a + a - a - -a * a ** a ^ a ^ +¶+a
-
-##> (should (ess-test-token= "+"))
-
-a + a - a - -a * a ** a ^ a ^ ++¶a
-
-
-### 5 : operators ----------------------------------------------------
-
-¶a:  a::  a:::  a::::  a:::=
-
-##! (ess-jump-token)
-##! (should (ess-test-token= ":"))
-
-a:¶  a::  a:::  a::::  a:::=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "::"))
-
-a:  a::¶  a:::  a::::  a:::=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ":::"))
-
-a:  a::  a:::¶  a::::  a:::=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ":::"))
-
-a:  a::  a:::  a:::¶:  a:::=
-
-##> (should (ess-jump-token ":"))
-##> (should (ess-token-before= ":::"))
-
-a:  a::  a:::  a::::¶  a:::=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ":::"))
-
-a:  a::  a:::  a::::  a:::¶=
-
-
-### 6 Assignment operators -------------------------------------------
-
-¶a <-  a <<-  a -> >  a ->> a >> a
-
-##! (ess-jump-token)
-##! (should (ess-test-token= "<-"))
-
-a <-¶  a <<-  a -> >  a ->> a >> a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "<<-"))
-
-a <-  a <<-¶  a -> >  a ->> a >> a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "->"))
-
-a <-  a <<-  a ->¶ >  a ->> a >> a
-
-##> (should (ess-test-token= ">"))
-
-a <-  a <<-  a -> >¶  a ->> a >> a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "->>"))
-
-a <-  a <<-  a -> >  a ->>¶ a >> a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ">"))
-##> (should (ess-test-token= ">"))
-
-a <-  a <<-  a -> >  a ->> a >>¶ a
-
-
-### 7 Inequality operators -------------------------------------------
-
-¶a < >  a >=  a > =  a <=
-
-##! (ess-jump-token)
-##! (should (ess-test-token= "<"))
-
-a <¶ >  a >=  a > =  a <=
-
-##> (should (ess-test-token= ">"))
-
-a < >¶  a >=  a > =  a <=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ">="))
-
-a < >  a >=¶  a > =  a <=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= ">"))
-
-a < >  a >=  a >¶ =  a <=
-
-##> (should (ess-test-token= "="))
-
-a < >  a >=  a > =¶  a <=
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "<="))
-
-a < >  a >=  a > =  a <=¶
-
-
-### 8 Special operators ----------------------------------------------
-
-¶~a~~a
-
-##! (should (ess-test-token= "~"))
-
-~¶a~~a
-
-##> (ess-jump-token)
-##> (should (ess-test-token= "~"))
-##> (should (ess-test-token= "~"))
-
-~a~~¶a
-
-
-
-##### Token refinement
-
-### 1 Parameter assignment -------------------------------------------
-
-call(param ¶= NULL)
-
-##! (should (ess-test-token= "="))
-##! (should (ess-refined-token= (ess-token-before) "param-assign"))
-
-call(param =¶ NULL)
-
-
-### 2 Quoted parameter names -----------------------------------------
-
-call(¶"param" = NULL)
-
-##! (should (ess-test-token= "string" "\"param\""))
-##! (should (ess-refined-token= (ess-token-before) "identifier"))
-
-call("param"¶ = NULL)
-
-
-### 2 Quoted call names ----------------------------------------------
-
-¶"call"()
-
-##! (should (ess-test-token= "string" "\"call\""))
-##! (should (ess-refined-token= (ess-token-before) "identifier"))
-
-"call"¶()
-
-
-
-##### Blanks, Characters and Comments
-
-### 1 ----------------------------------------------------------------
-
-text¶
-text
-
-##! (ess-skip-blanks-forward t)
-
-text
-¶text
-
-
-### 2 ----------------------------------------------------------------
-
-text¶ # comment
-
-##! (when (not (ess-skip-blanks-forward t))
-##!   (insert "failure"))
-
-text ¶# comment
-
-##> (when (ess-skip-blanks-forward t)
-##>   (insert "failure"))
-
-text ¶# comment
-
-
-### 3 ----------------------------------------------------------------
-
-text
-¶text
-
-##! (ess-skip-blanks-backward)
-
-text
-¶text
-
-##! (ess-skip-blanks-backward t)
-
-text¶
-text
-
-
-### 4 Form Feed character --------------------------------------------
-
-text
-
-
-¶text
-
-##! (ess-skip-blanks-backward t)
-
-text¶
-
-
-text
-
diff --git a/test/literate/tokens.el b/test/literate/tokens.el
deleted file mode 100644
index c549b82df9..0000000000
--- a/test/literate/tokens.el
+++ /dev/null
@@ -1,8 +0,0 @@
-;; -*- lexical-binding: t; -*-
-
-(defun ess-test-token= (type &optional value)
-  "Check that the next token conforms to TYPE and VALUE.
-This checks it back and forth and moves the point after the
-token."
-  (and (ess-jump-token type value)
-       (ess-token-before= type value)))
diff --git a/test/run-tests b/test/run-tests
index 45115b6a78..73e93f4d64 100755
--- a/test/run-tests
+++ b/test/run-tests
@@ -45,9 +45,11 @@
   (ess-test-load-locally "ess-test-r-mode.el" ess-test-path)
   (ess-test-load-locally "ess-test-r-package.el" ess-test-path)
   (ess-test-load-locally "ess-test-r-syntax.el" ess-test-path)
+  (ess-test-load-locally "ess-test-r-token.el" ess-test-path)
   (ess-test-load-locally "ess-test-r.el" ess-test-path)
   (ess-test-load-locally "ess-test-roxy.el" ess-test-path)
-  (ess-test-load-locally "ess-test-rd.el" ess-test-path))
+  (ess-test-load-locally "ess-test-rd.el" ess-test-path)
+  )
 (when (member "--r-indent" argv)
   (ess-test-load-locally "ess-test-indentation.el" ess-test-path))
 (when (member "--r-pkg" argv)
@@ -56,7 +58,6 @@
   (ess-test-load-locally "ess-test-literate.el" ess-test-path)
   (ess-test-load-locally "test-etest.el" etest-path)
   ;; (elt-deftest test-ess-r-code-fill () "code-fill.R")
-  (elt-deftest test-ess-r-tokens () "tokens.R")
   (elt-deftest test-ess-r-fontification () "fontification.R")
   (elt-deftest test-ess-r-keybindings () "keybindings.R"))