Support Clojure 1.12 array class tokens (#290)

* Support Clojure 1.12 array class tokens

The new `foo/3` was reader-illegal prior to 1.12.

Bring in and customize some tools.reader bits and bobs to support
reading this new syntax.

Clojure's `symbol` allows us to construct technically illegal symbols,
we take advantage of this to test and support this new syntax in any
version of Clojure via, for example, `(symbol "foo" "3")`.

Closes #279

* fix typo [skip ci]

Author: lread

CHANGELOG.adoc

@@ -24,7 +24,10 @@ A release with known breaking changes is marked with:
 * `sexpr` now 1) expands tag metadata to its long form 2) throws on invalid metadata
 https://github.com/clj-commons/rewrite-clj/issues/280[#280]
 (@lread)
-* Add support for Clojure 1.12 vector metadata
+* Add support for Clojure 1.12 vector metadata (ex. `(^[double] String/valueOf 3)` )
+https://github.com/clj-commons/rewrite-clj/issues/279[#279]
+(@lread)
+* Add support for Clojure 1.12 array class syntax (ex. `byte/3`)
 https://github.com/clj-commons/rewrite-clj/issues/279[#279]
 (@lread)
 * `rewrite-clj.paredit/barf-forward` on zipper created with `:track-position? true` now correctly barfs when current node has children

src/rewrite_clj/interop.cljc

@@ -39,3 +39,12 @@
   [data]
   #?(:clj (instance? clojure.lang.IMeta data)
      :cljs (implements? IWithMeta data)))
+
+(defn numeric?
+  "Checks whether a given character is numeric
+
+  Cribbed from clojure/cljs.tools.reader.impl.util."
+  [^Character ch]
+  (when ch
+    #?(:clj (Character/isDigit ch)
+       :cljs (gstring/isNumeric ch))))

src/rewrite_clj/parser/token.cljc

@@ -1,5 +1,6 @@
 (ns ^:no-doc rewrite-clj.parser.token
-  (:require [rewrite-clj.node.token :as ntoken]
+  (:require [rewrite-clj.interop :as interop]
+            [rewrite-clj.node.token :as ntoken]
             [rewrite-clj.reader :as r]))
 
 #?(:clj (set! *warn-on-reflection* true))
@@ -31,18 +32,51 @@
       (ntoken/token-node value value-string)
       (let [s (str value-string suffix)]
         (ntoken/token-node
-          (r/string->edn s)
+          (r/read-symbol s)
           s)))))
 
+(defn- number-literal?
+  "Checks whether the reader is at the start of a number literal
+
+  Cribbed and adapted from clojure.tools.reader.impl.commons"
+  [[c1 c2]]
+  (or (interop/numeric? c1)
+      (and (or (identical? \+ c1) (identical? \- c1))
+           (interop/numeric? c2))))
+
 (defn parse-token
-  "Parse a single token."
+  "Parse a single token. For example: symbol, number or character."
   [#?(:cljs ^not-native reader :default reader)]
   (let [first-char (r/next reader)
         s (->> (if (= first-char \\)
                  (read-to-char-boundary reader)
                  (read-to-boundary reader))
                (str first-char))
-        v (r/string->edn s)]
+        v (if (or (= first-char \\) ;; character like \n or \newline
+                  (= first-char \#) ;; something like ##Inf, ##Nan
+                  (number-literal? s))
+            (r/string->edn s)
+            (r/read-symbol s))]
     (if (symbol? v)
       (symbol-node reader v s)
       (ntoken/token-node v s))))
+
+(comment
+  (require '[clojure.tools.reader.reader-types :as rt])
+
+  (parse-token (rt/string-push-back-reader "foo"))
+  ;; => {:value foo, :string-value "foo", :map-qualifier nil}
+
+  (parse-token (rt/string-push-back-reader "42"))
+  ;; => {:value 42, :string-value "42"}
+
+  (parse-token (rt/string-push-back-reader "+42"))
+  ;; => {:value 42, :string-value "+42"}
+
+  (parse-token (rt/string-push-back-reader "\\newline"))
+  ;; => {:value \newline, :string-value "\\newline"}
+
+  (parse-token (rt/string-push-back-reader "##Inf"))
+  ;; => {:value ##Inf, :string-value "##Inf"}
+
+  :eoc)

src/rewrite_clj/reader.cljc

@@ -1,6 +1,7 @@
 (ns ^:no-doc rewrite-clj.reader
   (:refer-clojure :exclude [peek next])
-  (:require #?@(:clj [[clojure.java.io :as io]])
+  (:require [clojure.string :as string]
+            #?@(:clj [[clojure.java.io :as io]])
             [clojure.tools.reader.edn :as edn]
             [clojure.tools.reader.impl.commons :as reader-impl-commons]
             [clojure.tools.reader.impl.errors :as reader-impl-errors]
@@ -203,6 +204,54 @@
           (reader-impl-errors/throw-invalid reader :keyword token)))
       (reader-impl-errors/throw-single-colon reader))))
 
+(defn- parse-symbol
+  "Parses a string into a vector of the namespace and symbol
+
+  Cribbed from clojure/cljs.tools.reader.impl.commons/parse-symbol merging clj and cljs fns into single implementation
+  Added in equivalent of TRDR-73 patch to allow array class symbols (e.g. foobar/3)."
+  [^String token]
+  (when-not (or (identical? "" token)
+                (string/ends-with? token ":")
+                (string/starts-with? token "::"))
+    (if-let [ns-idx (string/index-of token "/")]
+      (let [ns (subs token 0 ns-idx)
+            ns-idx (inc ns-idx)]
+          (when-not (== ns-idx (count token))
+            (let [sym (subs token ns-idx)]
+              (cond
+                (contains? #{"1" "2" "3" "4" "5" "6" "7" "8" "9"} sym)
+                [ns sym]
+
+                (and (not (interop/numeric? (nth sym 0)))
+                     (not (identical? "" sym))
+                     (not (string/ends-with? ns ":"))
+                     (or (identical? sym "/")
+                         (nil? (string/index-of sym "/"))))
+                [ns sym]))))
+      (when (or (identical? token "/")
+                  (nil? (string/index-of token "/")))
+          [nil token]))))
+
+(defn read-symbol
+  "Return symbol parsed from `token`.
+
+  Cribbed from clojure/cljs.tools.reader.edn/read-symbol and - adapted to work on string"
+  [^String token]
+  (case token
+    ;; special symbols
+    "nil" nil
+    "true" true
+    "false" false
+    "/" '/
+
+    (or (when-let [p (parse-symbol token)]
+          (symbol (p 0) (p 1)))
+        ;; Throw in same way that tools.reader would when reading a string
+        ;; for exeption compatibility. Some users, like clojure-lsp, currently rely
+        ;; on parsing exception strings. A user having to resort
+        ;; to parsing exception messages is not great, but a separate issue.
+        (reader-impl-errors/throw-invalid nil :symbol token))))
+
 ;; ## Reader Types
 
 ;;

test/rewrite_clj/node/coercer_test.cljc

@@ -23,6 +23,7 @@
 
              ;; symbol/keyword/string/...
              ['symbol                :token      :symbol]
+             [(symbol "ns" "3")      :token      :symbol]
              ['namespace/symbol      :token      :symbol]
              [:keyword               :token      :keyword]
              [:1.5.1                 :token      :keyword]

test/rewrite_clj/node_test.cljc

@@ -24,6 +24,7 @@
              ["[1 2 3]"         :vector         :seq                true]
              ["\"string\""      :token          :string             true]
              ["symbol"          :token          :symbol             true]
+             ["foo/3"           :token          :symbol             true]
              ["43"              :token          :token              true]
              ["#_ nope"         :uneval         :uneval             false]
              ["  "              :whitespace     :whitespace         false]]]
@@ -201,3 +202,21 @@
       (is (= 'my.current.ns (custom-mqn-sexpr "#:: {:a 1 :b 2}")))
       (is (= 'my.aliased.ns (custom-mqn-sexpr "#::my-alias {:a 1 :b 2}")))
       (is (= 'my-alias-nope-unresolved (custom-mqn-sexpr "#::my-alias-nope {:a 1 :b 2}"))))))
+
+(deftest t-create-nodes
+  ;; The *-node creation fns were initially created to serve the parser, and are therefore not
+  ;; always entirely user-friendly, but are often useful.
+  ;; Here's a good place to add some tests as we see fit.
+  (doseq
+   [[n expected-node-type expected-str expected-sexpr]
+    [[(n/token-node (symbol "foobar" "3")) :symbol "foobar/3"        (symbol "foobar" "3")]
+     [(n/token-node (symbol "sym"))        :symbol "sym"             'sym]
+     [(n/token-node '\newline "\\newline") :token  "\\newline"       '\newline]
+     [(n/token-node 42)                    :token  "42"              42]
+     [(n/token-node +42 "+42")             :token  "+42"             42]
+     [(n/token-node "foo")                 :token  "\"foo\""         "foo"]]]
+
+    (testing expected-str
+      (is (= expected-node-type (proto/node-type n)))
+      (is (= expected-str (n/string n)))
+      (is (= expected-sexpr (n/sexpr n))))))

test/rewrite_clj/parser_test.cljc

@@ -52,6 +52,7 @@
            ["\\\\"                       \\]
            ["\\a"                        \a]
            ["\\space"                    \space]
+           ["\\u2202"                    \u2202]
            ["\\'"                        \']
            [":1.5"                       :1.5]
            [":1.5.0"                     :1.5.0]
@@ -69,6 +70,15 @@
       (is (= s (node/string n)))
       (is (= r (node/sexpr n))))))
 
+(deftest t-parsing-clojure-1-12-array-class-tokens
+  (doseq [dimension (range 1 10)]
+    (let [s (str "foobar/" dimension)
+          n (p/parse-string s)]
+      (is (= :token (node/tag n)))
+      (is (= s (node/string n)))
+      (is (= (symbol "foobar" (str dimension))
+             (node/sexpr n))))))
+
 (deftest t-parsing-garden-selectors
   ;; https://github.com/noprompt/garden
   (doseq [[s expected-r]
@@ -564,6 +574,8 @@
            ["\"abc"                 #".*EOF.*"]
            ["#\"abc"                #".*Unexpected EOF.*"]
            ["(def x 0]"             #".*Unmatched delimiter.*"]
+           ["foobar/0"              #".*Invalid symbol: foobar/0"]  ;; array class dimension can be 1 to 9
+           ["foobar/11"             #".*Invalid symbol: foobar/11"] ;; array class dimension can be 1 to 9
            ["##wtf"                 #".*Invalid token: ##wtf"]
            ["#="                    #".*:eval node expects 1 value.*"]
            ["#^"                    #".*:meta node expects 2 values.*"]
@@ -581,7 +593,7 @@
            ["#:: token"             #".*namespaced map expects a map*"]
            ["#::alias [a]"          #".*namespaced map expects a map*"]
            ["#:prefix [a]"          #".*namespaced map expects a map.*"]]]
-       (is (thrown-with-msg? ExceptionInfo p (p/parse-string s)))))
+       (is (thrown-with-msg? ExceptionInfo p (p/parse-string s)) s)))
 
 (deftest t-sexpr-exceptions
   (doseq [s ["#_42"                 ;; reader ignore/discard