File tree Expand file tree Collapse file tree 1 file changed +9
-5
lines changed
Expand file tree Collapse file tree 1 file changed +9
-5
lines changed Original file line number Diff line number Diff line change @@ -125,11 +125,15 @@ S_START.on(/./, makeState(TOKENS.SYM));
125125*/
126126let run = function ( str ) {
127127
128- let
129- lowerStr = str . replace ( / [ A - Z ] / g, c => c . toLowerCase ( ) ) , // The state machine only looks at lowercase strings
130- len = str . length ,
131- cursor = 0 ,
132- tokens = [ ] ; // return value
128+ // The state machine only looks at lowercase strings.
129+ // This selective `toLowerCase` is used because lowercasing the entire
130+ // string causes the length and character position to vary in some in some
131+ // non-English strings. This happens only on V8-based runtimes.
132+ let lowerStr = str . replace ( / [ A - Z ] / g, c => c . toLowerCase ( ) ) ;
133+ let len = str . length ;
134+ let tokens = [ ] ; // return value
135+
136+ var cursor = 0 ;
133137
134138 // Tokenize the string
135139 while ( cursor < len ) {
You can’t perform that action at this time.
0 commit comments