URL followed by   now works as expected in linkify-html (#184)

* Add AMPERSAND character as first-class token

Rather than it belonging to the SYMS text text token

* Update simple-html-tokenizer to latest version

This is the actual fix for the bug. It includes an update to ensure
  is always parsed into its non-encoded form

* Add tests for   following URL

Author: Nick Frasser

package.json

@@ -63,7 +63,7 @@
     "requirejs": "^2.1.22",
     "rollup": "~0.40.0",
     "run-sequence": "^1.1.5",
-    "simple-html-tokenizer": "git+https://github.com/nfrasser/simple-html-tokenizer.git#a907972303094c14680d2fd9870965429a893398",
+    "simple-html-tokenizer": "git+https://github.com/nfrasser/simple-html-tokenizer.git#04799f4638ec5ed903a4e5aa6e832269fa59be6b",
     "through2": "^2.0.1"
   },
   "optionalDependencies": {

src/linkify-html.js

@@ -164,13 +164,15 @@ function skipTagTokens(tagName, tokens, i, skippedTokens) {
 
 	while (i < tokens.length && stackCount > 0) {
 		let token = tokens[i];
+
 		if (token.type === StartTag && token.tagName.toUpperCase() === tagName) {
 			// Nested tag of the same type, "add to stack"
 			stackCount++;
 		} else if (token.type === EndTag && token.tagName.toUpperCase() === tagName) {
 			// Closing tag
 			stackCount--;
 		}
+
 		skippedTokens.push(token);
 		i++;
 	}

src/linkify/core/parser.js

@@ -41,6 +41,7 @@ import {
 	CLOSEBRACKET,
 	CLOSEANGLEBRACKET,
 	CLOSEPAREN,
+	AMPERSAND
 } from './tokens/text';
 
 import {
@@ -105,7 +106,7 @@ S_START
 .on(LOCALHOST, S_TLD)
 .on(NUM, S_DOMAIN);
 
-// Force URL for anything sane followed by protocol
+// Force URL for protocol followed by anything sane
 S_PROTOCOL_SLASH_SLASH
 .on(TLD, S_URL)
 .on(DOMAIN, S_URL)
@@ -159,7 +160,8 @@ let qsAccepting = [
 	SLASH,
 	TLD,
 	UNDERSCORE,
-	SYM
+	SYM,
+	AMPERSAND
 ];
 
 // Types of tokens that can follow a URL and be part of the query string
@@ -262,6 +264,7 @@ let localpartAccepting = [
 	QUERY,
 	UNDERSCORE,
 	SYM,
+	AMPERSAND,
 	TLD
 ];
 

src/linkify/core/scanner.js

@@ -32,6 +32,7 @@ import {
 	CLOSEBRACKET,
 	CLOSEANGLEBRACKET,
 	CLOSEPAREN,
+	AMPERSAND,
 	PUNCTUATION,
 	NL,
 	SYM
@@ -71,6 +72,7 @@ S_START
 .on(']', makeState(CLOSEBRACKET))
 .on('>', makeState(CLOSEANGLEBRACKET))
 .on(')', makeState(CLOSEPAREN))
+.on('&', makeState(AMPERSAND))
 .on([',', ';', '!', '"', '\''], makeState(PUNCTUATION));
 
 // Whitespace jumps

src/linkify/core/tokens/text.js

@@ -160,6 +160,8 @@ const CLOSEBRACKET = inheritsToken(']');
 const CLOSEANGLEBRACKET = inheritsToken('>');
 const CLOSEPAREN = inheritsToken(')');
 
+const AMPERSAND = inheritsToken('&');
+
 export {
 	TextToken as Base,
 	DOMAIN,
@@ -186,5 +188,6 @@ export {
 	CLOSEBRACE,
 	CLOSEBRACKET,
 	CLOSEANGLEBRACKET,
-	CLOSEPAREN
+	CLOSEPAREN,
+	AMPERSAND
 };

test/spec/linkify-html-test.js

@@ -55,6 +55,10 @@ describe('linkify-html', () => {
 			'Ignore tags like <script>var a = {}; a.ca = "Hello";</script> and <style>b.com {color: blue;}</style>',
 			'Ignore tags like <script>var a = {}; <a href="http://a.ca" class="linkified" target="_blank">a.ca</a> = "Hello";</script> and <style><a href="http://b.com" class="linkified" target="_blank">b.com</a> {color: blue;}</style>',
 			'Ignore tags like <script>var a = {}; a.ca = "Hello";</script> and <style>b.com {color: blue;}</style>'
+		], [
+			'6. Link followed by nbsp escape sequence https://github.com&nbsp;',
+			'6. Link followed by nbsp escape sequence <a href="https://github.com" class="linkified" target="_blank">https://github.com</a>\u00a0',
+			'6. Link followed by nbsp escape sequence <span href="https://github.com" class="my-linkify-class" target="_parent" rel="nofollow" onclick="console.log(\'Hello World!\')">https://github.com</span>\u00a0'
 		]
 	];
 

test/spec/linkify/core/scanner-test.js

@@ -25,6 +25,7 @@ const CLOSEBRACE = TEXTTOKENS.CLOSEBRACE;
 const CLOSEBRACKET = TEXTTOKENS.CLOSEBRACKET;
 const CLOSEANGLEBRACKET = TEXTTOKENS.CLOSEANGLEBRACKET;
 const CLOSEPAREN = TEXTTOKENS.CLOSEPAREN;
+const AMPERSAND = TEXTTOKENS.AMPERSAND;
 
 // The elements are
 // 1. input string
@@ -40,8 +41,8 @@ const tests = [
 	['+', [PLUS], ['+']],
 	['#', [POUND], ['#']],
 	['/', [SLASH], ['/']],
-	['&', [SYM], ['&']],
-	['&?<>(', [SYM, QUERY, OPENANGLEBRACKET, CLOSEANGLEBRACKET, OPENPAREN], ['&', '?', '<', '>', '(']],
+	['&', [AMPERSAND], ['&']],
+	['&?<>(', [AMPERSAND, QUERY, OPENANGLEBRACKET, CLOSEANGLEBRACKET, OPENPAREN], ['&', '?', '<', '>', '(']],
 	['([{}])', [OPENPAREN, OPENBRACKET, OPENBRACE, CLOSEBRACE, CLOSEBRACKET, CLOSEPAREN], ['(', '[', '{', '}', ']', ')']],
 	['!,;\'', [PUNCTUATION, PUNCTUATION, PUNCTUATION, PUNCTUATION], ['!', ',', ';', '\'']],
 	['hello', [DOMAIN], ['hello']],