Merge pull request #93 from SoapBox/query-string-parens

Mostly correct parsing for links containing or wrapped in brackets

Author: Nick Frasser

src/linkify/core/parser.js

@@ -33,7 +33,14 @@ TT_PROTOCOL		= TEXT_TOKENS.PROTOCOL,
 TT_QUERY		= TEXT_TOKENS.QUERY,
 TT_SLASH		= TEXT_TOKENS.SLASH,
 TT_SYM			= TEXT_TOKENS.SYM,
-TT_TLD			= TEXT_TOKENS.TLD;
+TT_TLD			= TEXT_TOKENS.TLD,
+TT_OPENBRACE	= TEXT_TOKENS.OPENBRACE,
+TT_OPENBRACKET	= TEXT_TOKENS.OPENBRACKET,
+TT_OPENPAREN	= TEXT_TOKENS.OPENPAREN,
+TT_CLOSEBRACE	= TEXT_TOKENS.CLOSEBRACE,
+TT_CLOSEBRACKET	= TEXT_TOKENS.CLOSEBRACKET,
+TT_CLOSEPAREN	= TEXT_TOKENS.CLOSEPAREN;
+
 // TT_WS 			= TEXT_TOKENS.WS;
 
 const
@@ -66,6 +73,15 @@ S_PSS_TLD_COLON			= makeState(), // (A) URL followed by colon (potential port nu
 S_PSS_TLD_PORT			= makeState(T_URL), // TLD followed by a port number
 S_URL					= makeState(T_URL), // Long URL with optional port and maybe query string
 S_URL_SYMS				= makeState(), // URL followed by some symbols (will not be part of the final URL)
+S_URL_OPENBRACE			= makeState(), // URL followed by {
+S_URL_OPENBRACKET		= makeState(), // URL followed by [
+S_URL_OPENPAREN			= makeState(), // URL followed by (
+S_URL_OPENBRACE_Q		= makeState(T_URL), // URL followed by { and some symbols that the URL can end it
+S_URL_OPENBRACKET_Q		= makeState(T_URL), // URL followed by [ and some symbols that the URL can end it
+S_URL_OPENPAREN_Q		= makeState(T_URL), // URL followed by ( and some symbols that the URL can end it
+S_URL_OPENBRACE_SYMS	= makeState(), // S_URL_OPENBRACE_Q followed by some symbols it cannot end it
+S_URL_OPENBRACKET_SYMS	= makeState(), // S_URL_OPENBRACKET_Q followed by some symbols it cannot end it
+S_URL_OPENPAREN_SYMS	= makeState(), // S_URL_OPENPAREN_Q followed by some symbols it cannot end it
 S_EMAIL_DOMAIN			= makeState(), // parsed string starts with local email info + @ with a potential domain name (C)
 S_EMAIL_DOMAIN_DOT		= makeState(), // (C) domain followed by DOT
 S_EMAIL					= makeState(T_EMAIL), // (C) Possible email address (could have more tlds)
@@ -144,8 +160,7 @@ let qsAccepting = [
 	TT_POUND,
 	TT_PROTOCOL,
 	TT_SLASH,
-	TT_TLD,
-	TT_SYM
+	TT_TLD
 ];
 
 // Types of tokens that can follow a URL and be part of the query string
@@ -155,9 +170,65 @@ let qsNonAccepting = [
 	TT_COLON,
 	TT_DOT,
 	TT_QUERY,
-	TT_PUNCTUATION
+	TT_PUNCTUATION,
+	TT_CLOSEBRACE,
+	TT_CLOSEBRACKET,
+	TT_CLOSEPAREN,
+	TT_OPENBRACE,
+	TT_OPENBRACKET,
+	TT_OPENPAREN,
+	TT_SYM
 ];
 
+// These states are responsible primarily for determining whether or not to
+// include the final round bracket.
+
+// URL, followed by an opening bracket
+S_URL.on(TT_OPENBRACE, S_URL_OPENBRACE);
+S_URL.on(TT_OPENBRACKET, S_URL_OPENBRACKET);
+S_URL.on(TT_OPENPAREN, S_URL_OPENPAREN);
+
+// URL with extra symbols at the end, followed by an opening bracket
+S_URL_SYMS.on(TT_OPENBRACE, S_URL_OPENBRACE);
+S_URL_SYMS.on(TT_OPENBRACKET, S_URL_OPENBRACKET);
+S_URL_SYMS.on(TT_OPENPAREN, S_URL_OPENPAREN);
+
+// Closing bracket component. This character WILL be included in the URL
+S_URL_OPENBRACE.on(TT_CLOSEBRACE, S_URL);
+S_URL_OPENBRACKET.on(TT_CLOSEBRACKET, S_URL);
+S_URL_OPENPAREN.on(TT_CLOSEPAREN, S_URL);
+S_URL_OPENBRACE_Q.on(TT_CLOSEBRACE, S_URL);
+S_URL_OPENBRACKET_Q.on(TT_CLOSEBRACKET, S_URL);
+S_URL_OPENPAREN_Q.on(TT_CLOSEPAREN, S_URL);
+S_URL_OPENBRACE_SYMS.on(TT_CLOSEBRACE, S_URL);
+S_URL_OPENBRACKET_SYMS.on(TT_CLOSEBRACKET, S_URL);
+S_URL_OPENPAREN_SYMS.on(TT_CLOSEPAREN, S_URL);
+
+// URL that beings with an opening bracket, followed by a symbols.
+// Note that the final state can still be `S_URL_OPENBRACE_Q` (if the URL only
+// has a single opening bracket for some reason).
+S_URL_OPENBRACE.on(qsAccepting, S_URL_OPENBRACE_Q);
+S_URL_OPENBRACKET.on(qsAccepting, S_URL_OPENBRACKET_Q);
+S_URL_OPENPAREN.on(qsAccepting, S_URL_OPENPAREN_Q);
+S_URL_OPENBRACE.on(qsNonAccepting, S_URL_OPENBRACE_SYMS);
+S_URL_OPENBRACKET.on(qsNonAccepting, S_URL_OPENBRACKET_SYMS);
+S_URL_OPENPAREN.on(qsNonAccepting, S_URL_OPENPAREN_SYMS);
+
+// URL that begins with an opening bracket, followed by some symbols
+S_URL_OPENBRACE_Q.on(qsAccepting, S_URL_OPENBRACE_Q);
+S_URL_OPENBRACKET_Q.on(qsAccepting, S_URL_OPENBRACKET_Q);
+S_URL_OPENPAREN_Q.on(qsAccepting, S_URL_OPENPAREN_Q);
+S_URL_OPENBRACE_Q.on(qsNonAccepting, S_URL_OPENBRACE_Q);
+S_URL_OPENBRACKET_Q.on(qsNonAccepting, S_URL_OPENBRACKET_Q);
+S_URL_OPENPAREN_Q.on(qsNonAccepting, S_URL_OPENPAREN_Q);
+
+S_URL_OPENBRACE_SYMS.on(qsAccepting, S_URL_OPENBRACE_Q);
+S_URL_OPENBRACKET_SYMS.on(qsAccepting, S_URL_OPENBRACKET_Q);
+S_URL_OPENPAREN_SYMS.on(qsAccepting, S_URL_OPENPAREN_Q);
+S_URL_OPENBRACE_SYMS.on(qsNonAccepting, S_URL_OPENBRACE_SYMS);
+S_URL_OPENBRACKET_SYMS.on(qsNonAccepting, S_URL_OPENBRACKET_SYMS);
+S_URL_OPENPAREN_SYMS.on(qsNonAccepting, S_URL_OPENPAREN_SYMS);
+
 // Account for the query string
 S_URL.on(qsAccepting, S_URL);
 S_URL_SYMS.on(qsAccepting, S_URL);

src/linkify/core/scanner.js

@@ -44,6 +44,12 @@ S_START.on('#', makeState(TOKENS.POUND));
 S_START.on('?', makeState(TOKENS.QUERY));
 S_START.on('/', makeState(TOKENS.SLASH));
 S_START.on(COLON, makeState(TOKENS.COLON));
+S_START.on('{', makeState(TOKENS.OPENBRACE));
+S_START.on('[', makeState(TOKENS.OPENBRACKET));
+S_START.on('(', makeState(TOKENS.OPENPAREN));
+S_START.on('}', makeState(TOKENS.CLOSEBRACE));
+S_START.on(']', makeState(TOKENS.CLOSEBRACKET));
+S_START.on(')', makeState(TOKENS.CLOSEPAREN));
 S_START.on(/[,;!]/, makeState(TOKENS.PUNCTUATION));
 
 // Whitespace jumps

src/linkify/core/tokens.js

@@ -161,6 +161,34 @@ class TLD extends TextToken {}
 */
 class WS extends TextToken {}
 
+/**
+	Opening/closing bracket classes
+*/
+
+class OPENBRACE extends TextToken {
+	constructor() { super('{'); }
+}
+
+class OPENBRACKET extends TextToken {
+	constructor() { super('['); }
+}
+
+class OPENPAREN extends TextToken {
+	constructor() { super('('); }
+}
+
+class CLOSEBRACE extends TextToken {
+	constructor() { super('}'); }
+}
+
+class CLOSEBRACKET extends TextToken {
+	constructor() { super(']'); }
+}
+
+class CLOSEPAREN extends TextToken {
+	constructor() { super(')'); }
+}
+
 let text = {
 	Base: TextToken,
 	DOMAIN,
@@ -178,7 +206,13 @@ let text = {
 	SLASH,
 	SYM,
 	TLD,
-	WS
+	WS,
+	OPENBRACE,
+	OPENBRACKET,
+	OPENPAREN,
+	CLOSEBRACE,
+	CLOSEBRACKET,
+	CLOSEPAREN
 };
 
 /******************************************************************************

test/spec/linkify/core/parser-test.js

@@ -37,7 +37,7 @@ var tests = [
 	], [
 		'This [i.imgur.com/ckSj2Ba.jpg)] should also work',
 		[TEXT, URL, TEXT],
-		['This [', 'i.imgur.com/ckSj2Ba.jpg)]', ' should also work']
+		['This [', 'i.imgur.com/ckSj2Ba.jpg', ')] should also work']
 	], [
 		'A link is http://nick.is.awesome/?q=nick+amazing&nick=yo%29%30hellp another is http://nick.con/?q=look',
 		[TEXT, URL, TEXT],
@@ -118,6 +118,18 @@ var tests = [
 		'Bu haritanın verileri Direniş İzleme Grubu\'nun yaptığı Türkiye İşçi Eylemleri haritası ile birleşebilir esasen. https://graphcommons.com/graphs/00af1cd8-5a67-40b1-86e5-32beae436f7c?show=Comments',
 		[TEXT, URL],
 		['Bu haritanın verileri Direniş İzleme Grubu\'nun yaptığı Türkiye İşçi Eylemleri haritası ile birleşebilir esasen. ', 'https://graphcommons.com/graphs/00af1cd8-5a67-40b1-86e5-32beae436f7c?show=Comments']
+	], [
+		'Links with brackets and parens https://en.wikipedia.org/wiki/Blur_[band] wat',
+		[TEXT, URL, TEXT],
+		['Links with brackets and parens ', 'https://en.wikipedia.org/wiki/Blur_[band]', ' wat'],
+	], [
+		'This has dots {https://msdn.microsoft.com/en-us/library/aa752574(VS.85).aspx}',
+		[TEXT, URL, TEXT],
+		['This has dots {', 'https://msdn.microsoft.com/en-us/library/aa752574(VS.85).aspx', '}']
+	], [ // This test is correct, will count nested brackets as being part of the first
+		'A really funky one (example.com/?id=asd2{hellow}and%20it%20continues(23&((@)) and it ends',
+		[TEXT, URL, TEXT],
+		['A really funky one (', 'example.com/?id=asd2{hellow}and%20it%20continues(23&((@)', ') and it ends']
 	]
 ];
 
@@ -131,13 +143,14 @@ describe('linkify/core/parser#run()', function () {
 			values = test[2],
 			result = parser.run(scanner.run(str));
 
+			expect(result.map(function (token) {
+				return token.toString();
+			})).to.eql(values);
+
 			expect(result.map(function (token) {
 				return token.constructor;
 			})).to.eql(types);
 
-			expect(result.map(function (token) {
-				return token.toString();
-			})).to.eql(values);
 		});
 	}
 

test/spec/linkify/core/scanner-test.js

@@ -18,7 +18,15 @@ QUERY		= TEXT_TOKENS.QUERY,
 SLASH		= TEXT_TOKENS.SLASH,
 SYM			= TEXT_TOKENS.SYM,
 TLD			= TEXT_TOKENS.TLD,
-WS			= TEXT_TOKENS.WS;
+WS			= TEXT_TOKENS.WS,
+
+OPENBRACE	= TEXT_TOKENS.OPENBRACE,
+OPENBRACKET	= TEXT_TOKENS.OPENBRACKET,
+OPENPAREN	= TEXT_TOKENS.OPENPAREN,
+CLOSEBRACE	= TEXT_TOKENS.CLOSEBRACE,
+CLOSEBRACKET	= TEXT_TOKENS.CLOSEBRACKET,
+CLOSEPAREN	= TEXT_TOKENS.CLOSEPAREN;
+
 
 // The elements are
 // 1. input string
@@ -35,7 +43,8 @@ var tests = [
 	['#', [POUND], ['#']],
 	['/', [SLASH], ['/']],
 	['&', [SYM], ['&']],
-	['&?<>(', [SYM, QUERY, SYM, SYM, SYM], ['&', '?', '<', '>', '(']],
+	['&?<>(', [SYM, QUERY, SYM, SYM, OPENPAREN], ['&', '?', '<', '>', '(']],
+	['([{}])', [OPENPAREN, OPENBRACKET, OPENBRACE, CLOSEBRACE, CLOSEBRACKET, CLOSEPAREN], ['(', '[', '{', '}', ']', ')']],
 	['!,;', [PUNCTUATION, PUNCTUATION, PUNCTUATION], ['!', ',', ';']],
 	['hello', [DOMAIN], ['hello']],
 	['Hello123', [DOMAIN], ['Hello123']],