Merge pull request #218 from bzz/tokenizer-flex-cgo

New, optional flex-based tokenizer

Author: bzz

.gitignore

@@ -8,3 +8,4 @@ Makefile.main
 build/
 vendor/
 java/lib/
+.vscode/

internal/tokenizer/common.go

@@ -0,0 +1,7 @@
+// Package tokenizer implements file tokenization used by the enry content
+// classifier. This package is an implementation detail of enry and should not
+// be imported by other packages.
+package tokenizer
+
+// ByteLimit defines the maximum prefix of an input text that will be tokenized.
+const ByteLimit = 100000