Upgrade tokenizers to 0.19.1 to deal with breaking change in tokenizers (huggingface#266)

Co-authored-by: Johannes Vass <[email protected]>

Author: scriptator

Cargo.lock

@@ -31,7 +31,7 @@ insta = { git = "https://github.com/OlivierDehaene/insta", rev = "f4f98c0410b91f
 is_close = "0.1.3"
 hf-hub = "0.3.2"
 anyhow = "1.0.75"
-tokenizers = { version = "^0.15.0", default-features = false, features = ["onig", "esaxx_fast"] }
+tokenizers = { version = "^0.19.1", default-features = false, features = ["onig", "esaxx_fast"] }
 serial_test = "2.0.0"
 
 [build-dependencies]

backends/candle/Cargo.toml

@@ -10,6 +10,6 @@ hf-hub = { version = "^0.3.0", features = ["tokio"], default-features = false }
 metrics = "^0.21"
 text-embeddings-backend = { path = "../backends" }
 thiserror = "^1.0"
-tokenizers = { version = "^0.15.2", default-features = false, features = ["onig", "esaxx_fast"] }
+tokenizers = { version = "^0.19.1", default-features = false, features = ["onig", "esaxx_fast"] }
 tracing = "^0.1"
 tokio = { version = "^1.25", features = ["rt", "rt-multi-thread", "parking_lot", "sync"] }

core/Cargo.toml

@@ -32,7 +32,7 @@ reqwest = { version = "0.11.14", features = [] }
 serde = "1.0.152"
 serde_json = "1.0.93"
 thiserror = "1.0.38"
-tokenizers = { version = "0.15.2", default-features=false, features=["onig", "esaxx_fast"] }
+tokenizers = { version = "0.19.1", default-features=false, features=["onig", "esaxx_fast"] }
 tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
 tracing = "0.1.37"
 tracing-opentelemetry = "0.21.0"