chore(deps): bump tokenizers from 0.21.4 to 0.22.2 (#9503)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
dependabot[bot]
2026-06-03 13:37:16 -04:00
committed by GitHub
parent 1205a4a042
commit 1a52f21c08
2 changed files with 4 additions and 37 deletions
Generated
+3 -36
View File
@@ -1732,7 +1732,7 @@ dependencies = [
"rayon",
"safetensors 0.7.0",
"thiserror 2.0.18",
"tokenizers 0.22.2",
"tokenizers",
"yoke 0.8.2",
"zip 7.2.0",
]
@@ -4543,7 +4543,7 @@ dependencies = [
"test-case",
"thiserror 1.0.69",
"tiktoken-rs",
"tokenizers 0.21.4",
"tokenizers",
"tokio",
"tokio-cron-scheduler",
"tokio-stream",
@@ -7692,7 +7692,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.13.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -10763,39 +10763,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "tokenizers"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
dependencies = [
"ahash",
"aho-corasick",
"compact_str 0.9.0",
"dary_heap",
"derive_builder",
"esaxx-rs",
"getrandom 0.3.4",
"itertools 0.14.0",
"log",
"macro_rules_attribute",
"monostate",
"onig",
"paste",
"rand 0.9.4",
"rayon",
"rayon-cond",
"regex",
"regex-syntax",
"serde",
"serde_json",
"spm_precompiled",
"thiserror 2.0.18",
"unicode-normalization-alignments",
"unicode-segmentation",
"unicode_categories",
]
[[package]]
name = "tokenizers"
version = "0.22.2"
+1 -1
View File
@@ -162,7 +162,7 @@ candle-core = { workspace = true, optional = true }
candle-nn = { workspace = true, optional = true }
candle-transformers = { version = "0.10", default-features = false, optional = true }
byteorder = { version = "1.5", default-features = false, features = ["std"], optional = true }
tokenizers = { version = "0.21", default-features = false, features = ["onig"], optional = true }
tokenizers = { version = "0.22", default-features = false, features = ["onig"], optional = true }
symphonia = { version = "0.5", default-features = false, features = ["aac", "adpcm", "alac", "isomp4", "mkv", "mp3", "pcm", "vorbis", "wav"], optional = true }
rubato = { version = "0.16", default-features = false, optional = true }
zip = { workspace = true }