From 8882b83b34718ae8103a178ac1dfa84bb502cb29 Mon Sep 17 00:00:00 2001 From: djramic Date: Fri, 2 Oct 2026 20:33:50 +0200 Subject: [PATCH] [Bugfix] Bump tokenizers to 0.23.2 for duplicate-pattern support (#59796) Signed-off-by: Djordje Ramic --- requirements/common.txt | 2 +- requirements/test/cpu.txt | 2 +- requirements/test/cuda.txt | 2 +- requirements/test/rocm.txt | 2 +- requirements/test/xpu.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index b6b33efa92..6e0b0db783 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -9,7 +9,7 @@ blake3 py-cpuinfo transformers >= 5.16.1, < 5.18.0 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export -tokenizers >= 0.22.0 # Required for fast incremental detokenization. +tokenizers >= 0.23.2 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.*; platform_machine != "s390x" protobuf == 7.36.1; platform_machine == "s390x" diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 11fc2991ed..926a5ceb1c 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -1165,7 +1165,7 @@ timm==1.0.17 # -r requirements/test/cuda.in # open-clip-torch # segmentation-models-pytorch -tokenizers==0.23.1 +tokenizers==0.23.2 # via # -r requirements/test/../common.txt # cohere diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index cebfd726d5..8f39bb900e 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1266,7 +1266,7 @@ timm==1.0.17 # -r requirements/test/cuda.in # open-clip-torch # segmentation-models-pytorch -tokenizers==0.23.1 +tokenizers==0.23.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 3714afaf9d..0df6f9c82a 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -1249,7 +1249,7 @@ timm==1.0.17 # -r requirements/test/rocm.in # open-clip-torch # segmentation-models-pytorch -tokenizers==0.23.1 +tokenizers==0.23.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 01a0fda14b..33b89a85fc 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1089,7 +1089,7 @@ timm==1.0.28 # via # -r requirements/test/xpu.in # open-clip-torch -tokenizers==0.23.1 +tokenizers==0.23.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt