[Bugfix] Bump tokenizers to 0.23.2 for duplicate-pattern support (#59796)

Signed-off-by: Djordje Ramic <djoramic@amd.com>

requirements/common.txt

@@ -9,7 +9,7 @@ blake3
 py-cpuinfo
 transformers >= 5.16.1, < 5.18.0
 huggingface_hub >= 1.31.0  # Required for the `huggingface_hub.utils.httpx` re-export
-tokenizers >= 0.22.0  # Required for fast incremental detokenization.
+tokenizers >= 0.23.2  # Required for fast incremental detokenization.
 safetensors >= 0.6.2  # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
 protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.*; platform_machine != "s390x"
 protobuf == 7.36.1; platform_machine == "s390x"

requirements/test/cpu.txt

@@ -1165,7 +1165,7 @@ timm==1.0.17
     #   -r requirements/test/cuda.in
     #   open-clip-torch
     #   segmentation-models-pytorch
-tokenizers==0.23.1
+tokenizers==0.23.2
     # via
     #   -r requirements/test/../common.txt
     #   cohere

requirements/test/cuda.txt

@@ -1266,7 +1266,7 @@ timm==1.0.17
     #   -r requirements/test/cuda.in
     #   open-clip-torch
     #   segmentation-models-pytorch
-tokenizers==0.23.1
+tokenizers==0.23.2
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt

requirements/test/rocm.txt

@@ -1249,7 +1249,7 @@ timm==1.0.17
     #   -r requirements/test/rocm.in
     #   open-clip-torch
     #   segmentation-models-pytorch
-tokenizers==0.23.1
+tokenizers==0.23.2
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt

requirements/test/xpu.txt

@@ -1089,7 +1089,7 @@ timm==1.0.28
     # via
     #   -r requirements/test/xpu.in
     #   open-clip-torch
-tokenizers==0.23.1
+tokenizers==0.23.2
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt