[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)

Signed-off-by: Bvicii <yizhanhuang2002@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Bvicii
2026-03-26 22:17:00 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent d86060122a
commit 999dfc1622
15 changed files with 195 additions and 28 deletions
+4
View File
@@ -291,6 +291,10 @@ class ModelConfig:
definitions"""
io_processor_plugin: str | None = None
"""IOProcessor plugin name to load at model startup"""
renderer_num_workers: int = 1
"""Number of worker threads in the renderer thread pool. This pool
handles async tokenization, chat template rendering, and multimodal
preprocessing."""
# Pooler config
pooler_config: PoolerConfig | None = None