[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)
Signed-off-by: Bvicii <yizhanhuang2002@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -291,6 +291,10 @@ class ModelConfig:
|
||||
definitions"""
|
||||
io_processor_plugin: str | None = None
|
||||
"""IOProcessor plugin name to load at model startup"""
|
||||
renderer_num_workers: int = 1
|
||||
"""Number of worker threads in the renderer thread pool. This pool
|
||||
handles async tokenization, chat template rendering, and multimodal
|
||||
preprocessing."""
|
||||
|
||||
# Pooler config
|
||||
pooler_config: PoolerConfig | None = None
|
||||
|
||||
Reference in New Issue
Block a user