[Bugfix] Offload blocking tokenizer ops to shared thread pool to unblock event loop (#34789)

Signed-off-by: Bvicii <yizhanhuang2002@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Bvicii
2026-03-26 22:17:00 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent d86060122a
commit 999dfc1622
15 changed files with 195 additions and 28 deletions
+3 -1
View File
@@ -34,6 +34,7 @@ class AsyncMicrobatchTokenizer:
tokenizer,
max_batch_size: int = 32,
batch_wait_timeout_s: float = 0.002,
executor: ThreadPoolExecutor | None = None,
) -> None:
self.tokenizer = tokenizer
self.max_batch_size = max_batch_size
@@ -47,7 +48,8 @@ class AsyncMicrobatchTokenizer:
self._batcher_tasks: list[Task] = []
# Single-thread executor for blocking tokenizer calls.
self._executor = ThreadPoolExecutor(max_workers=1)
# Accept an external executor to serialize with other tokenizer users.
self._executor = executor or ThreadPoolExecutor(max_workers=1)
# === Public async API ===
async def __call__(self, prompt, **kwargs) -> BatchEncoding: