[Perf][Frontend] Offload streaming derender detokenization (#57528)

Signed-off-by: Shrey Gajjar <shreygajjar007@gmail.com>
This commit is contained in:
SHREY GAJJAR
2026-09-24 10:39:21 +00:00
committed by GitHub
parent f34a0e0797
commit 5747d4500a
2 changed files with 56 additions and 11 deletions
+6 -4
View File
@@ -101,8 +101,10 @@ class OnlineDerenderer:
self._derender_completion_async = make_async(
self._derender_completion, executor=renderer._executor
)
# Replay is O(n) per chunk (unlike the O(delta) detok-only paths),
# so it must not run on the event loop either.
self._detokenize_delta_async = make_async(
self._detokenize_delta, executor=renderer._executor
)
# Replay is O(n) per chunk, so it must not run on the event loop.
self._derender_chat_stream_parsed_async = make_async(
self._derender_chat_stream_parsed, executor=renderer._executor
)
@@ -362,7 +364,7 @@ class OnlineDerenderer:
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = self._detokenize_delta(
new_text, updated_state = await self._detokenize_delta_async(
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
)
@@ -716,7 +718,7 @@ class OnlineDerenderer:
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = self._detokenize_delta(
new_text, updated_state = await self._detokenize_delta_async(
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
)
stream_choices.append(