[Perf][Rust Frontend] Coalesce decoded chunks per engine update (#55012)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-02 15:44:23 -07:00
committed by GitHub
parent 443febe723
commit ad127d9a0f
+7 -4
View File
@@ -201,11 +201,14 @@ pub async fn decoded_text_event_stream(
break;
}
}
// TODO: avoid generating a chunk every time a token is pushed
if intermediate && let Some(chunk) = decoder.next_chunk() {
decoded.append(chunk);
}
// Coalesce output per nonterminal engine update; terminal updates flush below.
if intermediate
&& finish_reason.is_none()
&& let Some(chunk) = decoder.next_chunk()
{
decoded.append(chunk);
}
let mut new_token_ids = output.token_ids;