[Bugfix][Frontend] Preserve token IDs in chat derender responses (#61065)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
Ting SUN
2026-10-12 07:55:06 +00:00
committed by GitHub
parent e3bbaf2369
commit f4114e068e
4 changed files with 22 additions and 3 deletions
@@ -126,7 +126,7 @@ async def _disagg(
],
},
"prompt_tokens": prompt_tokens,
"chat_request": chat_request,
"chat_request": {"return_token_ids": True, **chat_request},
},
)
assert resp.status_code == 200, resp.text
@@ -149,6 +149,7 @@ def _assert_parity(coupled: dict, disagg: dict) -> None:
assert d["message"].get("reasoning") == c["message"].get("reasoning")
assert _tool_sig(d) == _tool_sig(c)
assert d["finish_reason"] == c["finish_reason"]
assert d["token_ids"] == c["token_ids"]
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
assert disagg["usage"]["completion_tokens"] == len(c["token_ids"])
@@ -386,7 +387,7 @@ async def _run_stream_parity_case(
[tid for ch in coupled_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in coupled_choices]
+ ([0] if finish_only_tail else []),
{"model": MODEL, "messages": messages, **extra},
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
@@ -1311,7 +1311,7 @@ class TestDerenderChatStreamParsed:
async def test_include_reasoning_false_suppresses_reasoning(
self, parsed_derenderer
):
chat_request = _chat_request(include_reasoning=False)
chat_request = _chat_request(include_reasoning=False, return_token_ids=True)
chunk, _ = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
@@ -1320,6 +1320,7 @@ class TestDerenderChatStreamParsed:
chat_request=chat_request,
)
delta = chunk.choices[0].delta
assert chunk.choices[0].token_ids is None
assert delta.reasoning is None
assert delta.content == "c"
@@ -95,6 +95,10 @@ async def stream_chat_derender(
)
assert resp.status_code == 200, resp.text
data = resp.json()
if chat_request.get("return_token_ids") and chat_request.get(
"include_reasoning", True
):
assert data["chunk"]["choices"][0]["token_ids"] == tids
state = data["stream_state"]
choices.extend(data["chunk"]["choices"])
if on_chunk is not None:
+13
View File
@@ -160,6 +160,11 @@ class OnlineDerenderer:
else {}
)
return_token_ids = (
chat_request is not None
and chat_request.return_token_ids
and (not has_parser or chat_request.include_reasoning)
)
for choice in generate_response.choices:
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
@@ -201,6 +206,7 @@ class OnlineDerenderer:
message=message,
logprobs=resolved_logprobs,
finish_reason=choice.finish_reason,
token_ids=choice.token_ids if return_token_ids else None,
)
)
@@ -432,6 +438,7 @@ class OnlineDerenderer:
stream_choices: list[ChatCompletionResponseStreamChoice] = []
updated_state = state
return_token_ids = chat_request is not None and chat_request.return_token_ids
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = await self._detokenize_delta_async(
@@ -477,6 +484,7 @@ class OnlineDerenderer:
delta=delta,
logprobs=resolved_logprobs,
finish_reason=choice.finish_reason,
token_ids=choice.token_ids if return_token_ids else None,
)
)
@@ -583,6 +591,10 @@ class OnlineDerenderer:
tools_streamed = state.tools_streamed
last_tool_call_ids = list(state.last_tool_call_ids)
return_token_ids = (
chat_request.return_token_ids and chat_request.include_reasoning
)
# At most one choice: the caller (derender_chat_stream) already
# rejects >1 before dispatching here. role_sent/tools_streamed/
# output_token_ids below are updated for that single choice, not
@@ -660,6 +672,7 @@ class OnlineDerenderer:
index=choice.index,
delta=delta_message,
finish_reason=finish_reason,
token_ids=choice.token_ids if return_token_ids else None,
)
stream_choices.append(
maybe_filter_parallel_tool_calls(stream_choice, chat_request)