[Bugfix][Frontend] Preserve token IDs in chat derender responses (#61065)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -126,7 +126,7 @@ async def _disagg(
|
||||
],
|
||||
},
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"chat_request": chat_request,
|
||||
"chat_request": {"return_token_ids": True, **chat_request},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
@@ -149,6 +149,7 @@ def _assert_parity(coupled: dict, disagg: dict) -> None:
|
||||
assert d["message"].get("reasoning") == c["message"].get("reasoning")
|
||||
assert _tool_sig(d) == _tool_sig(c)
|
||||
assert d["finish_reason"] == c["finish_reason"]
|
||||
assert d["token_ids"] == c["token_ids"]
|
||||
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
|
||||
assert disagg["usage"]["completion_tokens"] == len(c["token_ids"])
|
||||
|
||||
@@ -386,7 +387,7 @@ async def _run_stream_parity_case(
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices]
|
||||
+ ([0] if finish_only_tail else []),
|
||||
{"model": MODEL, "messages": messages, **extra},
|
||||
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
|
||||
@@ -1311,7 +1311,7 @@ class TestDerenderChatStreamParsed:
|
||||
async def test_include_reasoning_false_suppresses_reasoning(
|
||||
self, parsed_derenderer
|
||||
):
|
||||
chat_request = _chat_request(include_reasoning=False)
|
||||
chat_request = _chat_request(include_reasoning=False, return_token_ids=True)
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
@@ -1320,6 +1320,7 @@ class TestDerenderChatStreamParsed:
|
||||
chat_request=chat_request,
|
||||
)
|
||||
delta = chunk.choices[0].delta
|
||||
assert chunk.choices[0].token_ids is None
|
||||
assert delta.reasoning is None
|
||||
assert delta.content == "c"
|
||||
|
||||
|
||||
@@ -95,6 +95,10 @@ async def stream_chat_derender(
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
if chat_request.get("return_token_ids") and chat_request.get(
|
||||
"include_reasoning", True
|
||||
):
|
||||
assert data["chunk"]["choices"][0]["token_ids"] == tids
|
||||
state = data["stream_state"]
|
||||
choices.extend(data["chunk"]["choices"])
|
||||
if on_chunk is not None:
|
||||
|
||||
@@ -160,6 +160,11 @@ class OnlineDerenderer:
|
||||
else {}
|
||||
)
|
||||
|
||||
return_token_ids = (
|
||||
chat_request is not None
|
||||
and chat_request.return_token_ids
|
||||
and (not has_parser or chat_request.include_reasoning)
|
||||
)
|
||||
for choice in generate_response.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
@@ -201,6 +206,7 @@ class OnlineDerenderer:
|
||||
message=message,
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
token_ids=choice.token_ids if return_token_ids else None,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -432,6 +438,7 @@ class OnlineDerenderer:
|
||||
stream_choices: list[ChatCompletionResponseStreamChoice] = []
|
||||
updated_state = state
|
||||
|
||||
return_token_ids = chat_request is not None and chat_request.return_token_ids
|
||||
for choice in generate_chunk.choices:
|
||||
delta_tids = choice.token_ids or []
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
@@ -477,6 +484,7 @@ class OnlineDerenderer:
|
||||
delta=delta,
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
token_ids=choice.token_ids if return_token_ids else None,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -583,6 +591,10 @@ class OnlineDerenderer:
|
||||
tools_streamed = state.tools_streamed
|
||||
last_tool_call_ids = list(state.last_tool_call_ids)
|
||||
|
||||
return_token_ids = (
|
||||
chat_request.return_token_ids and chat_request.include_reasoning
|
||||
)
|
||||
|
||||
# At most one choice: the caller (derender_chat_stream) already
|
||||
# rejects >1 before dispatching here. role_sent/tools_streamed/
|
||||
# output_token_ids below are updated for that single choice, not
|
||||
@@ -660,6 +672,7 @@ class OnlineDerenderer:
|
||||
index=choice.index,
|
||||
delta=delta_message,
|
||||
finish_reason=finish_reason,
|
||||
token_ids=choice.token_ids if return_token_ids else None,
|
||||
)
|
||||
stream_choices.append(
|
||||
maybe_filter_parallel_tool_calls(stream_choice, chat_request)
|
||||
|
||||
Reference in New Issue
Block a user