[Bugfix][Frontend] Preserve token IDs in chat derender responses (#61065)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -126,7 +126,7 @@ async def _disagg(
|
||||
],
|
||||
},
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"chat_request": chat_request,
|
||||
"chat_request": {"return_token_ids": True, **chat_request},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
@@ -149,6 +149,7 @@ def _assert_parity(coupled: dict, disagg: dict) -> None:
|
||||
assert d["message"].get("reasoning") == c["message"].get("reasoning")
|
||||
assert _tool_sig(d) == _tool_sig(c)
|
||||
assert d["finish_reason"] == c["finish_reason"]
|
||||
assert d["token_ids"] == c["token_ids"]
|
||||
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
|
||||
assert disagg["usage"]["completion_tokens"] == len(c["token_ids"])
|
||||
|
||||
@@ -386,7 +387,7 @@ async def _run_stream_parity_case(
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices]
|
||||
+ ([0] if finish_only_tail else []),
|
||||
{"model": MODEL, "messages": messages, **extra},
|
||||
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
|
||||
@@ -1311,7 +1311,7 @@ class TestDerenderChatStreamParsed:
|
||||
async def test_include_reasoning_false_suppresses_reasoning(
|
||||
self, parsed_derenderer
|
||||
):
|
||||
chat_request = _chat_request(include_reasoning=False)
|
||||
chat_request = _chat_request(include_reasoning=False, return_token_ids=True)
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
@@ -1320,6 +1320,7 @@ class TestDerenderChatStreamParsed:
|
||||
chat_request=chat_request,
|
||||
)
|
||||
delta = chunk.choices[0].delta
|
||||
assert chunk.choices[0].token_ids is None
|
||||
assert delta.reasoning is None
|
||||
assert delta.content == "c"
|
||||
|
||||
|
||||
@@ -95,6 +95,10 @@ async def stream_chat_derender(
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
if chat_request.get("return_token_ids") and chat_request.get(
|
||||
"include_reasoning", True
|
||||
):
|
||||
assert data["chunk"]["choices"][0]["token_ids"] == tids
|
||||
state = data["stream_state"]
|
||||
choices.extend(data["chunk"]["choices"])
|
||||
if on_chunk is not None:
|
||||
|
||||
Reference in New Issue
Block a user