[Bugfix][Frontend] Leave the final stop token out of chat derender text (#61757)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Kevin H. Luu
2026-10-15 21:51:34 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Cyrus Leung
parent 71536bd5f8
commit b7b20b1b8f
2 changed files with 132 additions and 3 deletions
@@ -348,6 +348,85 @@ def test_non_streaming_derender_initializes_parser_prefix(
assert choices[0].finish_reason == "length"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("stop", "include_stop_str_in_output", "dropped"),
[
("eos", False, True),
("stop_token_id", False, True),
("stop_string", False, False),
("eos", True, False),
],
)
async def test_parsed_derender_leaves_out_stop_token_like_engine(
tokenizer, stop, include_stop_str_in_output, dropped
):
"""A parser keeps special tokens in the decoded text, so the token that
stopped generation must be dropped the way the engine drops it."""
eos = tokenizer.eos_token_id
answer_ids = tokenizer.encode("answer", add_special_tokens=False)
bang = tokenizer.encode("!", add_special_tokens=False)[-1]
last_id, stop_reason = {
"eos": (eos, None),
"stop_token_id": (bang, bang),
"stop_string": (bang, "!"),
}[stop]
parser = MagicMock()
parser.parse.side_effect = lambda text, *args, **kwargs: (None, text, [])
parser.parse_delta.side_effect = lambda text, *args, **kwargs: DeltaMessage(
content=text
)
request = _chat_request(include_stop_str_in_output=include_stop_str_in_output)
with ThreadPoolExecutor(max_workers=1) as executor:
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
renderer.get_eos_token_id.return_value = eos
renderer._executor = executor
model_config = MagicMock()
model_config.hf_config.model_type = "llama"
model_config.model = MODEL_NAME
model_config.try_get_generation_config.return_value = {}
derenderer = OnlineDerenderer(
model_config=model_config,
renderer=renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="string",
)
derenderer.parser = MagicMock(return_value=parser)
def batch_content(token_ids: list[int], finish_reason: str) -> str:
choice = GenerateTokensChoice(
index=0,
token_ids=token_ids,
finish_reason=finish_reason,
stop_reason=stop_reason if finish_reason == "stop" else None,
)
response = GenerateTokensResponse(
request_id="test", prompt_token_ids=[11, 12], choices=[choice]
)
return derenderer._derender_chat(response, request)[0].message.content
# Same decode window and seeding; "length" never drops a token.
expected = batch_content(
answer_ids if dropped else answer_ids + [last_id], "length"
)
batch = batch_content(answer_ids + [last_id], "stop")
stream, _ = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
answer_ids + [last_id], finish_reason="stop", stop_reason=stop_reason
),
chat_request=request,
prompt_token_ids=[11, 12],
)
assert batch == expected
assert stream.choices[0].delta.content == expected
class TestDetokenizeDelta:
"""_detokenize_delta: chunked decode must equal one shot decode."""