[Bugfix][Frontend] Leave the final stop token out of chat derender text (#61757)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 5.5
Cyrus Leung
parent
71536bd5f8
commit
b7b20b1b8f
@@ -348,6 +348,85 @@ def test_non_streaming_derender_initializes_parser_prefix(
|
||||
assert choices[0].finish_reason == "length"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("stop", "include_stop_str_in_output", "dropped"),
|
||||
[
|
||||
("eos", False, True),
|
||||
("stop_token_id", False, True),
|
||||
("stop_string", False, False),
|
||||
("eos", True, False),
|
||||
],
|
||||
)
|
||||
async def test_parsed_derender_leaves_out_stop_token_like_engine(
|
||||
tokenizer, stop, include_stop_str_in_output, dropped
|
||||
):
|
||||
"""A parser keeps special tokens in the decoded text, so the token that
|
||||
stopped generation must be dropped the way the engine drops it."""
|
||||
eos = tokenizer.eos_token_id
|
||||
answer_ids = tokenizer.encode("answer", add_special_tokens=False)
|
||||
bang = tokenizer.encode("!", add_special_tokens=False)[-1]
|
||||
last_id, stop_reason = {
|
||||
"eos": (eos, None),
|
||||
"stop_token_id": (bang, bang),
|
||||
"stop_string": (bang, "!"),
|
||||
}[stop]
|
||||
|
||||
parser = MagicMock()
|
||||
parser.parse.side_effect = lambda text, *args, **kwargs: (None, text, [])
|
||||
parser.parse_delta.side_effect = lambda text, *args, **kwargs: DeltaMessage(
|
||||
content=text
|
||||
)
|
||||
request = _chat_request(include_stop_str_in_output=include_stop_str_in_output)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
renderer.get_eos_token_id.return_value = eos
|
||||
renderer._executor = executor
|
||||
model_config = MagicMock()
|
||||
model_config.hf_config.model_type = "llama"
|
||||
model_config.model = MODEL_NAME
|
||||
model_config.try_get_generation_config.return_value = {}
|
||||
derenderer = OnlineDerenderer(
|
||||
model_config=model_config,
|
||||
renderer=renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="string",
|
||||
)
|
||||
derenderer.parser = MagicMock(return_value=parser)
|
||||
|
||||
def batch_content(token_ids: list[int], finish_reason: str) -> str:
|
||||
choice = GenerateTokensChoice(
|
||||
index=0,
|
||||
token_ids=token_ids,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason if finish_reason == "stop" else None,
|
||||
)
|
||||
response = GenerateTokensResponse(
|
||||
request_id="test", prompt_token_ids=[11, 12], choices=[choice]
|
||||
)
|
||||
return derenderer._derender_chat(response, request)[0].message.content
|
||||
|
||||
# Same decode window and seeding; "length" never drops a token.
|
||||
expected = batch_content(
|
||||
answer_ids if dropped else answer_ids + [last_id], "length"
|
||||
)
|
||||
batch = batch_content(answer_ids + [last_id], "stop")
|
||||
stream, _ = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
answer_ids + [last_id], finish_reason="stop", stop_reason=stop_reason
|
||||
),
|
||||
chat_request=request,
|
||||
prompt_token_ids=[11, 12],
|
||||
)
|
||||
|
||||
assert batch == expected
|
||||
assert stream.choices[0].delta.content == expected
|
||||
|
||||
|
||||
class TestDetokenizeDelta:
|
||||
"""_detokenize_delta: chunked decode must equal one shot decode."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user