[Frontend] Report stop_reason on /inference/v1/generate and pass it through derender (#60251)

Signed-off-by: Hyeonki Hong <hhk7734@gmail.com>
Signed-off-by: Shimi Bandiel <shimib@google.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Hyeonki Hong <hhk7734@gmail.com>
Co-authored-by: Mimir <mimir@nousresearch.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Shimi Bandiel
2026-10-12 19:37:53 +08:00
committed by GitHub
co-authored by Hyeonki Hong Mimir Claude Bugen Zhao
parent 32b05addc1
commit 8d7b93fec2
8 changed files with 201 additions and 2 deletions
@@ -112,6 +112,38 @@ async def test_derender_chat_roundtrip(client):
assert data["choices"][0]["message"]["role"] == "assistant"
@pytest.mark.asyncio
async def test_derender_chat_passes_stop_reason_through(client):
"""The coupled server reports the stop string or token id that ended
generation; derender must carry the generate value through unchanged."""
gen_req = await _render_chat(client)
generate_response = _make_generate_response(gen_req["token_ids"][:3])
generate_response["choices"][0]["stop_reason"] = "\n\n"
response = await client.post(
"/v1/chat/completions/derender",
json={"model": MODEL_NAME, "generate_response": generate_response},
)
assert response.status_code == 200
assert response.json()["choices"][0]["stop_reason"] == "\n\n"
@pytest.mark.asyncio
async def test_derender_completion_passes_stop_reason_through(client):
gen_req = await _render_completion(client, "Hello world")
generate_response = _make_completion_generate_response(
gen_req["token_ids"][:3], "gen-stop"
)
generate_response["choices"][0]["stop_reason"] = 151645
response = await client.post(
"/v1/completions/derender",
json={"model": MODEL_NAME, "generate_responses": [generate_response]},
)
assert response.status_code == 200
assert response.json()["choices"][0]["stop_reason"] == 151645
@pytest.mark.asyncio
async def test_derender_chat_usage(client):
"""Supplied prompt_tokens flows through into usage correctly."""
@@ -195,6 +195,7 @@ def _make_stream_chunk(
metrics: PerRequestMetrics | None = None,
logprobs: dict | None = None,
prompt_token_ids: list[int] | None = None,
stop_reason: int | str | None = None,
) -> GenerateStreamResponse:
"""Build a GenerateStreamResponse SSE chunk."""
return GenerateTokensStreamResponse(
@@ -204,6 +205,7 @@ def _make_stream_chunk(
index=index,
token_ids=token_ids,
finish_reason=finish_reason,
stop_reason=stop_reason,
logprobs=logprobs,
)
],
@@ -886,6 +888,19 @@ class TestDerenderCompletionStream:
)
assert chunk.choices[0].finish_reason == "length"
@pytest.mark.asyncio
async def test_stop_reason_forwarded(self, derenderer, tokenizer):
"""stop_reason from the final generate chunk reaches the derendered
choice, matching what the coupled server reports."""
token_ids = tokenizer.encode("done")[:2]
chunk, _ = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
token_ids, finish_reason="stop", stop_reason="\n\n"
),
)
assert chunk.choices[0].stop_reason == "\n\n"
class TestStreamLogprobs:
"""Streaming derender must carry per-chunk logprobs with token ids
@@ -139,6 +139,7 @@ def _make_request_output(
request_id: str,
token_ids: list[int],
finish_reason: str | None = None,
stop_reason: int | str | None = None,
finished: bool = False,
prompt_token_ids: list[int] | None = None,
logprobs: list[dict[int, Any] | None] | None = None,
@@ -160,6 +161,7 @@ def _make_request_output(
cumulative_logprob=None,
logprobs=logprobs,
finish_reason=finish_reason,
stop_reason=stop_reason,
spec_decode_metrics=spec_decode_metrics,
)
],
@@ -378,6 +380,71 @@ async def test_stream_basic():
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
async def test_non_stream_reports_engine_stop_reason():
"""The matched stop token id from the engine lands on the choice."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10, 42],
finish_reason="stop",
stop_reason=42,
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=False,
)
response = await serving.serve_tokens(request)
choice = response.model_dump()["choices"][0]
assert choice["finish_reason"] == "stop"
assert choice["stop_reason"] == 42
@pytest.mark.asyncio
async def test_stream_reports_engine_stop_reason_on_final_chunk():
"""A matched stop string is reported with finish_reason and null before."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1",
token_ids=[20],
finish_reason="stop",
stop_reason="done",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if c != "[DONE]"]
assert len(data_chunks) == 2
assert data_chunks[0]["choices"][0]["stop_reason"] is None
assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
assert data_chunks[1]["choices"][0]["stop_reason"] == "done"
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):