[Frontend] Report stop_reason on /inference/v1/generate and pass it through derender (#60251)
Signed-off-by: Hyeonki Hong <hhk7734@gmail.com> Signed-off-by: Shimi Bandiel <shimib@google.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Hyeonki Hong <hhk7734@gmail.com> Co-authored-by: Mimir <mimir@nousresearch.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Hyeonki Hong
Mimir
Claude
Bugen Zhao
parent
32b05addc1
commit
8d7b93fec2
@@ -112,6 +112,38 @@ async def test_derender_chat_roundtrip(client):
|
||||
assert data["choices"][0]["message"]["role"] == "assistant"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_passes_stop_reason_through(client):
|
||||
"""The coupled server reports the stop string or token id that ended
|
||||
generation; derender must carry the generate value through unchanged."""
|
||||
gen_req = await _render_chat(client)
|
||||
generate_response = _make_generate_response(gen_req["token_ids"][:3])
|
||||
generate_response["choices"][0]["stop_reason"] = "\n\n"
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={"model": MODEL_NAME, "generate_response": generate_response},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["choices"][0]["stop_reason"] == "\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_passes_stop_reason_through(client):
|
||||
gen_req = await _render_completion(client, "Hello world")
|
||||
generate_response = _make_completion_generate_response(
|
||||
gen_req["token_ids"][:3], "gen-stop"
|
||||
)
|
||||
generate_response["choices"][0]["stop_reason"] = 151645
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={"model": MODEL_NAME, "generate_responses": [generate_response]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["choices"][0]["stop_reason"] == 151645
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_usage(client):
|
||||
"""Supplied prompt_tokens flows through into usage correctly."""
|
||||
|
||||
@@ -195,6 +195,7 @@ def _make_stream_chunk(
|
||||
metrics: PerRequestMetrics | None = None,
|
||||
logprobs: dict | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
stop_reason: int | str | None = None,
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a GenerateStreamResponse SSE chunk."""
|
||||
return GenerateTokensStreamResponse(
|
||||
@@ -204,6 +205,7 @@ def _make_stream_chunk(
|
||||
index=index,
|
||||
token_ids=token_ids,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason,
|
||||
logprobs=logprobs,
|
||||
)
|
||||
],
|
||||
@@ -886,6 +888,19 @@ class TestDerenderCompletionStream:
|
||||
)
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_reason_forwarded(self, derenderer, tokenizer):
|
||||
"""stop_reason from the final generate chunk reaches the derendered
|
||||
choice, matching what the coupled server reports."""
|
||||
token_ids = tokenizer.encode("done")[:2]
|
||||
chunk, _ = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
token_ids, finish_reason="stop", stop_reason="\n\n"
|
||||
),
|
||||
)
|
||||
assert chunk.choices[0].stop_reason == "\n\n"
|
||||
|
||||
|
||||
class TestStreamLogprobs:
|
||||
"""Streaming derender must carry per-chunk logprobs with token ids
|
||||
|
||||
@@ -139,6 +139,7 @@ def _make_request_output(
|
||||
request_id: str,
|
||||
token_ids: list[int],
|
||||
finish_reason: str | None = None,
|
||||
stop_reason: int | str | None = None,
|
||||
finished: bool = False,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
logprobs: list[dict[int, Any] | None] | None = None,
|
||||
@@ -160,6 +161,7 @@ def _make_request_output(
|
||||
cumulative_logprob=None,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason,
|
||||
spec_decode_metrics=spec_decode_metrics,
|
||||
)
|
||||
],
|
||||
@@ -378,6 +380,71 @@ async def test_stream_basic():
|
||||
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_reports_engine_stop_reason():
|
||||
"""The matched stop token id from the engine lands on the choice."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10, 42],
|
||||
finish_reason="stop",
|
||||
stop_reason=42,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
choice = response.model_dump()["choices"][0]
|
||||
assert choice["finish_reason"] == "stop"
|
||||
assert choice["stop_reason"] == 42
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_reports_engine_stop_reason_on_final_chunk():
|
||||
"""A matched stop string is reported with finish_reason and null before."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[20],
|
||||
finish_reason="stop",
|
||||
stop_reason="done",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
parsed = _parse_sse_chunks([chunk async for chunk in response])
|
||||
data_chunks = [c for c in parsed if c != "[DONE]"]
|
||||
assert len(data_chunks) == 2
|
||||
|
||||
assert data_chunks[0]["choices"][0]["stop_reason"] is None
|
||||
assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
|
||||
assert data_chunks[1]["choices"][0]["stop_reason"] == "done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
|
||||
|
||||
Reference in New Issue
Block a user