[Frontend] Omit absent fields from /inference/v1/generate stream chunks (#57264)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-17 05:58:56 +00:00
committed by GitHub
co-authored by Claude Fable 5.1
parent e6b1a5e5e3
commit 8538017f4b
3 changed files with 67 additions and 7 deletions
@@ -569,6 +569,40 @@ async def test_stream_omits_prompt_metadata_by_default():
assert "mm_placeholders" not in chunk
@pytest.mark.asyncio
async def test_stream_chunks_omit_absent_fields():
"""Absent optional fields are omitted from stream chunks, like Rust."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1", token_ids=[20], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=2),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert len(data_chunks) == 2
for chunk in data_chunks:
assert "usage" not in chunk
assert "prompt_token_ids" not in chunk
assert "mm_placeholders" not in chunk
assert "logprobs" not in chunk["choices"][0]
assert "finish_reason" not in data_chunks[0]["choices"][0]
assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
async def test_stream_text_only_prompt_omits_mm_placeholders_key():
engine = _mock_engine()