[Bugfix][Responses API] Reuse streamed item ids in final harmony response (#59859)

Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Yufeng He
2026-10-04 03:01:04 +00:00
committed by GitHub
co-authored by Flora Feng
parent 50b404e71e
commit 6f75cc7d5d
5 changed files with 134 additions and 12 deletions
@@ -67,7 +67,7 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.inputs import tokens_input
from vllm.logprobs import Logprob as SampleLogprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser.harmony import Segment
from vllm.parser.harmony import ChunkResult, HarmonyParser, Segment
from vllm.renderers import TokenizeParams
from vllm.renderers.online_renderer import (
OnlineRenderer,
@@ -2129,6 +2129,87 @@ async def test_stream_completed_response_reuses_streamed_items(monkeypatch):
assert function_call.call_id == "chatcmpl-tool-parser-id"
def _harmony_msg(channel: str, text: str, recipient: str | None = None):
msg = OpenAIHarmonyMessage.from_role_and_content(Role.ASSISTANT, text)
msg = msg.with_channel(channel)
return msg.with_recipient(recipient) if recipient else msg
def _harmony_segments(msg, *, streamed: bool = True) -> list[Segment]:
"""Segments the parser yields for one message: a content delta (absent
for a zero-delta message), then the completed message."""
delta = [Segment(msg.channel, msg.recipient, msg.content[0].text)]
completed = Segment(msg.channel, msg.recipient, "", completed_message=msg)
return (delta if streamed else []) + [completed]
async def _harmony_stream_events(segment_chunks: list[list[Segment]]):
serving = _make_serving_instance()
serving.use_harmony = True
parser = MagicMock(spec=HarmonyParser)
parser.process_chunk.side_effect = [
ChunkResult(segments=segments, reasoning_token_count=0)
for segments in segment_chunks
]
context = HarmonyContext([], [], frozenset({"get_weather"}), parser)
async def result_generator():
for token_id in range(len(segment_chunks)):
context.append_output(_make_request_output("", [token_id]))
yield context
return [
event
async for event in serving.responses_stream_generator(
request=ResponsesRequest(input="hi", stream=True, store=False),
sampling_params=SamplingParams(max_tokens=16),
result_generator=result_generator(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
@pytest.mark.asyncio
async def test_harmony_stream_completed_response_reuses_streamed_ids():
"""response.completed must carry the id and call_id each item was streamed
with, not ones minted when the harmony messages are converted again."""
messages = [
_harmony_msg("analysis", "think"),
_harmony_msg("commentary", '{"city": "Paris"}', "functions.get_weather"),
_harmony_msg("final", "Sunny"),
]
events = await _harmony_stream_events([_harmony_segments(msg) for msg in messages])
streamed = [e.item for e in events if e.type == "response.output_item.done"]
output = events[-1].response.output
assert [item.type for item in output] == ["reasoning", "function_call", "message"]
assert [item.id for item in output] == [item.id for item in streamed]
assert output[1].call_id == streamed[1].call_id
assert output[2].content[0].text == "Sunny"
@pytest.mark.asyncio
async def test_harmony_stream_unstreamed_item_keeps_own_id():
"""An item with no done event (zero-delta) must not take the streamed id
of a later item of the same type."""
unstreamed = _harmony_msg("analysis", "skipped")
reasoning = _harmony_msg("analysis", "think")
events = await _harmony_stream_events(
[_harmony_segments(unstreamed, streamed=False), _harmony_segments(reasoning)]
)
(streamed,) = [e.item for e in events if e.type == "response.output_item.done"]
first, second = events[-1].response.output
assert first.content[0].text == "skipped"
assert first.id != streamed.id
assert second.id == streamed.id
@pytest.mark.asyncio
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
async def test_streaming_parallel_tool_calls(