[Bugfix][Frontend] Preserve Cohere and Anthropic streaming errors (#61947)

Signed-off-by: Kevin <jia__kuan@163.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jovan
2026-10-17 06:10:16 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 90d6d871d6
commit 5be4266f5c
6 changed files with 531 additions and 375 deletions
@@ -49,6 +49,9 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatMessage,
)
from vllm.entrypoints.serve.engine.protocol import PromptTokenUsageInfo, UsageInfo
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
validation_exception_handler,
)
@@ -1034,7 +1037,7 @@ def _make_stream_chunk(
stop_reason: int | str | None = None,
choices: list[ChatCompletionResponseStreamChoice] | None = None,
usage: UsageInfo | None = None,
) -> str:
) -> ChatCompletionStreamResponse:
if choices is None:
choices = [
ChatCompletionResponseStreamChoice(
@@ -1051,7 +1054,7 @@ def _make_stream_chunk(
choices=choices,
usage=usage,
)
return f"data: {chunk.model_dump_json()}"
return chunk
def _tc(*, args, id=None, name=None):
@@ -1062,6 +1065,39 @@ def _tc(*, args, id=None, name=None):
)
@pytest.mark.asyncio
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_use"])
async def test_stream_error_preserves_message_and_closes_block(block_type):
async def source():
if block_type is not None:
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
if block_type == "text":
delta = DeltaMessage(content="Hello")
elif block_type == "thinking":
delta = DeltaMessage(reasoning="Thinking")
else:
delta = DeltaMessage(
tool_calls=[_tc(id="call_1", name="search", args='{"q":')]
)
yield _make_stream_chunk(delta=delta)
yield create_error_response(ValueError("backend failed"))
converter = _make_stream_converter()
events = _parse_sse_events(
[event async for event in converter.message_stream_converter(source())]
)
assert events[-1] == (
"error",
{
"type": "error",
"error": {"type": "internal_error", "message": "backend failed"},
},
)
assert not any(event_type == "message_stop" for event_type, _ in events)
if block_type is not None:
assert events[-2][0] == "content_block_stop"
class TestMessageStreamConverterToolUseContentBuffering:
"""Regression test for tool_use arguments being silently dropped.
@@ -1074,7 +1110,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
async def test_tool_use_args_not_dropped_when_content_in_same_chunk(
self,
):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=10, total_tokens=10),
@@ -1111,11 +1147,10 @@ class TestMessageStreamConverterToolUseContentBuffering:
completion_tokens=20,
),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1156,7 +1191,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
stream jumps straight from finish_reason to [DONE], skipping the
empty-choices usage chunk."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=10, total_tokens=10),
@@ -1178,11 +1213,10 @@ class TestMessageStreamConverterToolUseContentBuffering:
finish_reason="tool_calls",
)
# No empty-choices usage chunk — go straight to [DONE].
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1216,7 +1250,7 @@ class TestMessageStartIncludesTypeAndRole:
@pytest.mark.asyncio
async def test_message_start_contains_message_type_and_role(self):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(content="Hello"),
usage=UsageInfo(
@@ -1226,11 +1260,10 @@ class TestMessageStartIncludesTypeAndRole:
),
)
yield _make_stream_chunk(finish_reason="stop")
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1260,7 +1293,7 @@ class TestStreamingCacheUsageSemantics:
message_start omits cache fields. The final chunk carries
prompt_tokens_details, so message_delta carries resolved values."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant", content="hi"),
usage=UsageInfo(prompt_tokens=100, total_tokens=100),
@@ -1277,11 +1310,10 @@ class TestStreamingCacheUsageSemantics:
),
),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1305,7 +1337,7 @@ class TestStreamingCacheUsageSemantics:
"""When the final chunk reports cached_tokens=0, message_delta carries
cache fields = 0 (cache miss); message_start still omits them."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=50, total_tokens=50),
@@ -1322,11 +1354,10 @@ class TestStreamingCacheUsageSemantics:
),
),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1346,7 +1377,7 @@ class TestStreamingCacheUsageSemantics:
"""If --enable-prompt-tokens-details is off, no chunk carries cache
info; both message_start and message_delta omit cache fields."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=30, total_tokens=30),
@@ -1356,11 +1387,10 @@ class TestStreamingCacheUsageSemantics:
choices=[],
usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1628,7 +1658,7 @@ class TestStopSequenceReason:
@pytest.mark.asyncio
async def test_streaming_stop_string_maps_to_stop_sequence(self):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(content="hi"),
usage=UsageInfo(prompt_tokens=5, total_tokens=5, completion_tokens=0),
@@ -1638,11 +1668,10 @@ class TestStopSequenceReason:
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1654,7 +1683,7 @@ class TestStopSequenceReason:
async def test_streaming_no_stop_string_emits_explicit_null_stop_sequence(self):
"""exclude_unset=True drops stop_sequence unless it is set explicitly."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
yield _make_stream_chunk(delta=DeltaMessage(content="hi"))
yield _make_stream_chunk(finish_reason="stop")
@@ -1662,11 +1691,10 @@ class TestStopSequenceReason:
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1724,7 +1752,7 @@ class TestToolUseStopReason:
],
)
async def test_streaming(self, args, finish_reason, stop_reason, expected):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
yield _make_stream_chunk(
delta=DeltaMessage(
@@ -1738,11 +1766,10 @@ class TestToolUseStopReason:
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
events = _parse_sse_events(
[e async for e in converter.message_stream_converter(sse_input())]
[e async for e in converter.message_stream_converter(chunk_input())]
)
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
@@ -5,7 +5,7 @@
The stream-translation entry point is
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
async iterable of OpenAI SSE chunks into Cohere's
async iterable of OpenAI chunk objects into Cohere's
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
event stream.
@@ -15,7 +15,7 @@ state machine.
"""
import json
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Sequence
from typing import Any
import pytest
@@ -26,10 +26,7 @@ from vllm.entrypoints.cohere.cohere_chat_message import (
from vllm.entrypoints.cohere.cohere_chat_message import (
CitationSource,
)
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
MessageStartEvent,
)
from vllm.entrypoints.cohere.protocol import MessageStartEvent
from vllm.entrypoints.cohere.serving import (
_DONE_FRAME,
CohereServingChatV2,
@@ -41,6 +38,7 @@ from vllm.entrypoints.cohere.serving import (
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
# ----------------------------------------------------------------------
# Helpers
@@ -85,9 +83,8 @@ def _make_chunk(
usage: dict[str, Any] | None = None,
omit_choices: bool = False,
citations: list[Any] | None = None,
) -> str:
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
consumes."""
) -> ChatCompletionStreamResponse:
"""Build the raw chat chunk the production converter consumes."""
delta: dict[str, Any] = {}
if role is not None:
delta["role"] = role
@@ -119,27 +116,23 @@ def _make_chunk(
payload["choices"] = []
if usage is not None:
payload["usage"] = usage
# Use ChatCompletionStreamResponse to normalize the payload shape.
chunk = ChatCompletionStreamResponse.model_validate(payload)
return f"data: {chunk.model_dump_json(exclude_none=False)}\n\n"
return ChatCompletionStreamResponse.model_validate(payload)
def _make_done() -> str:
return "data: [DONE]\n\n"
async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
async def _stream_from(
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
for item in items:
yield item
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
async def _drain(
serving: CohereServingChatV2,
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
) -> list[str]:
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
the emitted SSE frames."""
request = CohereChatV2Request(
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
)
gen = serving._chat_completion_stream_to_v2(_stream_from(items), request)
gen = serving._chat_completion_stream_to_v2(_stream_from(items))
return [frame async for frame in gen]
@@ -793,15 +786,14 @@ class TestChatCompletionStreamToV2:
assert "content-end" not in types
@pytest.mark.asyncio
async def test_done_marker_in_middle_closes_open_block(self):
# Some upstreams send [DONE] without a trailing usage-only chunk.
async def test_stream_exhaustion_closes_open_block_without_usage(self):
# The raw stream may finish without a trailing usage-only chunk.
# The translator must still emit message-end before [DONE] so
# Cohere clients don't hang.
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="Hi"),
_make_done(),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
@@ -815,72 +807,65 @@ class TestChatCompletionStreamToV2:
assert frames[-1] == _DONE_FRAME
@pytest.mark.asyncio
async def test_skips_empty_and_non_data_lines(self):
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_call"])
async def test_upstream_error_preserves_message(self, block_type):
serving = _serving()
items = [
"\n",
"event: ping\n\n",
_make_chunk(role="assistant"),
"data: \n\n", # empty data
_make_chunk(content="Hi"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
items: list[ChatCompletionStreamResponse | ErrorResponse] = []
if block_type is not None:
items.append(_make_chunk(role="assistant"))
if block_type == "text":
items.append(_make_chunk(content="Hi"))
elif block_type == "thinking":
items.append(_make_chunk(reasoning="Thinking"))
else:
items.append(
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "call_1",
"function": {"name": "search", "arguments": '{"q":'},
}
]
)
)
items.append(serving.create_error_response(ValueError("backend failed")))
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
# Should still produce a complete lifecycle.
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types[0] == "message-start"
assert types[-1] == "message-end"
events = [_parse_event(frame) for frame in frames[:-1]]
error_ends = [event for event in events if event["type"] == "message-end"]
assert len(error_ends) == 1
assert error_ends[0]["delta"] == {
"finish_reason": "ERROR",
"error": "backend failed",
}
assert events[-1] == error_ends[0]
if block_type is not None:
assert events[-2]["type"] == (
"tool-call-end" if block_type == "tool_call" else "content-end"
)
@pytest.mark.asyncio
async def test_exception_in_chunk_parsing_emits_error_message_end(self):
# An invalid JSON payload after the first chunk triggers the
# error path: a synthetic message-end with finish_reason=ERROR
# followed by [DONE].
serving = _serving()
items = [
_make_chunk(role="assistant"),
"data: {not valid json}\n\n",
]
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
# Find the error-shaped message-end.
error_end = _parse_event(frames[-2])
assert error_end["type"] == "message-end"
assert error_end["delta"]["finish_reason"] == "ERROR"
assert "error" in error_end["delta"]
@pytest.mark.asyncio
async def test_error_message_end_is_sanitized(self):
@pytest.mark.parametrize("started", [False, True])
async def test_error_message_end_is_sanitized(self, started):
# The client-visible ``error`` field on the terminal message-end
# must be routed through ``sanitize_message`` so tracebacks,
# host filesystem paths, and Python object memory addresses
# can't leak from a mid-stream exception into the SSE payload.
serving = _serving()
async def _raises_mid_stream() -> AsyncGenerator[str, None]:
yield _make_chunk(role="assistant")
async def _raises_mid_stream() -> AsyncGenerator[
ChatCompletionStreamResponse | ErrorResponse, None
]:
if started:
yield _make_chunk(role="assistant")
raise RuntimeError(
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
)
request = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
stream=True,
)
frames = [
f
async for f in serving._chat_completion_stream_to_v2(
_raises_mid_stream(), request
)
f async for f in serving._chat_completion_stream_to_v2(_raises_mid_stream())
]
assert frames[-1] == _DONE_FRAME
error_end = _parse_event(frames[-2])
@@ -28,6 +28,7 @@ from vllm.entrypoints.generate.base.serving import build_per_request_timing_metr
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.chat_completion.serving import (
OpenAIServingChat,
@@ -43,7 +44,7 @@ from vllm.entrypoints.openai.models.serving import (
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason
from vllm.exceptions import QueueOverflowError, VLLMValidationError
from vllm.exceptions import GenerationError, QueueOverflowError, VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import PlaceholderRange
@@ -716,6 +717,149 @@ async def _collect_metrics_stream_chunks(
return chunks
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", [False, True])
async def test_streaming_chunks_keep_usage_and_metadata(raw):
serving = _build_minimal_metrics_serving_chat(False)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test"}],
stream=True,
stream_options={"include_usage": True},
)
metadata = RequestResponseMetadata(request_id="chatcmpl-test")
method = (
serving.chat_completion_stream_generator_raw
if raw
else serving.chat_completion_stream_generator
)
chunks = [
chunk
async for chunk in method(
request,
_single_request_output(_make_metrics_request_output()),
"chatcmpl-test",
"test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=metadata,
)
]
if raw:
assert all(isinstance(chunk, ChatCompletionStreamResponse) for chunk in chunks)
else:
assert chunks[-1] == "data: [DONE]\n\n"
first = json.loads(chunks[0].removeprefix("data: "))
assert first["choices"][0]["logprobs"] is None
usage = json.loads(chunks[-2].removeprefix("data: "))
assert "system_fingerprint" not in usage
assert "prompt_tokens_details" not in usage["usage"]
chunks = [
ChatCompletionStreamResponse.model_validate_json(
chunk.removeprefix("data: ")
)
for chunk in chunks[:-1]
]
assert len(chunks) == 3
assert chunks[0].choices[0].delta.role == "assistant"
assert chunks[1].choices[0].delta.content == "Hello"
assert chunks[1].choices[0].finish_reason == "stop"
assert chunks[2].choices == []
assert chunks[2].usage.prompt_tokens == 3
assert chunks[2].usage.completion_tokens == 2
assert metadata.final_usage_info.total_tokens == 5
@pytest.mark.asyncio
async def test_streaming_serialization_failure_emits_error_and_done():
serving = _build_minimal_metrics_serving_chat(False)
serving.parser_cls = HarmonyParser
parser = MagicMock()
parser.parse_delta.return_value = DeltaMessage(
content="Hello", extra_object=object()
)
serving._make_parser = MagicMock(return_value=parser)
request = ChatCompletionRequest(
model="test-model", messages=[{"role": "user", "content": "Test"}], stream=True
)
frames = [
frame
async for frame in serving.chat_completion_stream_generator(
request,
_single_request_output(_make_metrics_request_output()),
"chatcmpl-test",
"test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test"),
)
]
assert frames[-1] == "data: [DONE]\n\n"
error = ErrorResponse.model_validate_json(frames[-2].removeprefix("data: "))
assert "serialize" in error.error.message
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", [False, True])
@pytest.mark.parametrize(
"failure", ["parser", "before_first", "after_content", "generation"]
)
async def test_streaming_errors_preserve_envelope(raw, failure):
serving = _build_minimal_metrics_serving_chat(False)
error = (
GenerationError("backend failed")
if failure == "generation"
else ValueError("backend failed")
)
if failure == "parser":
serving.parser_cls = HarmonyParser
serving._make_parser = MagicMock(side_effect=error)
async def results():
if failure == "after_content":
output = _make_metrics_request_output()
output.outputs[0].finish_reason = None
output.finished = False
yield output
raise error
request = ChatCompletionRequest(
model="test-model", messages=[{"role": "user", "content": "Test"}], stream=True
)
method = (
serving.chat_completion_stream_generator_raw
if raw
else serving.chat_completion_stream_generator
)
chunks = [
chunk
async for chunk in method(
request,
results(),
"chatcmpl-test",
"test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test"),
)
]
if raw:
assert all(
isinstance(chunk, (ChatCompletionStreamResponse, ErrorResponse))
for chunk in chunks
)
assert isinstance(chunks[-1], ErrorResponse)
error_response = chunks[-1]
else:
assert chunks[-1] == "data: [DONE]\n\n"
assert chunks.count("data: [DONE]\n\n") == 1
error_response = ErrorResponse.model_validate_json(
chunks[-2].removeprefix("data: ")
)
assert error_response.error.message == "backend failed"
assert error_response.error.code == (500 if failure == "generation" else 400)
def test_build_per_request_timing_metrics_valid_timestamps():
metrics = build_per_request_timing_metrics(
_PER_REQUEST_STATS, num_generation_tokens=10