[Frontend] Add streaming parity tests and docs for derender (#57922)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-21 14:13:52 +00:00
committed by GitHub
parent 606d124b37
commit 8b98b7d0b4
5 changed files with 410 additions and 101 deletions
@@ -3,20 +3,25 @@
"""Round trip parity CI: render -> generate -> derender -> render == render.
Pins the coupled parsing path (``/v1/chat/completions`` on a normal GPU
Pins the coupled parsing path (`/v1/chat/completions` on a normal GPU
server which parses generated tokens incrementally as they stream out)
against the disaggregated path (``OnlineDerenderer.derender_chat`` via
``/v1/chat/completions/derender`` which parses the same tokens all at once,
out of process). A standard ``vllm serve`` GPU server mounts both, so one
against the disaggregated path (`OnlineDerenderer.derender_chat` via
`/v1/chat/completions/derender` which parses the same tokens all at once,
out of process). A standard `vllm serve` GPU server mounts both, so one
real generation lets both parsers run on identical input.
Both paths consume the same generated token IDs (extracted from the coupled
response's ``token_ids`` via ``return_token_ids=True``), so generation
response's `token_ids` via `return_token_ids=True`), so generation
nondeterminism is irrelevant. The only variable under test is whether the
two parsing code paths agree. Parity is asserted unconditionally as only the
stronger per case assertions (e.g. "a tool call was produced") are gated
behind the marker actually having been emitted since a 1.5B model is not
guaranteed to emit ``<think>`` / ``<tool_call>``.
guaranteed to emit `<think>` / `<tool_call>`.
The streaming cases do the same for `stream=true`. Each coupled chunk's
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
through the chunked derender path with `stream_state` threaded across calls
the way a client would, so both parsers see the same chunk boundaries.
"""
import json
@@ -24,7 +29,12 @@ import json
import httpx
import pytest
import pytest_asyncio
import regex as re
from tests.entrypoints.scale_out.derender.utils import (
assemble_stream,
stream_chat_derender,
)
from tests.utils import RemoteOpenAIServer
MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
@@ -228,8 +238,8 @@ async def test_parity_logprobs(client):
"""token_id:N resolution parity vs. the coupled server's real strings.
A real disaggregated worker only has token IDs so it emits logprobs
with ``token_id:N`` placeholders (``return_tokens_as_token_ids=True``
reproduces that shape here). ``/derender`` must resolve those
with `token_id:N` placeholders (`return_tokens_as_token_ids=True`
reproduces that shape here). `/derender` must resolve those
placeholders to the same token strings/bytes the coupled server
resolves them to directly.
"""
@@ -267,3 +277,166 @@ async def test_parity_logprobs(client):
for d_entry, r_entry in zip(d_content, r_content):
assert d_entry["token"] == r_entry["token"]
assert d_entry["bytes"] == r_entry["bytes"]
# ---------------------------------------------------------------------------
# Streaming helpers
# ---------------------------------------------------------------------------
async def _coupled_stream(
client: httpx.AsyncClient, messages: list[dict], **extra
) -> tuple[list[dict], list[int]]:
"""Stream `/v1/chat/completions` and return the choices that carry
`token_ids` along with the prompt token IDs.
The leading role only chunk has no `token_ids` and no counterpart on
`/inference/v1/generate`, so it is dropped.
"""
choices: list[dict] = []
prompt_token_ids: list[int] | None = None
async with client.stream(
"POST",
"/v1/chat/completions",
json={
"model": MODEL,
"messages": messages,
"temperature": 0,
"max_tokens": 128,
"return_token_ids": True,
"stream": True,
**extra,
},
) as resp:
assert resp.status_code == 200, await resp.aread()
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
data = line[len("data: ") :]
if data == "[DONE]":
break
chunk = json.loads(data)
assert "error" not in chunk, chunk
if chunk.get("prompt_token_ids") is not None:
prompt_token_ids = chunk["prompt_token_ids"]
choices.extend(
ch for ch in chunk["choices"] if ch.get("token_ids") is not None
)
assert prompt_token_ids is not None
return choices, prompt_token_ids
def _tool_call_id_shape(tool_call_id: str | None) -> str | None:
"""Drop the random suffix so IDs compare by format, e.g. `chatcmpl-tool-`.
Deterministic IDs such as kimi_k2's `functions.<name>:<idx>` are kept
whole."""
if tool_call_id is None:
return None
return re.sub(r"[0-9a-f]{16}$", "", tool_call_id)
async def _run_stream_parity_case(
client: httpx.AsyncClient,
messages: list[dict],
finish_only_tail: bool = False,
**extra,
) -> tuple[dict, list[dict]]:
"""Stream the coupled request, replay its chunks through the streaming
derender endpoint and assert the assembled messages agree. Returns the
assembled coupled message and the coupled choices.
Derender gets the engine's finish reason, since `"tool_calls"` is a chat
level rewrite of `"stop"` that derender has to reapply itself. With
`finish_only_tail` it goes on a trailing chunk with no tokens.
"""
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
finish_reason = coupled_choices[-1]["finish_reason"]
disagg = await stream_chat_derender(
client,
[tid for ch in coupled_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in coupled_choices]
+ ([0] if finish_only_tail else []),
{"model": MODEL, "messages": messages, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
)
coupled = assemble_stream(coupled_choices)
for message in (coupled, disagg):
for tc in message["tool_calls"]:
tc["id"] = _tool_call_id_shape(tc["id"])
assert disagg == coupled
return coupled, coupled_choices
# ---------------------------------------------------------------------------
# Streaming parity cases
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_stream_parity_plain(client):
"""Plain streamed detokenization parity."""
messages = [
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
]
await _run_stream_parity_case(client, messages)
@pytest.mark.asyncio
async def test_stream_parity_reasoning(client):
"""Streamed reasoning/content split parity."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
coupled, _ = await _run_stream_parity_case(
client, messages, include_reasoning=True, max_tokens=256
)
if not coupled["reasoning"]:
pytest.skip("Model did not emit a <think> block")
@pytest.mark.asyncio
async def test_stream_parity_tool_call(client):
"""Streamed tool call parity under `tool_choice="auto"`, where a tool
call also rewrites `finish_reason` to `"tool_calls"`."""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
coupled, _ = await _run_stream_parity_case(
client, messages, tools=TOOLS, tool_choice="auto", max_tokens=512
)
if not coupled["tool_calls"]:
pytest.skip("Model did not emit a tool call")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"chunking", ["engine_steps", "multi_token", "finish_only_tail"]
)
async def test_stream_parity_reasoning_and_tool_call(client, chunking):
"""Streamed reasoning + forced tool call parity across chunk shapes.
`multi_token` batches tokens per chunk with `stream_interval` the way
speculative decoding does, so replay has to reuse the original chunk
boundaries. `finish_only_tail` sends the finish reason on a chunk with no
tokens, which must still flush buffered tool call arguments.
"""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
extra: dict = {
"tools": TOOLS,
"tool_choice": FORCE_WEATHER_TOOL,
"include_reasoning": True,
"max_tokens": 512,
}
if chunking == "multi_token":
extra["stream_interval"] = 4
coupled, coupled_choices = await _run_stream_parity_case(
client,
messages,
finish_only_tail=chunking == "finish_only_tail",
**extra,
)
if chunking == "multi_token":
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
if not (coupled["reasoning"] and coupled["tool_calls"]):
pytest.skip("Model did not emit both a <think> block and a tool call")
@@ -5,24 +5,24 @@
Tests are split into two layers:
1. Unit tests (no server): covers ``_detokenize_delta`` correctness
(chunked == one-shot) and ``derender_completion_stream`` /
``derender_chat_stream`` logic via a real tokenizer on a tiny model.
1. Unit tests (no server): covers `_detokenize_delta` correctness
(chunked == one-shot) and `derender_completion_stream` /
`derender_chat_stream` logic via a real tokenizer on a tiny model.
The parser path is covered both with a deterministic stub parser and
with the real ``HarmonyParser`` (skipped without ``openai_harmony``).
with the real `HarmonyParser` (skipped without `openai_harmony`).
2. Integration tests (require a running render server): covers the full
HTTP round-trip through the streaming endpoint. Marked with
``@pytest.mark.asyncio`` and gated by the ``server`` / ``client``
fixtures from the sibling ``test_derender.py``.
`@pytest.mark.asyncio` and gated by the `server` / `client`
fixtures from the sibling `test_derender.py`.
"""
import json
from collections.abc import Callable
import pytest
import pytest_asyncio
from tests.entrypoints.scale_out.derender.utils import stream_chat_derender
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
@@ -356,7 +356,7 @@ class TestDetokenizeDelta:
"""prev_tokens must not grow with the number of chunks (bounded transport).
Guards that the carried decode window is a small constant
tail, so cumulative ``stream_state`` transport is O(n) and not O(n^2).
tail, so cumulative `stream_state` transport is O(n) and not O(n^2).
"""
token_ids = tokenizer.encode(
"a reasonably long ascii stream of tokens used to exercise the "
@@ -1616,86 +1616,6 @@ async def _render_parser_chat(client, messages: list[dict]) -> dict:
return resp.json()
async def _stream_chat_derender(
client,
output_ids: list[int],
chunk_sizes: list[int],
chat_request: dict,
prompt_tokens: int,
prompt_token_ids: list[int],
on_chunk: Callable[[list[dict]], None] | None = None,
) -> dict:
"""Feed `output_ids` through the streaming chat derender endpoint in
the given `chunk_sizes`, threading `stream_state` across calls and
return the assembled message.
If `on_chunk` is given, it is called after every chunk with a
snapshot (deep copy) of the `tool_calls` accumulator so far, letting
callers assert properties of the intermediate deltas (e.g. monotonic
argument growth) rather than only the final assembled result.
"""
state = None
content = ""
reasoning = ""
tool_calls: list[dict] = []
finish_reason = None
pos = 0
for i, size in enumerate(chunk_sizes):
tids = output_ids[pos : pos + size]
pos += size
is_last = i == len(chunk_sizes) - 1
resp = await client.post(
"/v1/chat/completions/derender",
json={
"stream": True,
"model": PARSER_MODEL,
"generate_chunk": {
"request_id": "stream-test",
"choices": [
{
"index": 0,
"token_ids": tids,
"finish_reason": "stop" if is_last else None,
}
],
},
"stream_state": state,
"prompt_tokens": prompt_tokens,
"prompt_token_ids": prompt_token_ids,
"chat_request": chat_request,
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
state = data["stream_state"]
delta = data["chunk"]["choices"][0]["delta"]
content += delta.get("content") or ""
reasoning += delta.get("reasoning") or ""
for tc in delta.get("tool_calls") or []:
idx = tc["index"]
while len(tool_calls) <= idx:
tool_calls.append({"id": None, "name": None, "arguments": ""})
if tc.get("id"):
tool_calls[idx]["id"] = tc["id"]
fn = tc.get("function") or {}
if fn.get("name"):
tool_calls[idx]["name"] = fn["name"]
if fn.get("arguments"):
tool_calls[idx]["arguments"] += fn["arguments"]
if is_last:
finish_reason = data["chunk"]["choices"][0]["finish_reason"]
if on_chunk is not None:
on_chunk([dict(tc) for tc in tool_calls])
return {
"content": content or None,
"reasoning": reasoning or None,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"chunking",
@@ -1755,7 +1675,7 @@ async def test_stream_parsed_matches_batch_reasoning(
"triples": [3] * (n // 3) + [n % 3],
}[chunking]
chunk_sizes = [c for c in chunk_sizes if c > 0]
streamed = await _stream_chat_derender(
streamed = await stream_chat_derender(
parser_client,
output_ids,
chunk_sizes,
@@ -1823,7 +1743,7 @@ async def test_stream_parsed_matches_batch_tool_call(parser_client, parser_token
if tool_calls and tool_calls[0]["arguments"]:
_snapshots.append(tool_calls[0]["arguments"])
streamed = await _stream_chat_derender(
streamed = await stream_chat_derender(
parser_client,
output_ids,
chunk_sizes,
@@ -1868,7 +1788,7 @@ async def test_stream_parsed_cjk_across_chunk_boundaries(
"messages": messages,
"include_reasoning": True,
}
streamed = await _stream_chat_derender(
streamed = await stream_chat_derender(
parser_client,
output_ids,
[1] * len(output_ids),
@@ -1896,7 +1816,7 @@ async def test_stream_parsed_include_reasoning_false(parser_client, parser_token
"messages": messages,
"include_reasoning": False,
}
streamed = await _stream_chat_derender(
streamed = await stream_chat_derender(
parser_client,
output_ids,
[1] * len(output_ids),
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers shared by the streaming chat derender tests."""
from collections.abc import Callable
import httpx
def assemble_stream(choices: list[dict]) -> dict:
"""Fold streamed chat choices into one message.
A tool call index must keep the first ID it was given. On the derender
side that is what pinning the IDs in `stream_state` guarantees.
"""
content = ""
reasoning = ""
tool_calls: list[dict] = []
finish_reason = None
for choice in choices:
delta = choice["delta"]
content += delta.get("content") or ""
reasoning += delta.get("reasoning") or ""
for tc in delta.get("tool_calls") or []:
idx = tc["index"]
while len(tool_calls) <= idx:
tool_calls.append({"id": None, "name": None, "arguments": ""})
if tc.get("id"):
assert tool_calls[idx]["id"] in (None, tc["id"]), (
f"tool call {idx} changed ID mid stream"
)
tool_calls[idx]["id"] = tc["id"]
fn = tc.get("function") or {}
if fn.get("name"):
tool_calls[idx]["name"] = fn["name"]
tool_calls[idx]["arguments"] += fn.get("arguments") or ""
finish_reason = choice.get("finish_reason") or finish_reason
return {
"content": content or None,
"reasoning": reasoning or None,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
async def stream_chat_derender(
client: httpx.AsyncClient,
output_ids: list[int],
chunk_sizes: list[int],
chat_request: dict,
prompt_tokens: int,
prompt_token_ids: list[int],
finish_reason: str = "stop",
on_chunk: Callable[[list[dict]], None] | None = None,
) -> dict:
"""Feed `output_ids` through the streaming chat derender endpoint in
the given `chunk_sizes`, threading `stream_state` across calls and
return the assembled message.
`finish_reason` goes on the last chunk, so a trailing size of 0 sends it
on a chunk with no tokens.
If `on_chunk` is given, it is called after every chunk with the
`tool_calls` assembled so far, letting callers assert properties of the
intermediate deltas (e.g. monotonic argument growth) rather than only
the final assembled result.
"""
state = None
choices: list[dict] = []
pos = 0
for i, size in enumerate(chunk_sizes):
tids = output_ids[pos : pos + size]
pos += size
is_last = i == len(chunk_sizes) - 1
resp = await client.post(
"/v1/chat/completions/derender",
json={
"stream": True,
"model": chat_request["model"],
"generate_chunk": {
"request_id": "stream-test",
"choices": [
{
"index": 0,
"token_ids": tids,
"finish_reason": finish_reason if is_last else None,
}
],
},
"stream_state": state,
"prompt_tokens": prompt_tokens,
"prompt_token_ids": prompt_token_ids,
"chat_request": chat_request,
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
state = data["stream_state"]
choices.extend(data["chunk"]["choices"])
if on_chunk is not None:
on_chunk(assemble_stream(choices)["tool_calls"])
return assemble_stream(choices)