[Frontend] Add streaming parity tests and docs for derender (#57922)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -3,20 +3,25 @@
|
||||
|
||||
"""Round trip parity CI: render -> generate -> derender -> render == render.
|
||||
|
||||
Pins the coupled parsing path (``/v1/chat/completions`` on a normal GPU
|
||||
Pins the coupled parsing path (`/v1/chat/completions` on a normal GPU
|
||||
server which parses generated tokens incrementally as they stream out)
|
||||
against the disaggregated path (``OnlineDerenderer.derender_chat`` via
|
||||
``/v1/chat/completions/derender`` which parses the same tokens all at once,
|
||||
out of process). A standard ``vllm serve`` GPU server mounts both, so one
|
||||
against the disaggregated path (`OnlineDerenderer.derender_chat` via
|
||||
`/v1/chat/completions/derender` which parses the same tokens all at once,
|
||||
out of process). A standard `vllm serve` GPU server mounts both, so one
|
||||
real generation lets both parsers run on identical input.
|
||||
|
||||
Both paths consume the same generated token IDs (extracted from the coupled
|
||||
response's ``token_ids`` via ``return_token_ids=True``), so generation
|
||||
response's `token_ids` via `return_token_ids=True`), so generation
|
||||
nondeterminism is irrelevant. The only variable under test is whether the
|
||||
two parsing code paths agree. Parity is asserted unconditionally as only the
|
||||
stronger per case assertions (e.g. "a tool call was produced") are gated
|
||||
behind the marker actually having been emitted since a 1.5B model is not
|
||||
guaranteed to emit ``<think>`` / ``<tool_call>``.
|
||||
guaranteed to emit `<think>` / `<tool_call>`.
|
||||
|
||||
The streaming cases do the same for `stream=true`. Each coupled chunk's
|
||||
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
|
||||
through the chunked derender path with `stream_state` threaded across calls
|
||||
the way a client would, so both parsers see the same chunk boundaries.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -24,7 +29,12 @@ import json
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import regex as re
|
||||
|
||||
from tests.entrypoints.scale_out.derender.utils import (
|
||||
assemble_stream,
|
||||
stream_chat_derender,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
|
||||
@@ -228,8 +238,8 @@ async def test_parity_logprobs(client):
|
||||
"""token_id:N resolution parity vs. the coupled server's real strings.
|
||||
|
||||
A real disaggregated worker only has token IDs so it emits logprobs
|
||||
with ``token_id:N`` placeholders (``return_tokens_as_token_ids=True``
|
||||
reproduces that shape here). ``/derender`` must resolve those
|
||||
with `token_id:N` placeholders (`return_tokens_as_token_ids=True`
|
||||
reproduces that shape here). `/derender` must resolve those
|
||||
placeholders to the same token strings/bytes the coupled server
|
||||
resolves them to directly.
|
||||
"""
|
||||
@@ -267,3 +277,166 @@ async def test_parity_logprobs(client):
|
||||
for d_entry, r_entry in zip(d_content, r_content):
|
||||
assert d_entry["token"] == r_entry["token"]
|
||||
assert d_entry["bytes"] == r_entry["bytes"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _coupled_stream(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[list[dict], list[int]]:
|
||||
"""Stream `/v1/chat/completions` and return the choices that carry
|
||||
`token_ids` along with the prompt token IDs.
|
||||
|
||||
The leading role only chunk has no `token_ids` and no counterpart on
|
||||
`/inference/v1/generate`, so it is dropped.
|
||||
"""
|
||||
choices: list[dict] = []
|
||||
prompt_token_ids: list[int] | None = None
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"messages": messages,
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
"return_token_ids": True,
|
||||
"stream": True,
|
||||
**extra,
|
||||
},
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
data = line[len("data: ") :]
|
||||
if data == "[DONE]":
|
||||
break
|
||||
chunk = json.loads(data)
|
||||
assert "error" not in chunk, chunk
|
||||
if chunk.get("prompt_token_ids") is not None:
|
||||
prompt_token_ids = chunk["prompt_token_ids"]
|
||||
choices.extend(
|
||||
ch for ch in chunk["choices"] if ch.get("token_ids") is not None
|
||||
)
|
||||
assert prompt_token_ids is not None
|
||||
return choices, prompt_token_ids
|
||||
|
||||
|
||||
def _tool_call_id_shape(tool_call_id: str | None) -> str | None:
|
||||
"""Drop the random suffix so IDs compare by format, e.g. `chatcmpl-tool-`.
|
||||
Deterministic IDs such as kimi_k2's `functions.<name>:<idx>` are kept
|
||||
whole."""
|
||||
if tool_call_id is None:
|
||||
return None
|
||||
return re.sub(r"[0-9a-f]{16}$", "", tool_call_id)
|
||||
|
||||
|
||||
async def _run_stream_parity_case(
|
||||
client: httpx.AsyncClient,
|
||||
messages: list[dict],
|
||||
finish_only_tail: bool = False,
|
||||
**extra,
|
||||
) -> tuple[dict, list[dict]]:
|
||||
"""Stream the coupled request, replay its chunks through the streaming
|
||||
derender endpoint and assert the assembled messages agree. Returns the
|
||||
assembled coupled message and the coupled choices.
|
||||
|
||||
Derender gets the engine's finish reason, since `"tool_calls"` is a chat
|
||||
level rewrite of `"stop"` that derender has to reapply itself. With
|
||||
`finish_only_tail` it goes on a trailing chunk with no tokens.
|
||||
"""
|
||||
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
|
||||
finish_reason = coupled_choices[-1]["finish_reason"]
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices]
|
||||
+ ([0] if finish_only_tail else []),
|
||||
{"model": MODEL, "messages": messages, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
for tc in message["tool_calls"]:
|
||||
tc["id"] = _tool_call_id_shape(tc["id"])
|
||||
assert disagg == coupled
|
||||
return coupled, coupled_choices
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming parity cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_plain(client):
|
||||
"""Plain streamed detokenization parity."""
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
|
||||
]
|
||||
await _run_stream_parity_case(client, messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_reasoning(client):
|
||||
"""Streamed reasoning/content split parity."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
coupled, _ = await _run_stream_parity_case(
|
||||
client, messages, include_reasoning=True, max_tokens=256
|
||||
)
|
||||
|
||||
if not coupled["reasoning"]:
|
||||
pytest.skip("Model did not emit a <think> block")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_tool_call(client):
|
||||
"""Streamed tool call parity under `tool_choice="auto"`, where a tool
|
||||
call also rewrites `finish_reason` to `"tool_calls"`."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, _ = await _run_stream_parity_case(
|
||||
client, messages, tools=TOOLS, tool_choice="auto", max_tokens=512
|
||||
)
|
||||
|
||||
if not coupled["tool_calls"]:
|
||||
pytest.skip("Model did not emit a tool call")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"chunking", ["engine_steps", "multi_token", "finish_only_tail"]
|
||||
)
|
||||
async def test_stream_parity_reasoning_and_tool_call(client, chunking):
|
||||
"""Streamed reasoning + forced tool call parity across chunk shapes.
|
||||
|
||||
`multi_token` batches tokens per chunk with `stream_interval` the way
|
||||
speculative decoding does, so replay has to reuse the original chunk
|
||||
boundaries. `finish_only_tail` sends the finish reason on a chunk with no
|
||||
tokens, which must still flush buffered tool call arguments.
|
||||
"""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
extra: dict = {
|
||||
"tools": TOOLS,
|
||||
"tool_choice": FORCE_WEATHER_TOOL,
|
||||
"include_reasoning": True,
|
||||
"max_tokens": 512,
|
||||
}
|
||||
if chunking == "multi_token":
|
||||
extra["stream_interval"] = 4
|
||||
coupled, coupled_choices = await _run_stream_parity_case(
|
||||
client,
|
||||
messages,
|
||||
finish_only_tail=chunking == "finish_only_tail",
|
||||
**extra,
|
||||
)
|
||||
|
||||
if chunking == "multi_token":
|
||||
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
|
||||
if not (coupled["reasoning"] and coupled["tool_calls"]):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
|
||||
@@ -5,24 +5,24 @@
|
||||
|
||||
Tests are split into two layers:
|
||||
|
||||
1. Unit tests (no server): covers ``_detokenize_delta`` correctness
|
||||
(chunked == one-shot) and ``derender_completion_stream`` /
|
||||
``derender_chat_stream`` logic via a real tokenizer on a tiny model.
|
||||
1. Unit tests (no server): covers `_detokenize_delta` correctness
|
||||
(chunked == one-shot) and `derender_completion_stream` /
|
||||
`derender_chat_stream` logic via a real tokenizer on a tiny model.
|
||||
The parser path is covered both with a deterministic stub parser and
|
||||
with the real ``HarmonyParser`` (skipped without ``openai_harmony``).
|
||||
with the real `HarmonyParser` (skipped without `openai_harmony`).
|
||||
|
||||
2. Integration tests (require a running render server): covers the full
|
||||
HTTP round-trip through the streaming endpoint. Marked with
|
||||
``@pytest.mark.asyncio`` and gated by the ``server`` / ``client``
|
||||
fixtures from the sibling ``test_derender.py``.
|
||||
`@pytest.mark.asyncio` and gated by the `server` / `client`
|
||||
fixtures from the sibling `test_derender.py`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.entrypoints.scale_out.derender.utils import stream_chat_derender
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
@@ -356,7 +356,7 @@ class TestDetokenizeDelta:
|
||||
"""prev_tokens must not grow with the number of chunks (bounded transport).
|
||||
|
||||
Guards that the carried decode window is a small constant
|
||||
tail, so cumulative ``stream_state`` transport is O(n) and not O(n^2).
|
||||
tail, so cumulative `stream_state` transport is O(n) and not O(n^2).
|
||||
"""
|
||||
token_ids = tokenizer.encode(
|
||||
"a reasonably long ascii stream of tokens used to exercise the "
|
||||
@@ -1616,86 +1616,6 @@ async def _render_parser_chat(client, messages: list[dict]) -> dict:
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def _stream_chat_derender(
|
||||
client,
|
||||
output_ids: list[int],
|
||||
chunk_sizes: list[int],
|
||||
chat_request: dict,
|
||||
prompt_tokens: int,
|
||||
prompt_token_ids: list[int],
|
||||
on_chunk: Callable[[list[dict]], None] | None = None,
|
||||
) -> dict:
|
||||
"""Feed `output_ids` through the streaming chat derender endpoint in
|
||||
the given `chunk_sizes`, threading `stream_state` across calls and
|
||||
return the assembled message.
|
||||
|
||||
If `on_chunk` is given, it is called after every chunk with a
|
||||
snapshot (deep copy) of the `tool_calls` accumulator so far, letting
|
||||
callers assert properties of the intermediate deltas (e.g. monotonic
|
||||
argument growth) rather than only the final assembled result.
|
||||
"""
|
||||
state = None
|
||||
content = ""
|
||||
reasoning = ""
|
||||
tool_calls: list[dict] = []
|
||||
finish_reason = None
|
||||
|
||||
pos = 0
|
||||
for i, size in enumerate(chunk_sizes):
|
||||
tids = output_ids[pos : pos + size]
|
||||
pos += size
|
||||
is_last = i == len(chunk_sizes) - 1
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"stream": True,
|
||||
"model": PARSER_MODEL,
|
||||
"generate_chunk": {
|
||||
"request_id": "stream-test",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": tids,
|
||||
"finish_reason": "stop" if is_last else None,
|
||||
}
|
||||
],
|
||||
},
|
||||
"stream_state": state,
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"prompt_token_ids": prompt_token_ids,
|
||||
"chat_request": chat_request,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
state = data["stream_state"]
|
||||
delta = data["chunk"]["choices"][0]["delta"]
|
||||
content += delta.get("content") or ""
|
||||
reasoning += delta.get("reasoning") or ""
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
idx = tc["index"]
|
||||
while len(tool_calls) <= idx:
|
||||
tool_calls.append({"id": None, "name": None, "arguments": ""})
|
||||
if tc.get("id"):
|
||||
tool_calls[idx]["id"] = tc["id"]
|
||||
fn = tc.get("function") or {}
|
||||
if fn.get("name"):
|
||||
tool_calls[idx]["name"] = fn["name"]
|
||||
if fn.get("arguments"):
|
||||
tool_calls[idx]["arguments"] += fn["arguments"]
|
||||
if is_last:
|
||||
finish_reason = data["chunk"]["choices"][0]["finish_reason"]
|
||||
if on_chunk is not None:
|
||||
on_chunk([dict(tc) for tc in tool_calls])
|
||||
|
||||
return {
|
||||
"content": content or None,
|
||||
"reasoning": reasoning or None,
|
||||
"tool_calls": tool_calls,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"chunking",
|
||||
@@ -1755,7 +1675,7 @@ async def test_stream_parsed_matches_batch_reasoning(
|
||||
"triples": [3] * (n // 3) + [n % 3],
|
||||
}[chunking]
|
||||
chunk_sizes = [c for c in chunk_sizes if c > 0]
|
||||
streamed = await _stream_chat_derender(
|
||||
streamed = await stream_chat_derender(
|
||||
parser_client,
|
||||
output_ids,
|
||||
chunk_sizes,
|
||||
@@ -1823,7 +1743,7 @@ async def test_stream_parsed_matches_batch_tool_call(parser_client, parser_token
|
||||
if tool_calls and tool_calls[0]["arguments"]:
|
||||
_snapshots.append(tool_calls[0]["arguments"])
|
||||
|
||||
streamed = await _stream_chat_derender(
|
||||
streamed = await stream_chat_derender(
|
||||
parser_client,
|
||||
output_ids,
|
||||
chunk_sizes,
|
||||
@@ -1868,7 +1788,7 @@ async def test_stream_parsed_cjk_across_chunk_boundaries(
|
||||
"messages": messages,
|
||||
"include_reasoning": True,
|
||||
}
|
||||
streamed = await _stream_chat_derender(
|
||||
streamed = await stream_chat_derender(
|
||||
parser_client,
|
||||
output_ids,
|
||||
[1] * len(output_ids),
|
||||
@@ -1896,7 +1816,7 @@ async def test_stream_parsed_include_reasoning_false(parser_client, parser_token
|
||||
"messages": messages,
|
||||
"include_reasoning": False,
|
||||
}
|
||||
streamed = await _stream_chat_derender(
|
||||
streamed = await stream_chat_derender(
|
||||
parser_client,
|
||||
output_ids,
|
||||
[1] * len(output_ids),
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers shared by the streaming chat derender tests."""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def assemble_stream(choices: list[dict]) -> dict:
|
||||
"""Fold streamed chat choices into one message.
|
||||
|
||||
A tool call index must keep the first ID it was given. On the derender
|
||||
side that is what pinning the IDs in `stream_state` guarantees.
|
||||
"""
|
||||
content = ""
|
||||
reasoning = ""
|
||||
tool_calls: list[dict] = []
|
||||
finish_reason = None
|
||||
for choice in choices:
|
||||
delta = choice["delta"]
|
||||
content += delta.get("content") or ""
|
||||
reasoning += delta.get("reasoning") or ""
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
idx = tc["index"]
|
||||
while len(tool_calls) <= idx:
|
||||
tool_calls.append({"id": None, "name": None, "arguments": ""})
|
||||
if tc.get("id"):
|
||||
assert tool_calls[idx]["id"] in (None, tc["id"]), (
|
||||
f"tool call {idx} changed ID mid stream"
|
||||
)
|
||||
tool_calls[idx]["id"] = tc["id"]
|
||||
fn = tc.get("function") or {}
|
||||
if fn.get("name"):
|
||||
tool_calls[idx]["name"] = fn["name"]
|
||||
tool_calls[idx]["arguments"] += fn.get("arguments") or ""
|
||||
finish_reason = choice.get("finish_reason") or finish_reason
|
||||
return {
|
||||
"content": content or None,
|
||||
"reasoning": reasoning or None,
|
||||
"tool_calls": tool_calls,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
|
||||
|
||||
async def stream_chat_derender(
|
||||
client: httpx.AsyncClient,
|
||||
output_ids: list[int],
|
||||
chunk_sizes: list[int],
|
||||
chat_request: dict,
|
||||
prompt_tokens: int,
|
||||
prompt_token_ids: list[int],
|
||||
finish_reason: str = "stop",
|
||||
on_chunk: Callable[[list[dict]], None] | None = None,
|
||||
) -> dict:
|
||||
"""Feed `output_ids` through the streaming chat derender endpoint in
|
||||
the given `chunk_sizes`, threading `stream_state` across calls and
|
||||
return the assembled message.
|
||||
|
||||
`finish_reason` goes on the last chunk, so a trailing size of 0 sends it
|
||||
on a chunk with no tokens.
|
||||
|
||||
If `on_chunk` is given, it is called after every chunk with the
|
||||
`tool_calls` assembled so far, letting callers assert properties of the
|
||||
intermediate deltas (e.g. monotonic argument growth) rather than only
|
||||
the final assembled result.
|
||||
"""
|
||||
state = None
|
||||
choices: list[dict] = []
|
||||
pos = 0
|
||||
for i, size in enumerate(chunk_sizes):
|
||||
tids = output_ids[pos : pos + size]
|
||||
pos += size
|
||||
is_last = i == len(chunk_sizes) - 1
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"stream": True,
|
||||
"model": chat_request["model"],
|
||||
"generate_chunk": {
|
||||
"request_id": "stream-test",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": tids,
|
||||
"finish_reason": finish_reason if is_last else None,
|
||||
}
|
||||
],
|
||||
},
|
||||
"stream_state": state,
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"prompt_token_ids": prompt_token_ids,
|
||||
"chat_request": chat_request,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
state = data["stream_state"]
|
||||
choices.extend(data["chunk"]["choices"])
|
||||
if on_chunk is not None:
|
||||
on_chunk(assemble_stream(choices)["tool_calls"])
|
||||
|
||||
return assemble_stream(choices)
|
||||
Reference in New Issue
Block a user