[Frontend] Add output_mode to /inference/v1/generate (RFC #56851 Phase 1) (#58588)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-03 19:38:25 +08:00
committed by GitHub
parent 5f30fc7031
commit bc21cba967
18 changed files with 1356 additions and 169 deletions
@@ -22,6 +22,10 @@ The streaming cases do the same for `stream=true`. Each coupled chunk's
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
through the chunked derender path with `stream_state` threaded across calls
the way a client would, so both parsers see the same chunk boundaries.
The text level cases pin `/inference/v1/generate` with `output_mode="text"`
against `/v1/completions/derender` on the same token IDs, batch and streaming,
plus logprob resolution and stop string handling.
"""
import json
@@ -440,3 +444,241 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
if not (coupled["reasoning"] and coupled["tool_calls"]):
pytest.skip("Model did not emit both a <think> block and a tool call")
# ---------------------------------------------------------------------------
# Inline text level: generate(output_mode="text") == derender(generate(...))
# ---------------------------------------------------------------------------
TEXT_MESSAGES = [
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
]
async def _render_token_ids(
client: httpx.AsyncClient, messages: list[dict]
) -> list[int]:
resp = await client.post(
"/v1/chat/completions/render", json={"model": MODEL, "messages": messages}
)
assert resp.status_code == 200, resp.text
return resp.json()["token_ids"]
def _generate_payload(
token_ids: list[int], output_mode: str, stream: bool = False, **sampling
) -> dict:
return {
"model": MODEL,
"token_ids": token_ids,
"sampling_params": {"temperature": 0, "max_tokens": 128, **sampling},
"output_mode": output_mode,
"stream": stream,
}
async def _generate(
client: httpx.AsyncClient, token_ids: list[int], output_mode: str, **sampling
) -> dict:
resp = await client.post(
"/inference/v1/generate",
json=_generate_payload(token_ids, output_mode, **sampling),
)
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["output_mode"] == output_mode
return data
async def _generate_text_stream(
client: httpx.AsyncClient, token_ids: list[int], **sampling
) -> list[dict]:
"""Stream generate at the text level and return the choice of every chunk
that has one."""
choices: list[dict] = []
async with client.stream(
"POST",
"/inference/v1/generate",
json=_generate_payload(token_ids, "text", stream=True, **sampling),
) as resp:
assert resp.status_code == 200, await resp.aread()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
assert "error" not in chunk, chunk
assert chunk["output_mode"] == "text"
choices.extend(chunk["choices"])
return choices
async def _derender_completion_text(
client: httpx.AsyncClient, output_ids: list[int], finish_reason: str
) -> str:
resp = await client.post(
"/v1/completions/derender",
json={
"model": MODEL,
"generate_responses": [
{
"request_id": "text-parity",
"choices": [
{
"index": 0,
"token_ids": output_ids,
"finish_reason": finish_reason,
}
],
}
],
},
)
assert resp.status_code == 200, resp.text
return resp.json()["choices"][0]["text"]
async def _derender_completion_stream_text(
client: httpx.AsyncClient, choices: list[dict]
) -> str:
"""Replay each generate chunk's token IDs through the streaming
completions derender endpoint with the same chunk boundaries."""
state = None
text = ""
for choice in choices:
resp = await client.post(
"/v1/completions/derender",
json={
"stream": True,
"model": MODEL,
"generate_chunk": {
"request_id": "text-parity-stream",
"choices": [
{
"index": 0,
"token_ids": choice.get("token_ids") or [],
"finish_reason": choice.get("finish_reason"),
}
],
},
"stream_state": state,
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
state = data["stream_state"]
text += "".join(ch["text"] for ch in data["chunk"]["choices"])
return text
@pytest.mark.asyncio
async def test_text_parity_batch(client):
"""Inline text matches /v1/completions/derender on the same token IDs."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
inline = (await _generate(client, token_ids, "text"))["choices"][0]
derendered = await _derender_completion_text(
client, inline["token_ids"], inline["finish_reason"]
)
assert inline["text"]
assert inline["text"] == derendered
@pytest.mark.asyncio
async def test_text_parity_stream(client):
"""Streamed inline text matches both the streaming and the non-streaming
derender of the same token IDs."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
choices = await _generate_text_stream(client, token_ids)
inline = "".join(ch["text"] for ch in choices)
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
finish_reason = choices[-1]["finish_reason"]
assert finish_reason is not None
assert inline == await _derender_completion_stream_text(client, choices)
assert inline == await _derender_completion_text(client, output_ids, finish_reason)
@pytest.mark.asyncio
async def test_text_parity_logprobs(client):
"""Inline logprob tokens and bytes match /derender resolving the
token_id:N placeholders of the tokens level."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
sampling = {"logprobs": 3}
tokens_choice = (await _generate(client, token_ids, "tokens", **sampling))[
"choices"
][0]
text_choice = (await _generate(client, token_ids, "text", **sampling))["choices"][0]
assert text_choice["token_ids"] == tokens_choice["token_ids"], (
"greedy (temperature=0) generation was expected to be deterministic "
"across the two generate calls"
)
disagg = await _disagg(
client,
tokens_choice["token_ids"],
len(token_ids),
tokens_choice["finish_reason"],
{"model": MODEL, "messages": TEXT_MESSAGES},
logprobs=tokens_choice["logprobs"],
)
inline_content = text_choice["logprobs"]["content"]
derender_content = disagg["choices"][0]["logprobs"]["content"]
assert len(inline_content) == len(derender_content)
for inline_entry, derender_entry in zip(inline_content, derender_content):
assert inline_entry["token"] == derender_entry["token"]
assert inline_entry["bytes"] == derender_entry["bytes"]
assert [
(top["token"], top["bytes"]) for top in inline_entry["top_logprobs"]
] == [(top["token"], top["bytes"]) for top in derender_entry["top_logprobs"]]
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_text_parity_stop_string(client, stream):
"""Inline text follows the engine's stop handling, like /v1/completions.
token_ids keep every generated token, so /derender brings the matched
stop string back. Its text must start with the inline text and only
differ after it.
"""
stop = "\n"
token_ids = await _render_token_ids(
client,
[{"role": "user", "content": "Write two short lines about the sea."}],
)
resp = await client.post(
"/v1/completions",
json={
"model": MODEL,
"prompt": token_ids,
"temperature": 0,
"max_tokens": 128,
"stop": [stop],
},
)
assert resp.status_code == 200, resp.text
coupled = resp.json()["choices"][0]
if stream:
choices = await _generate_text_stream(client, token_ids, stop=[stop])
inline_text = "".join(ch["text"] for ch in choices)
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
finish_reason = choices[-1]["finish_reason"]
else:
inline = (await _generate(client, token_ids, "text", stop=[stop]))["choices"][0]
inline_text = inline["text"]
output_ids = inline["token_ids"]
finish_reason = inline["finish_reason"]
assert inline_text == coupled["text"]
assert finish_reason == coupled["finish_reason"]
assert stop not in inline_text
derendered = await _derender_completion_text(client, output_ids, finish_reason)
assert derendered.startswith(inline_text)
if coupled["stop_reason"] != stop:
pytest.skip("Model did not emit the stop string")
assert derendered[len(inline_text) :].startswith(stop)
@@ -50,10 +50,11 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamChoice,
GenerateTokensStreamResponse,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.parser import Parser
@@ -195,10 +196,10 @@ def _make_stream_chunk(
prompt_token_ids: list[int] | None = None,
) -> GenerateStreamResponse:
"""Build a GenerateStreamResponse SSE chunk."""
return GenerateStreamResponse(
return GenerateTokensStreamResponse(
request_id=request_id,
choices=[
GenerateResponseStreamChoice(
GenerateTokensStreamChoice(
index=index,
token_ids=token_ids,
finish_reason=finish_reason,
@@ -233,7 +234,7 @@ def _make_usage_chunk(
request_id: str = "test-req",
) -> GenerateStreamResponse:
"""Build a usage only final SSE chunk (empty choices)."""
return GenerateStreamResponse(
return GenerateTokensStreamResponse(
request_id=request_id,
choices=[],
usage=UsageInfo(
@@ -326,11 +327,11 @@ def test_non_streaming_derender_initializes_parser_prefix(
)
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
response = GenerateResponse(
response = GenerateTokensResponse(
request_id="test",
prompt_token_ids=[11, 12],
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0,
token_ids=generated_ids,
finish_reason="length",
@@ -583,10 +584,10 @@ class TestPromptSeededLeadingSpace:
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -601,10 +602,10 @@ class TestPromptSeededLeadingSpace:
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -620,10 +621,10 @@ class TestPromptSeededLeadingSpace:
prompt_ids, output_ids, expected = leading_space_ids
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -639,10 +640,10 @@ class TestPromptSeededLeadingSpace:
):
_, output_ids, _ = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -655,10 +656,10 @@ class TestPromptSeededLeadingSpace:
prompt_ids, output_ids, expected = leading_space_ids
choices, _, _ = await derenderer.derender_completion(
[
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -1494,10 +1495,10 @@ class TestDerenderChatStreamHarmony:
content = "".join(d.content or "" for d in deltas)
batch_choices = await harmony_derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="test-harmony-batch",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -1743,10 +1744,10 @@ class TestServingDerenderStreamValidation:
could never fire since derender_chat_stream itself rejects anything
above 1 first."""
serving = self._make_serving(parser_configured=False)
two_choices = GenerateStreamResponse(
two_choices = GenerateTokensStreamResponse(
request_id="t",
choices=[
GenerateResponseStreamChoice(index=i, token_ids=[1]) for i in range(2)
GenerateTokensStreamChoice(index=i, token_ids=[1]) for i in range(2)
],
)
request = DerenderChatStreamRequest(
+33 -2
View File
@@ -6,6 +6,8 @@ from argparse import Namespace
from fastapi import FastAPI
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX
from vllm.tasks import SupportedTask
RENDER_PATHS = {
@@ -17,7 +19,9 @@ RENDER_PATHS = {
"/v1/responses/render",
}
GENERATE_PATH = "/inference/v1/generate"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
ABORT_PATH = "/inference/v1/abort_requests"
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH, ABORT_PATH}
def registered_paths(
@@ -66,4 +70,31 @@ def test_render_routes_remain_enabled_for_render_server():
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, "/abort_requests"}
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
def test_abort_route_without_tokens_only_requires_api_key():
"""A decode pool that loads a tokenizer serves generate without
--tokens-only and still needs to abort, but only behind --api-key."""
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
assert ABORT_PATH in paths
assert ABORT_PATH.startswith(GUARDED_PREFIX)
assert UNAUTHENTICATED_ABORT_PATH not in paths
def test_unauthenticated_abort_route_is_not_registered_twice():
"""The RLHF dev router registers /abort_requests before the scale-out
routers, so attaching the scale-out one too would add a shadowed duplicate."""
app = FastAPI()
app.state.args = Namespace(tokens_only=True)
@app.post(UNAUTHENTICATED_ABORT_PATH)
async def existing_abort_requests():
return None
attach_router(app)
paths = [route.path for route in app.routes]
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
assert ABORT_PATH in paths
@@ -20,7 +20,8 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
GenerateResponse,
GenerateTextResponse,
GenerateTokensResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
@@ -144,6 +145,7 @@ def _make_request_output(
num_cached_tokens: int | None = None,
index: int = 0,
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
text: str = "",
) -> RequestOutput:
return RequestOutput(
request_id=request_id,
@@ -153,7 +155,7 @@ def _make_request_output(
outputs=[
CompletionOutput(
index=index,
text="",
text=text,
token_ids=token_ids,
cumulative_logprob=None,
logprobs=logprobs,
@@ -239,7 +241,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert (
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
@@ -411,7 +413,7 @@ async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
if isinstance(chunk, dict) and chunk.get("metrics")
)["metrics"]["speculative_decoding"]
else:
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.metrics is not None
assert response.metrics.speculative_decoding is not None
payload = response.metrics.speculative_decoding.model_dump()
@@ -511,7 +513,7 @@ async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
if isinstance(chunk, dict)
)
else:
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.metrics is None
@@ -540,7 +542,7 @@ async def test_non_stream_returns_final_prompt_token_ids_when_requested():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.prompt_token_ids == final_prompt_token_ids
@@ -622,7 +624,7 @@ async def test_non_stream_prompt_metadata_requires_return_token_ids(
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
if return_token_ids:
assert response.prompt_token_ids == [1, 2, 3]
assert response.model_dump()["mm_placeholders"] == MM_PLACEHOLDERS
@@ -652,7 +654,7 @@ async def test_non_stream_text_only_prompt_has_no_mm_placeholders():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.prompt_token_ids == [1, 2, 3]
assert response.mm_placeholders is None
@@ -678,7 +680,7 @@ async def test_non_stream_encoder_decoder_omits_mm_placeholders():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.prompt_token_ids == [1, 2, 3]
assert response.mm_placeholders is None
@@ -1195,3 +1197,251 @@ async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
)
assert choice["sampling_mask"] == expected
async def _collect_sse(response) -> list[Any]:
return _parse_sse_chunks([chunk async for chunk in response])
def _text_request(**kwargs) -> GenerateRequest:
kwargs.setdefault("sampling_params", SamplingParams(max_tokens=10))
return GenerateRequest(
token_ids=[1, 2, 3], model=MODEL_NAME, output_mode="text", **kwargs
)
@pytest.mark.asyncio
async def test_non_stream_tokens_mode_omits_text_and_echoes_output_mode():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], text="ignored", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
)
)
assert isinstance(response, GenerateTokensResponse)
dumped = response.model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
@pytest.mark.asyncio
async def test_non_stream_text_mode_returns_text_and_token_ids():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10, 20],
text="Hello",
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request())
assert isinstance(response, GenerateTextResponse)
assert response.output_mode == "text"
assert response.choices[0].text == "Hello"
assert response.choices[0].token_ids == [10, 20]
assert response.choices[0].finish_reason == "stop"
assert response.usage is not None
assert response.usage.completion_tokens == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("no_tokenizer", ["tokens_only", "skip_tokenizer_init"])
async def test_text_mode_rejected_without_tokenizer(no_tokenizer: str):
"""The no-op detokenizer returns "", so text mode must fail instead
of answering 200 with empty text."""
engine = _mock_engine()
kwargs: dict[str, Any] = {}
if no_tokenizer == "tokens_only":
kwargs["force_no_detokenize"] = True
else:
engine.model_config.skip_tokenizer_init = True
engine.renderer = _build_renderer(engine.model_config)
engine.generate = MagicMock()
serving = _build_serving_tokens(engine, **kwargs)
response = await serving.serve_tokens(_text_request())
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert "requires a tokenizer" in response.error.message
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_text_mode_logprobs_have_decoded_tokens_and_bytes():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
text="é",
logprobs=[
{
10: Logprob(logprob=-0.5, decoded_token="é"),
11: Logprob(logprob=-1.5, decoded_token="e"),
}
],
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=2))
)
assert isinstance(response, GenerateTextResponse)
logprobs = response.choices[0].logprobs
assert logprobs is not None and logprobs.content is not None
entry = logprobs.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
@pytest.mark.asyncio
async def test_text_mode_logprobs_keep_placeholders_with_return_tokens_as_ids():
"""--return-tokens-as-token-ids keeps token_id:N placeholders, as it does
on /v1/completions, while text is still decoded."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
text="é",
logprobs=[{10: Logprob(logprob=-0.5, decoded_token="é")}],
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine, return_tokens_as_token_ids=True)
response = await serving.serve_tokens(
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=1))
)
assert isinstance(response, GenerateTextResponse)
assert response.choices[0].text == "é"
logprobs = response.choices[0].logprobs
assert logprobs is not None and logprobs.content is not None
entry = logprobs.content[0]
assert (entry.token, entry.bytes) == ("token_id:10", None)
assert [t.token for t in entry.top_logprobs] == ["token_id:10"]
@pytest.mark.asyncio
async def test_stream_text_mode_emits_text_deltas():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], text="Hel")
yield _make_request_output("req-1", token_ids=[20], text="lo")
yield _make_request_output(
"req-1", token_ids=[30], text=" world", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request(stream=True))
parsed = await _collect_sse(response)
assert parsed[-1] == "[DONE]"
data_chunks = parsed[:-1]
assert {c["output_mode"] for c in data_chunks} == {"text"}
assert [c["choices"][0]["text"] for c in data_chunks] == ["Hel", "lo", " world"]
assert [c["choices"][0]["token_ids"] for c in data_chunks] == [[10], [20], [30]]
assert data_chunks[-1]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
async def test_stream_text_mode_emits_chunks_without_new_token_ids():
"""Text flushed from the stop string buffer and the final abort output
carry no token IDs but must still reach the client."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], text="a")
yield _make_request_output("req-1", token_ids=[], text="b")
yield _make_request_output(
"req-1", token_ids=[], text="", finish_reason="abort", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request(stream=True))
data_chunks = (await _collect_sse(response))[:-1]
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b", ""]
assert data_chunks[1]["choices"][0]["token_ids"] == []
assert data_chunks[2]["choices"][0]["finish_reason"] == "abort"
@pytest.mark.asyncio
async def test_stream_text_mode_skips_outputs_without_text_or_finish():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], text="a")
yield _make_request_output("req-1", token_ids=[], text="")
yield _make_request_output(
"req-1", token_ids=[20], text="b", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request(stream=True))
data_chunks = (await _collect_sse(response))[:-1]
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b"]
@pytest.mark.asyncio
async def test_stream_text_mode_usage_chunk_echoes_output_mode():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], text="a", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
_text_request(stream=True, stream_options=StreamOptions(include_usage=True))
)
parsed = await _collect_sse(response)
usage_chunk = parsed[-2]
assert usage_chunk["output_mode"] == "text"
assert usage_chunk["choices"] == []
assert usage_chunk["usage"]["completion_tokens"] == 1
@@ -10,9 +10,21 @@ fail loudly if the validator semantics ever drift.
import json
import pytest
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
GenerateStreamResponse,
GenerateTextResponse,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
@@ -110,3 +122,120 @@ def test_generate_request_rejects_placeholder_outside_prompt():
kwargs_data={"image": ["encoded"]},
),
)
def test_output_mode_defaults_to_tokens():
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
def test_output_mode_text_is_accepted():
payload = {**_base_payload(), "output_mode": "text"}
assert GenerateRequest.model_validate(payload).output_mode == "text"
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
def test_output_mode_rejects_unsupported_levels(output_mode):
"""Levels the server does not implement fail instead of falling back to
tokens which would answer 200 with the wrong response shape."""
payload = {**_base_payload(), "output_mode": output_mode}
with pytest.raises(ValidationError, match="output_mode"):
GenerateRequest.model_validate(payload)
def test_text_mode_rejects_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "text",
}
with pytest.raises(ValidationError, match="detokenize"):
GenerateRequest.model_validate(payload)
def test_tokens_mode_allows_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "tokens",
}
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
def test_response_without_output_mode_parses_as_tokens():
"""Existing clients and older servers never send output_mode."""
parsed = TypeAdapter(GenerateResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensResponse)
assert parsed.output_mode == "tokens"
def test_stream_chunk_without_output_mode_parses_as_tokens():
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensStreamResponse)
def test_text_response_and_chunk_dispatch_on_output_mode():
response = TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
assert isinstance(response, GenerateTextResponse)
assert isinstance(chunk, GenerateTextStreamResponse)
def test_text_response_requires_text_on_every_choice():
with pytest.raises(ValidationError, match="text"):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0}]}
)
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
def test_response_rejects_unknown_output_mode(output_mode):
with pytest.raises(ValidationError):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": output_mode, "choices": []}
)
def test_derender_requests_accept_responses_without_output_mode():
DerenderChatRequest.model_validate(
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
)
DerenderChatStreamRequest.model_validate(
{
"stream": True,
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
}
)
def test_derender_requests_reject_text_responses():
"""Text responses are already detokenized with stop strings applied, so
re-decoding their token_ids would silently undo the truncation."""
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatRequest.model_validate({"generate_response": text})
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
def test_tokens_response_echoes_output_mode_without_text():
dumped = GenerateTokensResponse(
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
).model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
@@ -19,6 +19,7 @@ from vllm.v1.engine.detokenizer import check_stop_strings
MODEL_NAME = "Qwen/Qwen3-0.6B"
GEN_ENDPOINT = "/inference/v1/generate"
ABORT_ENDPOINT = "/inference/v1/abort_requests"
def get_vocab_size(model_name):
@@ -590,3 +591,187 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
return tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
def _text_mode_payload(token_ids, **sampling_params) -> dict:
return {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
"output_mode": "text",
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
resp.raise_for_status()
data = resp.json()
assert data["output_mode"] == "text"
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": {"enable_thinking": False},
}
chat_resp = await client.post("/v1/chat/completions", json=payload)
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
assert data["choices"][0]["text"] == chat_content
assert data["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
payload = _text_mode_payload(token_ids)
non_stream = await client.post(GEN_ENDPOINT, json=payload)
non_stream.raise_for_status()
chunks = []
async with client.stream(
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json.loads(line[len("data: ") :]))
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
assert chunks[-1]["choices"][0]["finish_reason"] is not None
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
assert streamed_text == non_stream.json()["choices"][0]["text"]
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"overrides, expected",
[
pytest.param(
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
),
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
],
)
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
payload = {
**_text_mode_payload([1, 2, 3], max_tokens=5),
**overrides,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert expected in resp.json()["error"]["message"]
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_text_mode_rejected_when_tokens_only(client):
"""Without a tokenizer the detokenizer returns "", so the request must
fail instead of answering 200 with empty text."""
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
assert resp.status_code == 400
assert "requires a tokenizer" in resp.json()["error"]["message"]
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
payload["sampling_params"]["detokenize"] = False
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_delivers_abort_finish_chunk(
client, tokenizer, messages
):
"""The final output after an abort has no new token IDs, so it
reaches a text stream only because text mode emits finish-only chunks."""
payload = {
**_text_mode_payload(
_chat_prompt_token_ids(tokenizer, messages),
max_tokens=900,
ignore_eos=True,
),
"stream": True,
"request_id": "text-abort-e2e",
}
chunks = []
aborted = False
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
chunks.append(chunk)
if not aborted and chunk["choices"] and chunk["choices"][0]["text"]:
abort = await client.post(
ABORT_ENDPOINT,
json={"request_ids": [chunk["request_id"]]},
)
assert abort.status_code == 200
aborted = True
assert aborted
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
final_choice = chunks[-1]["choices"][0]
assert final_choice["finish_reason"] == "abort"
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
assert generated < 900
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_abort_requests_is_served_without_tokens_only(client):
resp = await client.post(
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 200
resp = await client.post(ABORT_ENDPOINT, json={})
assert resp.status_code == 400
# The unauthenticated path is only served with --tokens-only.
resp = await client.post(
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 404
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
@@ -36,3 +38,50 @@ def test_logprobs_minus_one_emits_all_tokens():
num_output_top_logprobs=-1,
)
assert len(result.content[0].top_logprobs) == 2
def test_tokens_level_logprobs_have_no_bytes():
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=2,
)
entry = result.content[0]
assert entry.bytes is None
assert all(t.bytes is None for t in entry.top_logprobs)
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
tokenizer = MagicMock()
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
],
num_output_top_logprobs=2,
tokenizer=tokenizer,
)
entry = result.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
tokenizer.decode.assert_not_called()
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
tokenizer = MagicMock()
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
num_output_top_logprobs=1,
tokenizer=tokenizer,
)
assert [e.token for e in result.content] == ["<7>", "<9>"]
assert result.content[0].top_logprobs[0].token == "<7>"
assert result.content[1].bytes == list(b"<9>")