Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -22,6 +22,10 @@ The streaming cases do the same for `stream=true`. Each coupled chunk's
|
||||
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
|
||||
through the chunked derender path with `stream_state` threaded across calls
|
||||
the way a client would, so both parsers see the same chunk boundaries.
|
||||
|
||||
The text level cases pin `/inference/v1/generate` with `output_mode="text"`
|
||||
against `/v1/completions/derender` on the same token IDs, batch and streaming,
|
||||
plus logprob resolution and stop string handling.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -440,3 +444,241 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
|
||||
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
|
||||
if not (coupled["reasoning"] and coupled["tool_calls"]):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline text level: generate(output_mode="text") == derender(generate(...))
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TEXT_MESSAGES = [
|
||||
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
|
||||
]
|
||||
|
||||
|
||||
async def _render_token_ids(
|
||||
client: httpx.AsyncClient, messages: list[dict]
|
||||
) -> list[int]:
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/render", json={"model": MODEL, "messages": messages}
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["token_ids"]
|
||||
|
||||
|
||||
def _generate_payload(
|
||||
token_ids: list[int], output_mode: str, stream: bool = False, **sampling
|
||||
) -> dict:
|
||||
return {
|
||||
"model": MODEL,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"temperature": 0, "max_tokens": 128, **sampling},
|
||||
"output_mode": output_mode,
|
||||
"stream": stream,
|
||||
}
|
||||
|
||||
|
||||
async def _generate(
|
||||
client: httpx.AsyncClient, token_ids: list[int], output_mode: str, **sampling
|
||||
) -> dict:
|
||||
resp = await client.post(
|
||||
"/inference/v1/generate",
|
||||
json=_generate_payload(token_ids, output_mode, **sampling),
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == output_mode
|
||||
return data
|
||||
|
||||
|
||||
async def _generate_text_stream(
|
||||
client: httpx.AsyncClient, token_ids: list[int], **sampling
|
||||
) -> list[dict]:
|
||||
"""Stream generate at the text level and return the choice of every chunk
|
||||
that has one."""
|
||||
choices: list[dict] = []
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"/inference/v1/generate",
|
||||
json=_generate_payload(token_ids, "text", stream=True, **sampling),
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
assert "error" not in chunk, chunk
|
||||
assert chunk["output_mode"] == "text"
|
||||
choices.extend(chunk["choices"])
|
||||
return choices
|
||||
|
||||
|
||||
async def _derender_completion_text(
|
||||
client: httpx.AsyncClient, output_ids: list[int], finish_reason: str
|
||||
) -> str:
|
||||
resp = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"generate_responses": [
|
||||
{
|
||||
"request_id": "text-parity",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": output_ids,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
async def _derender_completion_stream_text(
|
||||
client: httpx.AsyncClient, choices: list[dict]
|
||||
) -> str:
|
||||
"""Replay each generate chunk's token IDs through the streaming
|
||||
completions derender endpoint with the same chunk boundaries."""
|
||||
state = None
|
||||
text = ""
|
||||
for choice in choices:
|
||||
resp = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"stream": True,
|
||||
"model": MODEL,
|
||||
"generate_chunk": {
|
||||
"request_id": "text-parity-stream",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": choice.get("token_ids") or [],
|
||||
"finish_reason": choice.get("finish_reason"),
|
||||
}
|
||||
],
|
||||
},
|
||||
"stream_state": state,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
state = data["stream_state"]
|
||||
text += "".join(ch["text"] for ch in data["chunk"]["choices"])
|
||||
return text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_batch(client):
|
||||
"""Inline text matches /v1/completions/derender on the same token IDs."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
inline = (await _generate(client, token_ids, "text"))["choices"][0]
|
||||
|
||||
derendered = await _derender_completion_text(
|
||||
client, inline["token_ids"], inline["finish_reason"]
|
||||
)
|
||||
|
||||
assert inline["text"]
|
||||
assert inline["text"] == derendered
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_stream(client):
|
||||
"""Streamed inline text matches both the streaming and the non-streaming
|
||||
derender of the same token IDs."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
choices = await _generate_text_stream(client, token_ids)
|
||||
inline = "".join(ch["text"] for ch in choices)
|
||||
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
|
||||
finish_reason = choices[-1]["finish_reason"]
|
||||
|
||||
assert finish_reason is not None
|
||||
assert inline == await _derender_completion_stream_text(client, choices)
|
||||
assert inline == await _derender_completion_text(client, output_ids, finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_logprobs(client):
|
||||
"""Inline logprob tokens and bytes match /derender resolving the
|
||||
token_id:N placeholders of the tokens level."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
sampling = {"logprobs": 3}
|
||||
tokens_choice = (await _generate(client, token_ids, "tokens", **sampling))[
|
||||
"choices"
|
||||
][0]
|
||||
text_choice = (await _generate(client, token_ids, "text", **sampling))["choices"][0]
|
||||
assert text_choice["token_ids"] == tokens_choice["token_ids"], (
|
||||
"greedy (temperature=0) generation was expected to be deterministic "
|
||||
"across the two generate calls"
|
||||
)
|
||||
|
||||
disagg = await _disagg(
|
||||
client,
|
||||
tokens_choice["token_ids"],
|
||||
len(token_ids),
|
||||
tokens_choice["finish_reason"],
|
||||
{"model": MODEL, "messages": TEXT_MESSAGES},
|
||||
logprobs=tokens_choice["logprobs"],
|
||||
)
|
||||
|
||||
inline_content = text_choice["logprobs"]["content"]
|
||||
derender_content = disagg["choices"][0]["logprobs"]["content"]
|
||||
assert len(inline_content) == len(derender_content)
|
||||
for inline_entry, derender_entry in zip(inline_content, derender_content):
|
||||
assert inline_entry["token"] == derender_entry["token"]
|
||||
assert inline_entry["bytes"] == derender_entry["bytes"]
|
||||
assert [
|
||||
(top["token"], top["bytes"]) for top in inline_entry["top_logprobs"]
|
||||
] == [(top["token"], top["bytes"]) for top in derender_entry["top_logprobs"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_text_parity_stop_string(client, stream):
|
||||
"""Inline text follows the engine's stop handling, like /v1/completions.
|
||||
|
||||
token_ids keep every generated token, so /derender brings the matched
|
||||
stop string back. Its text must start with the inline text and only
|
||||
differ after it.
|
||||
"""
|
||||
stop = "\n"
|
||||
token_ids = await _render_token_ids(
|
||||
client,
|
||||
[{"role": "user", "content": "Write two short lines about the sea."}],
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
"/v1/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"prompt": token_ids,
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
"stop": [stop],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
coupled = resp.json()["choices"][0]
|
||||
|
||||
if stream:
|
||||
choices = await _generate_text_stream(client, token_ids, stop=[stop])
|
||||
inline_text = "".join(ch["text"] for ch in choices)
|
||||
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
|
||||
finish_reason = choices[-1]["finish_reason"]
|
||||
else:
|
||||
inline = (await _generate(client, token_ids, "text", stop=[stop]))["choices"][0]
|
||||
inline_text = inline["text"]
|
||||
output_ids = inline["token_ids"]
|
||||
finish_reason = inline["finish_reason"]
|
||||
|
||||
assert inline_text == coupled["text"]
|
||||
assert finish_reason == coupled["finish_reason"]
|
||||
assert stop not in inline_text
|
||||
|
||||
derendered = await _derender_completion_text(client, output_ids, finish_reason)
|
||||
assert derendered.startswith(inline_text)
|
||||
if coupled["stop_reason"] != stop:
|
||||
pytest.skip("Model did not emit the stop string")
|
||||
assert derendered[len(inline_text) :].startswith(stop)
|
||||
|
||||
@@ -50,10 +50,11 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamChoice,
|
||||
GenerateTokensStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.parser import Parser
|
||||
@@ -195,10 +196,10 @@ def _make_stream_chunk(
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a GenerateStreamResponse SSE chunk."""
|
||||
return GenerateStreamResponse(
|
||||
return GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(
|
||||
GenerateTokensStreamChoice(
|
||||
index=index,
|
||||
token_ids=token_ids,
|
||||
finish_reason=finish_reason,
|
||||
@@ -233,7 +234,7 @@ def _make_usage_chunk(
|
||||
request_id: str = "test-req",
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a usage only final SSE chunk (empty choices)."""
|
||||
return GenerateStreamResponse(
|
||||
return GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=UsageInfo(
|
||||
@@ -326,11 +327,11 @@ def test_non_streaming_derender_initializes_parser_prefix(
|
||||
)
|
||||
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
|
||||
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
|
||||
response = GenerateResponse(
|
||||
response = GenerateTokensResponse(
|
||||
request_id="test",
|
||||
prompt_token_ids=[11, 12],
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0,
|
||||
token_ids=generated_ids,
|
||||
finish_reason="length",
|
||||
@@ -583,10 +584,10 @@ class TestPromptSeededLeadingSpace:
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -601,10 +602,10 @@ class TestPromptSeededLeadingSpace:
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -620,10 +621,10 @@ class TestPromptSeededLeadingSpace:
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -639,10 +640,10 @@ class TestPromptSeededLeadingSpace:
|
||||
):
|
||||
_, output_ids, _ = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -655,10 +656,10 @@ class TestPromptSeededLeadingSpace:
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices, _, _ = await derenderer.derender_completion(
|
||||
[
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -1494,10 +1495,10 @@ class TestDerenderChatStreamHarmony:
|
||||
content = "".join(d.content or "" for d in deltas)
|
||||
|
||||
batch_choices = await harmony_derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="test-harmony-batch",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -1743,10 +1744,10 @@ class TestServingDerenderStreamValidation:
|
||||
could never fire since derender_chat_stream itself rejects anything
|
||||
above 1 first."""
|
||||
serving = self._make_serving(parser_configured=False)
|
||||
two_choices = GenerateStreamResponse(
|
||||
two_choices = GenerateTokensStreamResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(index=i, token_ids=[1]) for i in range(2)
|
||||
GenerateTokensStreamChoice(index=i, token_ids=[1]) for i in range(2)
|
||||
],
|
||||
)
|
||||
request = DerenderChatStreamRequest(
|
||||
|
||||
@@ -6,6 +6,8 @@ from argparse import Namespace
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
|
||||
from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
RENDER_PATHS = {
|
||||
@@ -17,7 +19,9 @@ RENDER_PATHS = {
|
||||
"/v1/responses/render",
|
||||
}
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
ABORT_PATH = "/inference/v1/abort_requests"
|
||||
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH, ABORT_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
@@ -66,4 +70,31 @@ def test_render_routes_remain_enabled_for_render_server():
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, "/abort_requests"}
|
||||
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
|
||||
|
||||
|
||||
def test_abort_route_without_tokens_only_requires_api_key():
|
||||
"""A decode pool that loads a tokenizer serves generate without
|
||||
--tokens-only and still needs to abort, but only behind --api-key."""
|
||||
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
|
||||
|
||||
assert ABORT_PATH in paths
|
||||
assert ABORT_PATH.startswith(GUARDED_PREFIX)
|
||||
assert UNAUTHENTICATED_ABORT_PATH not in paths
|
||||
|
||||
|
||||
def test_unauthenticated_abort_route_is_not_registered_twice():
|
||||
"""The RLHF dev router registers /abort_requests before the scale-out
|
||||
routers, so attaching the scale-out one too would add a shadowed duplicate."""
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(tokens_only=True)
|
||||
|
||||
@app.post(UNAUTHENTICATED_ABORT_PATH)
|
||||
async def existing_abort_requests():
|
||||
return None
|
||||
|
||||
attach_router(app)
|
||||
|
||||
paths = [route.path for route in app.routes]
|
||||
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
|
||||
assert ABORT_PATH in paths
|
||||
|
||||
@@ -20,7 +20,8 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateTextResponse,
|
||||
GenerateTokensResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
@@ -144,6 +145,7 @@ def _make_request_output(
|
||||
num_cached_tokens: int | None = None,
|
||||
index: int = 0,
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
|
||||
text: str = "",
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=request_id,
|
||||
@@ -153,7 +155,7 @@ def _make_request_output(
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=index,
|
||||
text="",
|
||||
text=text,
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=None,
|
||||
logprobs=logprobs,
|
||||
@@ -239,7 +241,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert (
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
@@ -411,7 +413,7 @@ async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
|
||||
if isinstance(chunk, dict) and chunk.get("metrics")
|
||||
)["metrics"]["speculative_decoding"]
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.metrics is not None
|
||||
assert response.metrics.speculative_decoding is not None
|
||||
payload = response.metrics.speculative_decoding.model_dump()
|
||||
@@ -511,7 +513,7 @@ async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
|
||||
if isinstance(chunk, dict)
|
||||
)
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
@@ -540,7 +542,7 @@ async def test_non_stream_returns_final_prompt_token_ids_when_requested():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.prompt_token_ids == final_prompt_token_ids
|
||||
|
||||
|
||||
@@ -622,7 +624,7 @@ async def test_non_stream_prompt_metadata_requires_return_token_ids(
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
if return_token_ids:
|
||||
assert response.prompt_token_ids == [1, 2, 3]
|
||||
assert response.model_dump()["mm_placeholders"] == MM_PLACEHOLDERS
|
||||
@@ -652,7 +654,7 @@ async def test_non_stream_text_only_prompt_has_no_mm_placeholders():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.prompt_token_ids == [1, 2, 3]
|
||||
assert response.mm_placeholders is None
|
||||
|
||||
@@ -678,7 +680,7 @@ async def test_non_stream_encoder_decoder_omits_mm_placeholders():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.prompt_token_ids == [1, 2, 3]
|
||||
assert response.mm_placeholders is None
|
||||
|
||||
@@ -1195,3 +1197,251 @@ async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
|
||||
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
|
||||
)
|
||||
assert choice["sampling_mask"] == expected
|
||||
|
||||
|
||||
async def _collect_sse(response) -> list[Any]:
|
||||
return _parse_sse_chunks([chunk async for chunk in response])
|
||||
|
||||
|
||||
def _text_request(**kwargs) -> GenerateRequest:
|
||||
kwargs.setdefault("sampling_params", SamplingParams(max_tokens=10))
|
||||
return GenerateRequest(
|
||||
token_ids=[1, 2, 3], model=MODEL_NAME, output_mode="text", **kwargs
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_tokens_mode_omits_text_and_echoes_output_mode():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[10], text="ignored", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
dumped = response.model_dump()
|
||||
assert dumped["output_mode"] == "tokens"
|
||||
assert "text" not in dumped["choices"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_text_mode_returns_text_and_token_ids():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10, 20],
|
||||
text="Hello",
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request())
|
||||
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert response.output_mode == "text"
|
||||
assert response.choices[0].text == "Hello"
|
||||
assert response.choices[0].token_ids == [10, 20]
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert response.usage is not None
|
||||
assert response.usage.completion_tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("no_tokenizer", ["tokens_only", "skip_tokenizer_init"])
|
||||
async def test_text_mode_rejected_without_tokenizer(no_tokenizer: str):
|
||||
"""The no-op detokenizer returns "", so text mode must fail instead
|
||||
of answering 200 with empty text."""
|
||||
engine = _mock_engine()
|
||||
kwargs: dict[str, Any] = {}
|
||||
if no_tokenizer == "tokens_only":
|
||||
kwargs["force_no_detokenize"] = True
|
||||
else:
|
||||
engine.model_config.skip_tokenizer_init = True
|
||||
engine.renderer = _build_renderer(engine.model_config)
|
||||
engine.generate = MagicMock()
|
||||
serving = _build_serving_tokens(engine, **kwargs)
|
||||
|
||||
response = await serving.serve_tokens(_text_request())
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert "requires a tokenizer" in response.error.message
|
||||
engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_mode_logprobs_have_decoded_tokens_and_bytes():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
text="é",
|
||||
logprobs=[
|
||||
{
|
||||
10: Logprob(logprob=-0.5, decoded_token="é"),
|
||||
11: Logprob(logprob=-1.5, decoded_token="e"),
|
||||
}
|
||||
],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=2))
|
||||
)
|
||||
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
logprobs = response.choices[0].logprobs
|
||||
assert logprobs is not None and logprobs.content is not None
|
||||
entry = logprobs.content[0]
|
||||
assert (entry.token, entry.bytes) == ("é", [195, 169])
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("é", [195, 169]),
|
||||
("e", [101]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_mode_logprobs_keep_placeholders_with_return_tokens_as_ids():
|
||||
"""--return-tokens-as-token-ids keeps token_id:N placeholders, as it does
|
||||
on /v1/completions, while text is still decoded."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
text="é",
|
||||
logprobs=[{10: Logprob(logprob=-0.5, decoded_token="é")}],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine, return_tokens_as_token_ids=True)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=1))
|
||||
)
|
||||
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert response.choices[0].text == "é"
|
||||
logprobs = response.choices[0].logprobs
|
||||
assert logprobs is not None and logprobs.content is not None
|
||||
entry = logprobs.content[0]
|
||||
assert (entry.token, entry.bytes) == ("token_id:10", None)
|
||||
assert [t.token for t in entry.top_logprobs] == ["token_id:10"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_emits_text_deltas():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], text="Hel")
|
||||
yield _make_request_output("req-1", token_ids=[20], text="lo")
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[30], text=" world", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request(stream=True))
|
||||
parsed = await _collect_sse(response)
|
||||
|
||||
assert parsed[-1] == "[DONE]"
|
||||
data_chunks = parsed[:-1]
|
||||
assert {c["output_mode"] for c in data_chunks} == {"text"}
|
||||
assert [c["choices"][0]["text"] for c in data_chunks] == ["Hel", "lo", " world"]
|
||||
assert [c["choices"][0]["token_ids"] for c in data_chunks] == [[10], [20], [30]]
|
||||
assert data_chunks[-1]["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_emits_chunks_without_new_token_ids():
|
||||
"""Text flushed from the stop string buffer and the final abort output
|
||||
carry no token IDs but must still reach the client."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], text="a")
|
||||
yield _make_request_output("req-1", token_ids=[], text="b")
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[], text="", finish_reason="abort", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request(stream=True))
|
||||
data_chunks = (await _collect_sse(response))[:-1]
|
||||
|
||||
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b", ""]
|
||||
assert data_chunks[1]["choices"][0]["token_ids"] == []
|
||||
assert data_chunks[2]["choices"][0]["finish_reason"] == "abort"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_skips_outputs_without_text_or_finish():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], text="a")
|
||||
yield _make_request_output("req-1", token_ids=[], text="")
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[20], text="b", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request(stream=True))
|
||||
data_chunks = (await _collect_sse(response))[:-1]
|
||||
|
||||
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_usage_chunk_echoes_output_mode():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[10], text="a", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
_text_request(stream=True, stream_options=StreamOptions(include_usage=True))
|
||||
)
|
||||
parsed = await _collect_sse(response)
|
||||
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["output_mode"] == "text"
|
||||
assert usage_chunk["choices"] == []
|
||||
assert usage_chunk["usage"]["completion_tokens"] == 1
|
||||
|
||||
@@ -10,9 +10,21 @@ fail loudly if the validator semantics ever drift.
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateStreamResponse,
|
||||
GenerateTextResponse,
|
||||
GenerateTextStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
@@ -110,3 +122,120 @@ def test_generate_request_rejects_placeholder_outside_prompt():
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_output_mode_defaults_to_tokens():
|
||||
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_output_mode_text_is_accepted():
|
||||
payload = {**_base_payload(), "output_mode": "text"}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
|
||||
def test_output_mode_rejects_unsupported_levels(output_mode):
|
||||
"""Levels the server does not implement fail instead of falling back to
|
||||
tokens which would answer 200 with the wrong response shape."""
|
||||
payload = {**_base_payload(), "output_mode": output_mode}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_text_mode_rejects_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "text",
|
||||
}
|
||||
with pytest.raises(ValidationError, match="detokenize"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_tokens_mode_allows_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "tokens",
|
||||
}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_response_without_output_mode_parses_as_tokens():
|
||||
"""Existing clients and older servers never send output_mode."""
|
||||
parsed = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensResponse)
|
||||
assert parsed.output_mode == "tokens"
|
||||
|
||||
|
||||
def test_stream_chunk_without_output_mode_parses_as_tokens():
|
||||
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_and_chunk_dispatch_on_output_mode():
|
||||
response = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert isinstance(chunk, GenerateTextStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_requires_text_on_every_choice():
|
||||
with pytest.raises(ValidationError, match="text"):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0}]}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
|
||||
def test_response_rejects_unknown_output_mode(output_mode):
|
||||
with pytest.raises(ValidationError):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": output_mode, "choices": []}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_accept_responses_without_output_mode():
|
||||
DerenderChatRequest.model_validate(
|
||||
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
|
||||
)
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{
|
||||
"stream": True,
|
||||
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_reject_text_responses():
|
||||
"""Text responses are already detokenized with stop strings applied, so
|
||||
re-decoding their token_ids would silently undo the truncation."""
|
||||
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatRequest.model_validate({"generate_response": text})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
|
||||
|
||||
def test_tokens_response_echoes_output_mode_without_text():
|
||||
dumped = GenerateTokensResponse(
|
||||
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
|
||||
).model_dump()
|
||||
assert dumped["output_mode"] == "tokens"
|
||||
assert "text" not in dumped["choices"][0]
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.v1.engine.detokenizer import check_stop_strings
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
ABORT_ENDPOINT = "/inference/v1/abort_requests"
|
||||
|
||||
|
||||
def get_vocab_size(model_name):
|
||||
@@ -590,3 +591,187 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
|
||||
return tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
|
||||
def _text_mode_payload(token_ids, **sampling_params) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
|
||||
"output_mode": "text",
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == "text"
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
chat_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
|
||||
|
||||
assert data["choices"][0]["text"] == chat_content
|
||||
assert data["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
payload = _text_mode_payload(token_ids)
|
||||
|
||||
non_stream = await client.post(GEN_ENDPOINT, json=payload)
|
||||
non_stream.raise_for_status()
|
||||
|
||||
chunks = []
|
||||
async with client.stream(
|
||||
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json.loads(line[len("data: ") :]))
|
||||
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
|
||||
assert streamed_text == non_stream.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
pytest.param(
|
||||
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
|
||||
),
|
||||
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
|
||||
],
|
||||
)
|
||||
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
|
||||
payload = {
|
||||
**_text_mode_payload([1, 2, 3], max_tokens=5),
|
||||
**overrides,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
|
||||
assert resp.status_code == 400
|
||||
assert expected in resp.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
async def test_text_mode_rejected_when_tokens_only(client):
|
||||
"""Without a tokenizer the detokenizer returns "", so the request must
|
||||
fail instead of answering 200 with empty text."""
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
|
||||
assert resp.status_code == 400
|
||||
assert "requires a tokenizer" in resp.json()["error"]["message"]
|
||||
|
||||
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
|
||||
payload["sampling_params"]["detokenize"] = False
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_delivers_abort_finish_chunk(
|
||||
client, tokenizer, messages
|
||||
):
|
||||
"""The final output after an abort has no new token IDs, so it
|
||||
reaches a text stream only because text mode emits finish-only chunks."""
|
||||
payload = {
|
||||
**_text_mode_payload(
|
||||
_chat_prompt_token_ids(tokenizer, messages),
|
||||
max_tokens=900,
|
||||
ignore_eos=True,
|
||||
),
|
||||
"stream": True,
|
||||
"request_id": "text-abort-e2e",
|
||||
}
|
||||
|
||||
chunks = []
|
||||
aborted = False
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
chunks.append(chunk)
|
||||
if not aborted and chunk["choices"] and chunk["choices"][0]["text"]:
|
||||
abort = await client.post(
|
||||
ABORT_ENDPOINT,
|
||||
json={"request_ids": [chunk["request_id"]]},
|
||||
)
|
||||
assert abort.status_code == 200
|
||||
aborted = True
|
||||
|
||||
assert aborted
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
final_choice = chunks[-1]["choices"][0]
|
||||
assert final_choice["finish_reason"] == "abort"
|
||||
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
|
||||
assert generated < 900
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
async def test_abort_requests_is_served_without_tokens_only(client):
|
||||
resp = await client.post(
|
||||
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
resp = await client.post(ABORT_ENDPOINT, json={})
|
||||
assert resp.status_code == 400
|
||||
|
||||
# The unauthenticated path is only served with --tokens-only.
|
||||
resp = await client.post(
|
||||
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -36,3 +38,50 @@ def test_logprobs_minus_one_emits_all_tokens():
|
||||
num_output_top_logprobs=-1,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
|
||||
def test_tokens_level_logprobs_have_no_bytes():
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.bytes is None
|
||||
assert all(t.bytes is None for t in entry.top_logprobs)
|
||||
|
||||
|
||||
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
tokenizer = MagicMock()
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[
|
||||
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert (entry.token, entry.bytes) == ("é", [195, 169])
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("é", [195, 169]),
|
||||
("e", [101]),
|
||||
]
|
||||
tokenizer.decode.assert_not_called()
|
||||
|
||||
|
||||
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
assert result.content[0].top_logprobs[0].token == "<7>"
|
||||
assert result.content[1].bytes == list(b"<9>")
|
||||
|
||||
Reference in New Issue
Block a user