[Bugfix][Frontend] Seed derender detokenization from the prompt (#59046)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-02 03:17:56 +00:00
committed by GitHub
parent 8faacd707f
commit fcfacc3499
9 changed files with 760 additions and 163 deletions
@@ -27,6 +27,11 @@ async def client(server):
yield http_client
@pytest.fixture(scope="module")
def tokenizer():
return get_tokenizer(MODEL_NAME)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -372,6 +377,58 @@ async def test_derender_chat_model_omitted_resolves_served_name(client):
assert response.json()["model"] == MODEL_NAME
@pytest.mark.asyncio
async def test_derender_chat_leading_space_seeded_from_prompt(client, tokenizer):
"""prompt_token_ids keeps the first token's leading space on this
Metaspace tokenizer, matching the coupled path."""
gen_req = await _render_chat(client)
prompt_token_ids = gen_req["token_ids"]
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
unseeded = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(output_ids),
},
)
seeded = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(output_ids),
"prompt_token_ids": prompt_token_ids,
},
)
assert unseeded.status_code == 200
assert seeded.status_code == 200
unseeded_content = unseeded.json()["choices"][0]["message"]["content"]
seeded_content = seeded.json()["choices"][0]["message"]["content"]
assert not unseeded_content.startswith(" ")
assert seeded_content.startswith(" ")
assert seeded_content.lstrip(" ") == unseeded_content
@pytest.mark.asyncio
async def test_derender_chat_oversized_prompt_token_ids_rejected(client):
"""prompt_token_ids longer than max_model_len returns 400."""
gen_req = await _render_chat(client)
synthetic_ids = gen_req["token_ids"][:3]
oversized_prompt_ids = [42] * 1_000_000
response = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(synthetic_ids),
"prompt_token_ids": oversized_prompt_ids,
},
)
assert response.status_code == 400
assert "max_model_len" in response.json()["error"]["message"]
# ---------------------------------------------------------------------------
# Completion derender tests
# ---------------------------------------------------------------------------
@@ -486,6 +543,84 @@ async def test_derender_completion_prompt_tokens_length_mismatch(client):
assert response.status_code == 400
@pytest.mark.asyncio
async def test_derender_completion_prompt_token_ids_length_mismatch(client):
"""len(prompt_token_ids) != len(generate_responses) returns 400, the
same as the existing prompt_tokens length check."""
gr1 = await _render_completion(client, "Hello")
ids1 = gr1["token_ids"][:3]
response = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(ids1, gr1["request_id"]),
],
"prompt_token_ids": [[1, 2], [3, 4]],
},
)
assert response.status_code == 400
@pytest.mark.asyncio
async def test_derender_completion_leading_space_seeded_from_prompt(client, tokenizer):
"""Completions counterpart of
test_derender_chat_leading_space_seeded_from_prompt."""
gr1 = await _render_completion(client, "Hello world")
prompt_token_ids = gr1["token_ids"]
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
unseeded = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(output_ids, gr1["request_id"]),
],
},
)
seeded = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(output_ids, gr1["request_id"]),
],
"prompt_token_ids": [prompt_token_ids],
},
)
assert unseeded.status_code == 200
assert seeded.status_code == 200
unseeded_text = unseeded.json()["choices"][0]["text"]
seeded_text = seeded.json()["choices"][0]["text"]
assert not unseeded_text.startswith(" ")
assert seeded_text.startswith(" ")
assert seeded_text.lstrip(" ") == unseeded_text
@pytest.mark.asyncio
async def test_derender_completion_oversized_prompt_token_ids_rejected(client):
"""A prompt_token_ids entry longer than max_model_len returns 400."""
gr1 = await _render_completion(client, "Hello")
ids1 = gr1["token_ids"][:3]
oversized_prompt_ids = [42] * 1_000_000
response = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(ids1, gr1["request_id"]),
],
"prompt_token_ids": [oversized_prompt_ids],
},
)
assert response.status_code == 400
assert "max_model_len" in response.json()["error"]["message"]
@pytest.mark.asyncio
async def test_derender_completion_empty_generate_responses(client):
"""Empty generate_responses list returns 400."""
@@ -20,11 +20,17 @@ Tests are split into two layers:
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import pytest_asyncio
from pydantic import ValidationError
from tokenizers.decoders import DecodeStream
import vllm.renderers.online_derenderer as online_derenderer_module
from tests.entrypoints.scale_out.derender.utils import stream_chat_derender
from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
@@ -32,7 +38,17 @@ from vllm.entrypoints.generate.base.protocol import (
PerRequestMetrics,
SpeculativeDecodingMetrics,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
GenerateResponseChoice,
@@ -41,6 +57,12 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.parser import Parser
from vllm.renderers.online_derenderer import (
OnlineDerenderer,
_decode_params,
_seed_stream_state,
)
from vllm.tokenizers import get_tokenizer
from vllm.utils import random_uuid
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@@ -170,6 +192,7 @@ def _make_stream_chunk(
usage: dict | None = None,
metrics: PerRequestMetrics | None = None,
logprobs: dict | None = None,
prompt_token_ids: list[int] | None = None,
) -> GenerateStreamResponse:
"""Build a GenerateStreamResponse SSE chunk."""
return GenerateStreamResponse(
@@ -184,6 +207,7 @@ def _make_stream_chunk(
],
usage=UsageInfo(**usage) if usage else None,
metrics=metrics,
prompt_token_ids=prompt_token_ids,
)
@@ -228,18 +252,12 @@ def _make_usage_chunk(
@pytest.fixture(scope="module")
def tokenizer():
"""Load the tiny tokenizer used across unit tests."""
from vllm.tokenizers import get_tokenizer
return get_tokenizer(MODEL_NAME)
@pytest.fixture(scope="module")
def derenderer(tokenizer, request):
"""Construct a minimal OnlineDerenderer backed by a stub renderer."""
from unittest.mock import MagicMock
from vllm.renderers.online_derenderer import OnlineDerenderer
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
@@ -273,10 +291,6 @@ def parsed_derenderer(tokenizer):
`renderer._executor` must be a real `ThreadPoolExecutor`.
`loop.run_in_executor` cannot submit work to a `MagicMock`.
"""
from unittest.mock import MagicMock
from vllm.renderers.online_derenderer import OnlineDerenderer
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
renderer._executor = ThreadPoolExecutor(max_workers=2)
@@ -433,6 +447,57 @@ class TestDetokenizeDelta:
# Bounded by a small constant independent of len(token_ids)
assert max_window <= 32
def test_truncated_multibyte_char_is_held_back(self, derenderer, tokenizer):
"""Output ending mid-character emits no U+FFFD: batch == stream == engine."""
emoji_ids = tokenizer.encode("🌈", add_special_tokens=False)
byte_ids = emoji_ids[-4:]
assert len(byte_ids) == 4
output_ids = byte_ids[:-1]
batch, _ = derenderer._detokenize_delta(
tokenizer, output_ids, DerenderStreamState()
)
streamed = self._chunked(derenderer, tokenizer, [[t] for t in output_ids])
engine = _decode_stream_expected(tokenizer, [], output_ids)
assert "�" not in batch
assert batch == streamed == engine
full, _ = derenderer._detokenize_delta(
tokenizer, byte_ids, DerenderStreamState()
)
assert full.endswith("🌈")
@pytest.mark.parametrize(
("skip", "spaces", "preserve", "fast", "expected"),
[
(True, True, False, True, (True, True)),
(True, False, False, True, (True, True)),
(True, False, False, False, (True, False)),
(False, False, False, True, (False, False)),
(False, False, True, True, (False, False)),
(True, False, True, True, (False, False)),
(True, True, True, True, (False, True)),
],
)
def test_decode_params_mirror_engine(
self, monkeypatch, skip, spaces, preserve, fast, expected
):
"""Flags match the engine's fast or slow detokenizer for the tokenizer."""
monkeypatch.setattr(
online_derenderer_module, "uses_fast_detokenizer", lambda _: fast
)
request = CompletionRequest(
model=MODEL_NAME,
prompt="x",
skip_special_tokens=skip,
spaces_between_special_tokens=spaces,
)
assert _decode_params(None, request, preserve_special=preserve) == expected
def test_decode_params_default_without_request(self, tokenizer):
assert _decode_params(tokenizer, None) == (True, True)
def test_n_independent_streams_same_result(self, derenderer, tokenizer):
"""N parallel streams with the same token sequence give the same text."""
token_ids = tokenizer.encode("parallel streams")[:8]
@@ -451,6 +516,242 @@ class TestDetokenizeDelta:
assert results[0] == self._one_shot(tokenizer, token_ids)
# ---------------------------------------------------------------------------
# Prompt seeded detokenization (matches the engine's primed detokenizer)
# ---------------------------------------------------------------------------
def _decode_stream_expected(
tokenizer, prompt_ids: list[int], output_ids: list[int]
) -> str:
"""Engine-equivalent decode via `DecodeStream`, primed by stepping the
prompt (`DecodeStream(ids=...)` needs tokenizers >= 0.22)."""
stream = DecodeStream(skip_special_tokens=True)
for tid in prompt_ids:
stream.step(tokenizer.backend_tokenizer, tid)
return "".join(
s
for tid in output_ids
if (s := stream.step(tokenizer.backend_tokenizer, tid)) is not None
)
@pytest.fixture(scope="module")
def leading_space_ids(tokenizer):
"""(prompt_ids, output_ids, expected) where seeded and unseeded decode
differ by the leading space."""
prompt_ids = tokenizer.encode("[INST] Hi [/INST]", add_special_tokens=False)
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
expected = _decode_stream_expected(tokenizer, prompt_ids, output_ids)
unseeded = tokenizer.decode(output_ids, skip_special_tokens=True)
assert expected.startswith(" ") and not unseeded.startswith(" "), (
"fixture no longer reproduces the leading-space divergence; "
"pick a different prompt/output pair"
)
return prompt_ids, output_ids, expected
class TestSeedStreamState:
"""`_seed_stream_state`: pure helper building the initial decode state."""
def test_seeds_prev_tokens_from_prompt_tail(self, tokenizer, leading_space_ids):
prompt_ids, _, _ = leading_space_ids
state = _seed_stream_state(tokenizer, prompt_ids, skip_special_tokens=True)
assert state.prev_tokens
assert state.read_offset == len(state.prev_tokens)
def test_none_prompt_token_ids_returns_empty_state(self, tokenizer):
assert (
_seed_stream_state(tokenizer, None, skip_special_tokens=True)
== DerenderStreamState()
)
def test_empty_prompt_token_ids_returns_empty_state(self, tokenizer):
assert (
_seed_stream_state(tokenizer, [], skip_special_tokens=True)
== DerenderStreamState()
)
class TestPromptSeededLeadingSpace:
"""Every decode entry point keeps the first token's leading space when
given prompt_token_ids."""
@pytest.mark.asyncio
async def test_chat_batch_seeded_via_explicit_prompt_token_ids(
self, derenderer, leading_space_ids
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
),
prompt_token_ids=prompt_ids,
)
assert choices[0].message.content == expected
@pytest.mark.asyncio
async def test_chat_batch_seeded_via_generate_response_fallback(
self, derenderer, leading_space_ids
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
prompt_token_ids=prompt_ids,
),
)
assert choices[0].message.content == expected
@pytest.mark.asyncio
async def test_chat_batch_explicit_takes_precedence_over_fallback(
self, derenderer, tokenizer, leading_space_ids
):
prompt_ids, output_ids, expected = leading_space_ids
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
prompt_token_ids=wrong_ids,
),
prompt_token_ids=prompt_ids,
)
assert choices[0].message.content == expected
@pytest.mark.asyncio
async def test_chat_batch_without_prompt_context_stays_unseeded(
self, derenderer, leading_space_ids
):
_, output_ids, _ = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
),
)
assert not choices[0].message.content.startswith(" ")
@pytest.mark.asyncio
async def test_completion_batch_seeded(self, derenderer, leading_space_ids):
prompt_ids, output_ids, expected = leading_space_ids
choices, _, _ = await derenderer.derender_completion(
[
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
)
],
prompt_token_ids=[prompt_ids],
)
assert choices[0].text == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("from_generate_chunk", [False, True])
async def test_chat_stream_first_chunk_seeded(
self, derenderer, leading_space_ids, from_generate_chunk
):
prompt_ids, output_ids, expected = leading_space_ids
mid = len(output_ids) // 2
chunk1, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
output_ids[:mid],
prompt_token_ids=prompt_ids if from_generate_chunk else None,
),
prompt_token_ids=None if from_generate_chunk else prompt_ids,
)
# Only needed on the first chunk; the carried state already holds
# the seeded window from here on.
chunk2, _ = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"),
state=state,
)
streamed = (chunk1.choices[0].delta.content or "") + (
chunk2.choices[0].delta.content or ""
)
assert streamed == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("from_generate_chunk", [False, True])
async def test_completion_stream_first_chunk_seeded(
self, derenderer, leading_space_ids, from_generate_chunk
):
prompt_ids, output_ids, expected = leading_space_ids
mid = len(output_ids) // 2
chunk1, state = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
output_ids[:mid],
prompt_token_ids=prompt_ids if from_generate_chunk else None,
),
prompt_token_ids=None if from_generate_chunk else prompt_ids,
)
chunk2, _ = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"),
state=state,
)
assert chunk1.choices[0].text + chunk2.choices[0].text == expected
@pytest.mark.asyncio
async def test_parser_path_ephemeral_state_seeded(
self, parsed_derenderer, monkeypatch, leading_space_ids
):
"""The parser path's per-call detok window is seeded too, so
`parse_delta` sees the same text as coupled serving."""
class _EchoParser(Parser):
def parse_delta(
self,
delta_text,
delta_token_ids,
request,
prompt_token_ids=None,
*,
finished,
):
return DeltaMessage(content=delta_text) if delta_text else None
monkeypatch.setattr(parsed_derenderer, "parser", _EchoParser)
prompt_ids, output_ids, expected = leading_space_ids
chunk, _ = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(output_ids, finish_reason="stop"),
chat_request=_chat_request(),
prompt_token_ids=prompt_ids,
)
assert chunk.choices[0].delta.content == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
"stream_method",
@@ -460,8 +761,6 @@ async def test_streaming_detokenization_runs_off_event_loop(
derenderer, monkeypatch, stream_method
):
"""Streaming detokenization runs on the renderer executor."""
import vllm.renderers.online_derenderer as online_derenderer_module
event_loop_thread_id = threading.get_ident()
executor_thread_id = derenderer.renderer._executor.submit(
threading.get_ident
@@ -561,8 +860,6 @@ class TestDerenderCompletionStream:
@pytest.mark.asyncio
async def test_skip_special_tokens_threaded(self, derenderer, tokenizer):
"""completion_request.skip_special_tokens is honored (not hardcoded True)."""
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
eos = tokenizer.eos_token_id
if eos is None:
pytest.skip("tokenizer has no eos token to exercise special stripping")
@@ -789,8 +1086,6 @@ class TestDerenderChatStream:
def _chat_request(**kwargs):
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
kwargs.setdefault("messages", [{"role": "user", "content": "hi"}])
kwargs.setdefault("model", MODEL_NAME)
return ChatCompletionRequest(**kwargs)
@@ -1072,8 +1367,6 @@ def harmony_encode():
@pytest.fixture(scope="module")
def harmony_tokenizer():
pytest.importorskip("openai_harmony")
from vllm.tokenizers import get_tokenizer
return get_tokenizer(HARMONY_MODEL, trust_remote_code=True)
@@ -1085,10 +1378,7 @@ def harmony_derenderer(harmony_tokenizer):
with a real tokenizer and parser, so the replay path is exercised
against Harmony's actual channel grammar rather than a stub.
"""
from unittest.mock import MagicMock
from vllm.parser.harmony import HarmonyParser
from vllm.renderers.online_derenderer import OnlineDerenderer
renderer = MagicMock()
renderer.get_tokenizer.return_value = harmony_tokenizer
@@ -1224,20 +1514,14 @@ class TestDerenderStreamStateValidation:
"""DerenderStreamState rejects malformed caller supplied offsets/lengths."""
def test_negative_prefix_offset_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(prefix_offset=-1)
def test_negative_read_offset_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(read_offset=-1)
def test_prev_tokens_over_cap_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(prev_tokens=["a"] * 1025)
@@ -1246,14 +1530,10 @@ class TestDerenderStreamStateValidation:
assert len(state.prev_tokens) == 1024
def test_output_chunk_lens_mismatch_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(output_token_ids=[1, 2, 3], output_chunk_lens=[1, 1])
def test_output_chunk_lens_zero_entry_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(output_token_ids=[1, 2], output_chunk_lens=[0, 2])
@@ -1264,14 +1544,40 @@ class TestDerenderStreamStateValidation:
assert state.output_chunk_lens == [1, 2]
_GEN = {"request_id": "t", "choices": []}
_GEN_BAD = {**_GEN, "prompt_token_ids": [-1]}
@pytest.mark.parametrize(
"model_cls,payload",
[
(DerenderChatRequest, {"generate_response": _GEN, "prompt_token_ids": [-1]}),
(DerenderChatRequest, {"generate_response": _GEN_BAD}),
(
DerenderCompletionRequest,
{"generate_responses": [_GEN], "prompt_token_ids": [[-1]]},
),
(
DerenderChatStreamRequest,
{"stream": True, "generate_chunk": _GEN, "prompt_token_ids": [-1]},
),
(
DerenderCompletionStreamRequest,
{"stream": True, "generate_chunk": _GEN_BAD},
),
],
)
def test_negative_prompt_token_ids_rejected(model_cls, payload):
"""Negative prompt ids would reach convert_ids_to_tokens, so they get a
400 at parse time like `GenerateRequest.token_ids` does."""
with pytest.raises(ValidationError, match="prompt_token_ids"):
model_cls.model_validate(payload)
class TestServingDerenderStreamErrorHandling:
"""Malformed stream_state must surface as 400 and not an unhandled 500."""
def _make_serving(self, side_effect: Exception):
from unittest.mock import AsyncMock, MagicMock
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
models = MagicMock()
models.is_base_model.return_value = True
models.model_config = MagicMock()
@@ -1289,10 +1595,6 @@ class TestServingDerenderStreamErrorHandling:
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
async def test_completion_stream_bad_state_returns_400(self, exc):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderCompletionStreamRequest,
)
serving = self._make_serving(exc)
request = DerenderCompletionStreamRequest(
stream=True,
@@ -1307,10 +1609,6 @@ class TestServingDerenderStreamErrorHandling:
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
async def test_chat_stream_bad_state_returns_400(self, exc):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(exc)
request = DerenderChatStreamRequest(
stream=True,
@@ -1329,13 +1627,6 @@ class TestServingDerenderStreamValidation:
before touching the tokenizer)."""
def _make_serving(self, *, parser_configured: bool, max_model_len: int = 100_000):
from unittest.mock import AsyncMock, MagicMock
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
)
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
models = MagicMock()
models.is_base_model.return_value = True
models.model_config = MagicMock()
@@ -1353,10 +1644,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_missing_chat_request_with_parser_rejected(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
@@ -1371,10 +1658,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_missing_chat_request_without_parser_ok(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=False)
request = DerenderChatStreamRequest(
stream=True,
@@ -1390,10 +1673,6 @@ class TestServingDerenderStreamValidation:
the same way it rejects a missing chat_request. Without it,
parse_delta cannot tell whether the prompt left reasoning open and
would misclassify reasoning content as plain content."""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
@@ -1409,10 +1688,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_prompt_token_ids_present_with_parser_ok(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
@@ -1426,10 +1701,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_oversized_output_token_ids_rejected(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=False, max_model_len=4)
request = DerenderChatStreamRequest(
stream=True,
@@ -1451,10 +1722,6 @@ class TestServingDerenderStreamValidation:
parser configured deployment rescans it in full on every chunk
(is_reasoning_end / adjust_initial_state_from_prompt), so it must be
bounded the same way output_token_ids is."""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True, max_model_len=4)
request = DerenderChatStreamRequest(
stream=True,
@@ -1475,10 +1742,6 @@ class TestServingDerenderStreamValidation:
DerenderStreamState is threaded through every choice). The check
could never fire since derender_chat_stream itself rejects anything
above 1 first."""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=False)
two_choices = GenerateStreamResponse(
request_id="t",
@@ -1503,16 +1766,12 @@ class TestServingDerenderStreamValidation:
@pytest.fixture(scope="module")
def server():
from tests.utils import RemoteLaunchRenderServer
with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
import httpx
async with httpx.AsyncClient(
base_url=server.url_for(""), timeout=30.0
) as http_client:
@@ -1776,8 +2035,6 @@ _PARSER_TOOLS = [
@pytest.fixture(scope="module")
def parser_server():
from tests.utils import RemoteLaunchRenderServer
args = [
"--enable-auto-tool-choice",
"--tool-call-parser",
@@ -1791,8 +2048,6 @@ def parser_server():
@pytest_asyncio.fixture
async def parser_client(parser_server):
import httpx
async with httpx.AsyncClient(
base_url=parser_server.url_for(""), timeout=60.0
) as http_client:
@@ -1801,8 +2056,6 @@ async def parser_client(parser_server):
@pytest.fixture(scope="module")
def parser_tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer(PARSER_MODEL)