Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
+13
View File
@@ -9,6 +9,7 @@ import numpy as np
import pytest
from vllm.benchmarks.forward.batch import build_batch
from vllm.benchmarks.forward.main import run_case
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
from vllm.platforms import current_platform
@@ -84,6 +85,18 @@ def test_jitter_kv_lens():
jitter_kv_lens(case, 0, 1000, 0)
@pytest.mark.parametrize("capacity", [0, 2])
def test_bench_forward_rejects_oversized_verify(capacity):
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
runner = SimpleNamespace(num_speculative_steps=capacity)
case = parse_case("verify=2*4@64:3")
with pytest.raises(
ValueError,
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
):
run_case(runner, case, SimpleNamespace())
@pytest.mark.benchmark
# TODO(woosuk): Support ROCm and re-enable this test there.
@pytest.mark.skipif(
+16
View File
@@ -107,6 +107,8 @@ AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
# Static per-tensor: only the aiter quant op is matched by the pass
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
# Blockwise
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
]
@@ -197,6 +199,9 @@ class TestModel(torch.nn.Module):
# Blockwise path
if self.use_aiter_fusion and self.use_aiter_quant_op:
return [rocm_aiter_ops.get_group_quant_op()]
elif self.group_shape.is_per_tensor():
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_tensor_quant_op()]
else:
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_token_quant_op()]
@@ -221,6 +226,17 @@ class TestModel(torch.nn.Module):
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
AiterRMSFp8GroupQuantPattern.FUSED_OP,
]
elif self.group_shape.is_per_tensor():
# Static per-tensor aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSNormStaticQuantPattern,
AiterRMSNormStaticQuantPattern,
)
return [
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
AiterRMSNormStaticQuantPattern.FUSED_OP,
]
else:
# Per-token aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
File diff suppressed because it is too large Load Diff
@@ -49,6 +49,9 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatMessage,
)
from vllm.entrypoints.serve.engine.protocol import PromptTokenUsageInfo, UsageInfo
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
validation_exception_handler,
)
@@ -1034,7 +1037,7 @@ def _make_stream_chunk(
stop_reason: int | str | None = None,
choices: list[ChatCompletionResponseStreamChoice] | None = None,
usage: UsageInfo | None = None,
) -> str:
) -> ChatCompletionStreamResponse:
if choices is None:
choices = [
ChatCompletionResponseStreamChoice(
@@ -1051,7 +1054,7 @@ def _make_stream_chunk(
choices=choices,
usage=usage,
)
return f"data: {chunk.model_dump_json()}"
return chunk
def _tc(*, args, id=None, name=None):
@@ -1062,6 +1065,39 @@ def _tc(*, args, id=None, name=None):
)
@pytest.mark.asyncio
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_use"])
async def test_stream_error_preserves_message_and_closes_block(block_type):
async def source():
if block_type is not None:
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
if block_type == "text":
delta = DeltaMessage(content="Hello")
elif block_type == "thinking":
delta = DeltaMessage(reasoning="Thinking")
else:
delta = DeltaMessage(
tool_calls=[_tc(id="call_1", name="search", args='{"q":')]
)
yield _make_stream_chunk(delta=delta)
yield create_error_response(ValueError("backend failed"))
converter = _make_stream_converter()
events = _parse_sse_events(
[event async for event in converter.message_stream_converter(source())]
)
assert events[-1] == (
"error",
{
"type": "error",
"error": {"type": "internal_error", "message": "backend failed"},
},
)
assert not any(event_type == "message_stop" for event_type, _ in events)
if block_type is not None:
assert events[-2][0] == "content_block_stop"
class TestMessageStreamConverterToolUseContentBuffering:
"""Regression test for tool_use arguments being silently dropped.
@@ -1074,7 +1110,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
async def test_tool_use_args_not_dropped_when_content_in_same_chunk(
self,
):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=10, total_tokens=10),
@@ -1111,11 +1147,10 @@ class TestMessageStreamConverterToolUseContentBuffering:
completion_tokens=20,
),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1156,7 +1191,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
stream jumps straight from finish_reason to [DONE], skipping the
empty-choices usage chunk."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=10, total_tokens=10),
@@ -1178,11 +1213,10 @@ class TestMessageStreamConverterToolUseContentBuffering:
finish_reason="tool_calls",
)
# No empty-choices usage chunk — go straight to [DONE].
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1216,7 +1250,7 @@ class TestMessageStartIncludesTypeAndRole:
@pytest.mark.asyncio
async def test_message_start_contains_message_type_and_role(self):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(content="Hello"),
usage=UsageInfo(
@@ -1226,11 +1260,10 @@ class TestMessageStartIncludesTypeAndRole:
),
)
yield _make_stream_chunk(finish_reason="stop")
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1260,7 +1293,7 @@ class TestStreamingCacheUsageSemantics:
message_start omits cache fields. The final chunk carries
prompt_tokens_details, so message_delta carries resolved values."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant", content="hi"),
usage=UsageInfo(prompt_tokens=100, total_tokens=100),
@@ -1277,11 +1310,10 @@ class TestStreamingCacheUsageSemantics:
),
),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1305,7 +1337,7 @@ class TestStreamingCacheUsageSemantics:
"""When the final chunk reports cached_tokens=0, message_delta carries
cache fields = 0 (cache miss); message_start still omits them."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=50, total_tokens=50),
@@ -1322,11 +1354,10 @@ class TestStreamingCacheUsageSemantics:
),
),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1346,7 +1377,7 @@ class TestStreamingCacheUsageSemantics:
"""If --enable-prompt-tokens-details is off, no chunk carries cache
info; both message_start and message_delta omit cache fields."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(role="assistant"),
usage=UsageInfo(prompt_tokens=30, total_tokens=30),
@@ -1356,11 +1387,10 @@ class TestStreamingCacheUsageSemantics:
choices=[],
usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1628,7 +1658,7 @@ class TestStopSequenceReason:
@pytest.mark.asyncio
async def test_streaming_stop_string_maps_to_stop_sequence(self):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(
delta=DeltaMessage(content="hi"),
usage=UsageInfo(prompt_tokens=5, total_tokens=5, completion_tokens=0),
@@ -1638,11 +1668,10 @@ class TestStopSequenceReason:
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1654,7 +1683,7 @@ class TestStopSequenceReason:
async def test_streaming_no_stop_string_emits_explicit_null_stop_sequence(self):
"""exclude_unset=True drops stop_sequence unless it is set explicitly."""
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
yield _make_stream_chunk(delta=DeltaMessage(content="hi"))
yield _make_stream_chunk(finish_reason="stop")
@@ -1662,11 +1691,10 @@ class TestStopSequenceReason:
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
output = []
async for event in converter.message_stream_converter(sse_input()):
async for event in converter.message_stream_converter(chunk_input()):
output.append(event)
events = _parse_sse_events(output)
@@ -1724,7 +1752,7 @@ class TestToolUseStopReason:
],
)
async def test_streaming(self, args, finish_reason, stop_reason, expected):
async def sse_input():
async def chunk_input():
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
yield _make_stream_chunk(
delta=DeltaMessage(
@@ -1738,11 +1766,10 @@ class TestToolUseStopReason:
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
events = _parse_sse_events(
[e async for e in converter.message_stream_converter(sse_input())]
[e async for e in converter.message_stream_converter(chunk_input())]
)
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
@@ -5,7 +5,7 @@
The stream-translation entry point is
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
async iterable of OpenAI SSE chunks into Cohere's
async iterable of OpenAI chunk objects into Cohere's
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
event stream.
@@ -15,7 +15,7 @@ state machine.
"""
import json
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Sequence
from typing import Any
import pytest
@@ -26,10 +26,7 @@ from vllm.entrypoints.cohere.cohere_chat_message import (
from vllm.entrypoints.cohere.cohere_chat_message import (
CitationSource,
)
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
MessageStartEvent,
)
from vllm.entrypoints.cohere.protocol import MessageStartEvent
from vllm.entrypoints.cohere.serving import (
_DONE_FRAME,
CohereServingChatV2,
@@ -41,6 +38,7 @@ from vllm.entrypoints.cohere.serving import (
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
# ----------------------------------------------------------------------
# Helpers
@@ -85,9 +83,8 @@ def _make_chunk(
usage: dict[str, Any] | None = None,
omit_choices: bool = False,
citations: list[Any] | None = None,
) -> str:
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
consumes."""
) -> ChatCompletionStreamResponse:
"""Build the raw chat chunk the production converter consumes."""
delta: dict[str, Any] = {}
if role is not None:
delta["role"] = role
@@ -119,27 +116,23 @@ def _make_chunk(
payload["choices"] = []
if usage is not None:
payload["usage"] = usage
# Use ChatCompletionStreamResponse to normalize the payload shape.
chunk = ChatCompletionStreamResponse.model_validate(payload)
return f"data: {chunk.model_dump_json(exclude_none=False)}\n\n"
return ChatCompletionStreamResponse.model_validate(payload)
def _make_done() -> str:
return "data: [DONE]\n\n"
async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
async def _stream_from(
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
for item in items:
yield item
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
async def _drain(
serving: CohereServingChatV2,
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
) -> list[str]:
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
the emitted SSE frames."""
request = CohereChatV2Request(
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
)
gen = serving._chat_completion_stream_to_v2(_stream_from(items), request)
gen = serving._chat_completion_stream_to_v2(_stream_from(items))
return [frame async for frame in gen]
@@ -793,15 +786,14 @@ class TestChatCompletionStreamToV2:
assert "content-end" not in types
@pytest.mark.asyncio
async def test_done_marker_in_middle_closes_open_block(self):
# Some upstreams send [DONE] without a trailing usage-only chunk.
async def test_stream_exhaustion_closes_open_block_without_usage(self):
# The raw stream may finish without a trailing usage-only chunk.
# The translator must still emit message-end before [DONE] so
# Cohere clients don't hang.
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="Hi"),
_make_done(),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
@@ -815,72 +807,65 @@ class TestChatCompletionStreamToV2:
assert frames[-1] == _DONE_FRAME
@pytest.mark.asyncio
async def test_skips_empty_and_non_data_lines(self):
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_call"])
async def test_upstream_error_preserves_message(self, block_type):
serving = _serving()
items = [
"\n",
"event: ping\n\n",
_make_chunk(role="assistant"),
"data: \n\n", # empty data
_make_chunk(content="Hi"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
items: list[ChatCompletionStreamResponse | ErrorResponse] = []
if block_type is not None:
items.append(_make_chunk(role="assistant"))
if block_type == "text":
items.append(_make_chunk(content="Hi"))
elif block_type == "thinking":
items.append(_make_chunk(reasoning="Thinking"))
else:
items.append(
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "call_1",
"function": {"name": "search", "arguments": '{"q":'},
}
]
)
)
items.append(serving.create_error_response(ValueError("backend failed")))
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
# Should still produce a complete lifecycle.
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types[0] == "message-start"
assert types[-1] == "message-end"
events = [_parse_event(frame) for frame in frames[:-1]]
error_ends = [event for event in events if event["type"] == "message-end"]
assert len(error_ends) == 1
assert error_ends[0]["delta"] == {
"finish_reason": "ERROR",
"error": "backend failed",
}
assert events[-1] == error_ends[0]
if block_type is not None:
assert events[-2]["type"] == (
"tool-call-end" if block_type == "tool_call" else "content-end"
)
@pytest.mark.asyncio
async def test_exception_in_chunk_parsing_emits_error_message_end(self):
# An invalid JSON payload after the first chunk triggers the
# error path: a synthetic message-end with finish_reason=ERROR
# followed by [DONE].
serving = _serving()
items = [
_make_chunk(role="assistant"),
"data: {not valid json}\n\n",
]
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
# Find the error-shaped message-end.
error_end = _parse_event(frames[-2])
assert error_end["type"] == "message-end"
assert error_end["delta"]["finish_reason"] == "ERROR"
assert "error" in error_end["delta"]
@pytest.mark.asyncio
async def test_error_message_end_is_sanitized(self):
@pytest.mark.parametrize("started", [False, True])
async def test_error_message_end_is_sanitized(self, started):
# The client-visible ``error`` field on the terminal message-end
# must be routed through ``sanitize_message`` so tracebacks,
# host filesystem paths, and Python object memory addresses
# can't leak from a mid-stream exception into the SSE payload.
serving = _serving()
async def _raises_mid_stream() -> AsyncGenerator[str, None]:
yield _make_chunk(role="assistant")
async def _raises_mid_stream() -> AsyncGenerator[
ChatCompletionStreamResponse | ErrorResponse, None
]:
if started:
yield _make_chunk(role="assistant")
raise RuntimeError(
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
)
request = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
stream=True,
)
frames = [
f
async for f in serving._chat_completion_stream_to_v2(
_raises_mid_stream(), request
)
f async for f in serving._chat_completion_stream_to_v2(_raises_mid_stream())
]
assert frames[-1] == _DONE_FRAME
error_end = _parse_event(frames[-2])
@@ -28,6 +28,7 @@ from vllm.entrypoints.generate.base.serving import build_per_request_timing_metr
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.chat_completion.serving import (
OpenAIServingChat,
@@ -43,7 +44,7 @@ from vllm.entrypoints.openai.models.serving import (
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason
from vllm.exceptions import QueueOverflowError, VLLMValidationError
from vllm.exceptions import GenerationError, QueueOverflowError, VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import PlaceholderRange
@@ -716,6 +717,149 @@ async def _collect_metrics_stream_chunks(
return chunks
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", [False, True])
async def test_streaming_chunks_keep_usage_and_metadata(raw):
serving = _build_minimal_metrics_serving_chat(False)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test"}],
stream=True,
stream_options={"include_usage": True},
)
metadata = RequestResponseMetadata(request_id="chatcmpl-test")
method = (
serving.chat_completion_stream_generator_raw
if raw
else serving.chat_completion_stream_generator
)
chunks = [
chunk
async for chunk in method(
request,
_single_request_output(_make_metrics_request_output()),
"chatcmpl-test",
"test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=metadata,
)
]
if raw:
assert all(isinstance(chunk, ChatCompletionStreamResponse) for chunk in chunks)
else:
assert chunks[-1] == "data: [DONE]\n\n"
first = json.loads(chunks[0].removeprefix("data: "))
assert first["choices"][0]["logprobs"] is None
usage = json.loads(chunks[-2].removeprefix("data: "))
assert "system_fingerprint" not in usage
assert "prompt_tokens_details" not in usage["usage"]
chunks = [
ChatCompletionStreamResponse.model_validate_json(
chunk.removeprefix("data: ")
)
for chunk in chunks[:-1]
]
assert len(chunks) == 3
assert chunks[0].choices[0].delta.role == "assistant"
assert chunks[1].choices[0].delta.content == "Hello"
assert chunks[1].choices[0].finish_reason == "stop"
assert chunks[2].choices == []
assert chunks[2].usage.prompt_tokens == 3
assert chunks[2].usage.completion_tokens == 2
assert metadata.final_usage_info.total_tokens == 5
@pytest.mark.asyncio
async def test_streaming_serialization_failure_emits_error_and_done():
serving = _build_minimal_metrics_serving_chat(False)
serving.parser_cls = HarmonyParser
parser = MagicMock()
parser.parse_delta.return_value = DeltaMessage(
content="Hello", extra_object=object()
)
serving._make_parser = MagicMock(return_value=parser)
request = ChatCompletionRequest(
model="test-model", messages=[{"role": "user", "content": "Test"}], stream=True
)
frames = [
frame
async for frame in serving.chat_completion_stream_generator(
request,
_single_request_output(_make_metrics_request_output()),
"chatcmpl-test",
"test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test"),
)
]
assert frames[-1] == "data: [DONE]\n\n"
error = ErrorResponse.model_validate_json(frames[-2].removeprefix("data: "))
assert "serialize" in error.error.message
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", [False, True])
@pytest.mark.parametrize(
"failure", ["parser", "before_first", "after_content", "generation"]
)
async def test_streaming_errors_preserve_envelope(raw, failure):
serving = _build_minimal_metrics_serving_chat(False)
error = (
GenerationError("backend failed")
if failure == "generation"
else ValueError("backend failed")
)
if failure == "parser":
serving.parser_cls = HarmonyParser
serving._make_parser = MagicMock(side_effect=error)
async def results():
if failure == "after_content":
output = _make_metrics_request_output()
output.outputs[0].finish_reason = None
output.finished = False
yield output
raise error
request = ChatCompletionRequest(
model="test-model", messages=[{"role": "user", "content": "Test"}], stream=True
)
method = (
serving.chat_completion_stream_generator_raw
if raw
else serving.chat_completion_stream_generator
)
chunks = [
chunk
async for chunk in method(
request,
results(),
"chatcmpl-test",
"test-model",
conversation=[],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test"),
)
]
if raw:
assert all(
isinstance(chunk, (ChatCompletionStreamResponse, ErrorResponse))
for chunk in chunks
)
assert isinstance(chunks[-1], ErrorResponse)
error_response = chunks[-1]
else:
assert chunks[-1] == "data: [DONE]\n\n"
assert chunks.count("data: [DONE]\n\n") == 1
error_response = ErrorResponse.model_validate_json(
chunks[-2].removeprefix("data: ")
)
assert error_response.error.message == "backend failed"
assert error_response.error.code == (500 if failure == "generation" else 400)
def test_build_per_request_timing_metrics_valid_timestamps():
metrics = build_per_request_timing_metrics(
_PER_REQUEST_STATS, num_generation_tokens=10
@@ -1188,6 +1332,15 @@ async def test_streaming_reasoning_usage_lifecycle_edges(
await iterator.aclose()
error_chunks = [c for c in chunks if isinstance(c, dict) and "error" in c]
if name == "empty-finish":
# A choice that finishes without output still gets its last chunk, unparsed.
assert any(
choice.get("finish_reason") == "stop"
for item in chunks
if isinstance(item, dict)
for choice in item.get("choices", [])
)
parsers[0].parse_delta.assert_not_called()
call_count = sum(parser.count_reasoning_tokens.call_count for parser in parsers)
if expected_tokens is not None:
assert outcome == "exhausted" and chunks[-1] == "[DONE]" and not error_chunks
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -17,7 +18,7 @@ from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.outputs import CompletionOutput, RequestOutput, WeightVersionSpan
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -310,6 +311,76 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
assert response.metrics.speculative_decoding.num_spec_steps == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("opt_in", [False, True])
async def test_completion_weight_versions(stream: bool, opt_in: bool):
"""Each choice gets its spans on its final output, even with no tokens."""
expected = [{"version": "v1", "start": 0, "end": 2}]
spans = [WeightVersionSpan(**span) for span in expected]
def output(index, token_ids, finish_reason=None, weight_versions=None):
return CompletionOutput(
index=index,
text="x" * len(token_ids),
token_ids=token_ids,
cumulative_logprob=None,
logprobs=None,
finish_reason=finish_reason,
weight_versions=weight_versions,
)
# Choice 1 is aborted before its first token.
final = [output(1, [], "abort", [])]
if stream:
steps = [[output(0, [100])], [output(0, [101], "length", spans), *final]]
else:
steps = [[output(0, [100, 101], "length", spans), *final]]
async def mock_generate(*args, **kwargs):
for i, outputs in enumerate(steps):
yield RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=outputs,
finished=i == len(steps) - 1,
)
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
max_tokens=10,
n=2,
stream=stream,
return_weight_versions=opt_in,
)
response = await _build_serving_completion(mock_engine).create_completion(request)
if stream:
finals = {}
async for line in response:
payload = line.removeprefix("data: ").strip()
if payload == "[DONE]":
continue
for choice in json.loads(payload)["choices"]:
if choice.get("finish_reason"):
finals[choice["index"]] = choice
else:
assert choice.get("weight_versions") is None
got = [finals[i].get("weight_versions") for i in (0, 1)]
else:
got = [choice["weight_versions"] for choice in response.model_dump()["choices"]]
assert got == ([expected, []] if opt_in else [None, None])
def _build_online_renderer_for_completion(
model_config: MockModelConfig,
renderer: Any | None = None,
@@ -10,7 +10,7 @@ import torch
from tokenizers import Tokenizer, models, pre_tokenizers, processors
from transformers import TokenizersBackend
from vllm import TokensPrompt
from vllm import PoolingParams, TokensPrompt
from vllm.entrypoints.pooling.scoring.io_processor import (
BiEncoderIOProcessor,
CrossEncoderIOProcessor,
@@ -18,7 +18,9 @@ from vllm.entrypoints.pooling.scoring.io_processor import (
LateInteractionIOProcessor,
_apply_post_tokenization_to_token_type_ids,
)
from vllm.entrypoints.pooling.scoring.typing import ScoringData
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
from vllm.renderers import TokenizeParams
@@ -81,6 +83,49 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
assert nonbinding["prompt_token_ids"] == expected
@pytest.mark.parametrize(
"default_template,request_template,expected",
[
("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"),
(
"{{ messages[0].content }} {{ messages[1].content }}",
"{{ messages[1].content }} {{ messages[0].content }}",
"bc a",
),
(None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"),
(None, None, "abc"),
],
)
def test_offline_scoring_uses_default_template_unless_overridden(
llm_reranker_processor, default_template, request_template, expected
):
processor = llm_reranker_processor
processor.chat_template = default_template
processor.renderer = SimpleNamespace(
default_cmpl_tok_params=TokenizeParams(max_total_tokens=32)
)
ctx = OfflineScoringInputsContext(
pooling_task="classify",
tokenization_kwargs=None,
lora_request=None,
priorities=None,
scoring_data=ScoringData(data_1=["a"], data_2=["bc"]),
chat_template=request_template,
pooling_params=PoolingParams(),
)
factory, _ = processor.get_request_factory_offline(ctx)
(request,) = factory()
text, prompt = processor.get_score_prompt(
request["data_1"],
request["data_2"],
request["tok_params"].get_encode_kwargs(),
chat_template=request["chat_template"],
)
assert text == expected
assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected)
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
"""The cross-encoder segment boundary must survive truncate + pad.
@@ -10,6 +10,7 @@ import pytest
import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateTokensResponse,
@@ -1133,6 +1134,50 @@ async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer):
assert "<think>" not in msg["content"]
@pytest.mark.asyncio
@pytest.mark.parametrize("include_reasoning", [True, False])
async def test_e2e_parsed_logprobs_follow_include_reasoning(
parser_client, parser_tokenizer, include_reasoning
):
"""With a parser configured, batch derender returns logprobs only when
reasoning is shown. Hidden reasoning drops them, as the coupled chat
endpoint does, because decoded logprob tokens would reveal it."""
messages = [{"role": "user", "content": "What is 2+3?"}]
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
output_ids = _require_markers_survive(
parser_tokenizer, "<think>Two plus three.</think>Five.", "</think>"
)
generate_response = _e2e_generate_response(output_ids)
generate_response["choices"][0]["logprobs"] = {
"content": [
{"token_id": tid, "logprob": -0.5, "rank": 1, "top_logprobs": []}
for tid in output_ids
]
}
resp = await parser_client.post(
"/v1/chat/completions/derender",
json={
"model": PARSER_MODEL,
"generate_response": generate_response,
"prompt_tokens": len(gen_req["token_ids"]),
"chat_request": {
"model": PARSER_MODEL,
"messages": messages,
"include_reasoning": include_reasoning,
},
},
)
assert resp.status_code == 200, resp.text
choice = resp.json()["choices"][0]
assert choice["message"]["content"] == "Five."
if include_reasoning:
assert choice["message"]["reasoning"]
assert len(choice["logprobs"]["content"]) == len(output_ids)
else:
assert choice["message"]["reasoning"] is None
assert choice["logprobs"] is None
@pytest.mark.asyncio
async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer):
"""<tool_call> extracted into tool_calls field."""
@@ -1446,19 +1491,12 @@ _HERMES_TOOL_CALL_TEXT = (
)
@pytest.mark.parametrize(
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
[
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
("required", "Hello there", "stop", "stop"),
],
)
def test_derender_chat_finish_reason_matches_chat_completion(
tool_choice, text, engine_finish_reason, expected_finish_reason
def _derender_hermes_chat(
text: str,
tool_choice: str = "auto",
engine_finish_reason: str = "stop",
parser_cls: type | None = None,
):
"""/derender applies the same finish_reason rule as the coupled chat
path: ``tool_calls`` only for a real call at a natural stop."""
tokenizer = get_tokenizer(MODEL_NAME)
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
@@ -1474,6 +1512,8 @@ def test_derender_chat_finish_reason_matches_chat_completion(
enable_auto_tools=True,
tool_parser="hermes",
)
if parser_cls is not None:
derenderer.parser = parser_cls
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Weather in Paris?"}],
@@ -1488,6 +1528,54 @@ def test_derender_chat_finish_reason_matches_chat_completion(
)
(choice,) = derenderer._derender_chat(response, request)
return choice
@pytest.mark.parametrize(
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
[
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
("required", "Hello there", "stop", "stop"),
],
)
def test_derender_chat_finish_reason_matches_chat_completion(
tool_choice, text, engine_finish_reason, expected_finish_reason
):
"""/derender applies the same finish_reason rule as the coupled chat
path: ``tool_calls`` only for a real call at a natural stop."""
choice = _derender_hermes_chat(text, tool_choice, engine_finish_reason)
assert choice.finish_reason == expected_finish_reason
assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
def test_derender_chat_tool_call_id_matches_chat_completion():
"""Batch /derender uses the coupled chat's default tool call ID format."""
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT)
(tool_call,) = choice.message.tool_calls
assert tool_call.id.startswith("chatcmpl-tool-")
def test_derender_chat_keeps_parser_tool_call_id():
"""Batch /derender keeps the parser's tool call ID (Kimi K2 style
``functions.<name>:<idx>``), since chat templates reference it in
multi-turn history."""
class _KimiIdParser:
def __init__(self, *args, **kwargs):
pass
def parse(self, model_output, request, **kwargs):
call = FunctionCall(
id="functions.get_weather:1",
name="get_weather",
arguments='{"city": "Paris"}',
)
return None, None, [call]
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT, parser_cls=_KimiIdParser)
(tool_call,) = choice.message.tool_calls
assert tool_call.id == "functions.get_weather:1"
@@ -142,12 +142,28 @@ def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
]
def _tool_id_shapes(response_choice: dict) -> list[str | None]:
return [
_tool_call_id_shape(tc["id"])
for tc in (response_choice["message"].get("tool_calls") or [])
]
def _engine_finish_reason(coupled_finish_reason: str) -> str:
"""Undo the chat level `"tool_calls"` rewrite of `"stop"` so derender
gets what the engine reported and has to reapply the rewrite itself."""
if coupled_finish_reason == "tool_calls":
return "stop"
return coupled_finish_reason
def _assert_parity(coupled: dict, disagg: dict) -> None:
"""Both paths saw the same tokens, so they must agree unconditionally."""
c, d = coupled["choices"][0], disagg["choices"][0]
assert d["message"]["content"] == c["message"]["content"]
assert d["message"].get("reasoning") == c["message"].get("reasoning")
assert _tool_sig(d) == _tool_sig(c)
assert _tool_id_shapes(d) == _tool_id_shapes(c)
assert d["finish_reason"] == c["finish_reason"]
assert d["token_ids"] == c["token_ids"]
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
@@ -166,7 +182,7 @@ async def _run_parity_case(
client,
ch["token_ids"],
coupled["usage"]["prompt_tokens"],
ch["finish_reason"],
_engine_finish_reason(ch["finish_reason"]),
chat_request,
)
return coupled, disagg
@@ -316,13 +332,18 @@ async def test_parity_logprobs(client):
async def _coupled_stream(
client: httpx.AsyncClient, messages: list[dict], **extra
) -> tuple[list[dict], list[int]]:
client: httpx.AsyncClient,
messages: list[dict],
require_token_ids: bool = True,
**extra,
) -> tuple[list[dict], list[int] | None]:
"""Stream `/v1/chat/completions` and return the choices that carry
`token_ids` along with the prompt token IDs.
The leading role only chunk has no `token_ids` and no counterpart on
`/inference/v1/generate`, so it is dropped.
`/inference/v1/generate`, so it is dropped. With `require_token_ids`
off every choice is kept and `prompt_token_ids` may be `None`, for
requests where the coupled server hides token ids (hidden reasoning).
"""
choices: list[dict] = []
prompt_token_ids: list[int] | None = None
@@ -351,9 +372,12 @@ async def _coupled_stream(
if chunk.get("prompt_token_ids") is not None:
prompt_token_ids = chunk["prompt_token_ids"]
choices.extend(
ch for ch in chunk["choices"] if ch.get("token_ids") is not None
ch
for ch in chunk["choices"]
if not require_token_ids or ch.get("token_ids") is not None
)
assert prompt_token_ids is not None
if require_token_ids:
assert prompt_token_ids is not None
return choices, prompt_token_ids
@@ -390,7 +414,7 @@ async def _run_stream_parity_case(
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
finish_reason=_engine_finish_reason(finish_reason),
)
coupled = assemble_stream(coupled_choices)
for message in (coupled, disagg):
@@ -473,6 +497,151 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
pytest.skip("Model did not emit both a <think> block and a tool call")
# ---------------------------------------------------------------------------
# Streaming logprobs on the parser path
# ---------------------------------------------------------------------------
def _logprob_tokens(choice: dict) -> list[tuple[str, list[int] | None]] | None:
lp = choice.get("logprobs")
if lp is None or lp.get("content") is None:
return None
return [(e["token"], e.get("bytes")) for e in lp["content"]]
def _sampled_generate_logprobs(choice: dict) -> dict | None:
"""`GenerateLogProbs` for one coupled stream chunk from its `token_ids`
and per-token logprob values, without `top_logprobs`.
The coupled stream's `token_ids` are the integer ids a GPU-less worker
would have sent, so one stream is both the derender input and the string
reference. A second greedy run is not needed, and greedy decoding is not
bit-reproducible across calls on every platform anyway.
"""
lp = choice.get("logprobs")
if lp is None or lp.get("content") is None:
return None
content = lp["content"]
assert len(content) == len(choice["token_ids"]), choice
return {
"content": [
{"token_id": tid, "logprob": entry["logprob"], "top_logprobs": []}
for tid, entry in zip(choice["token_ids"], content)
]
}
async def _run_stream_logprob_parity_case(
client: httpx.AsyncClient, messages: list[dict], **extra
) -> tuple[dict, list[dict], list[dict]]:
"""Stream with logprobs through the coupled server, replay the same
chunks through the streaming derender endpoint and compare per chunk.
Returns the assembled coupled message, the coupled choices and the
derender choices.
"""
extra = {"logprobs": True, **extra}
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
finish_reason = coupled_choices[-1]["finish_reason"]
derender_choices: list[dict] = []
disagg = await stream_chat_derender(
client,
[tid for ch in coupled_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in coupled_choices],
{"model": MODEL, "messages": messages, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
logprobs=[_sampled_generate_logprobs(ch) for ch in coupled_choices],
raw_choices=derender_choices,
)
coupled = assemble_stream(coupled_choices)
for message in (coupled, disagg):
for tc in message["tool_calls"]:
tc["id"] = _tool_call_id_shape(tc["id"])
assert disagg == coupled
assert len(derender_choices) == len(coupled_choices)
for ref, der in zip(coupled_choices, derender_choices):
assert _logprob_tokens(der) == _logprob_tokens(ref), (ref, der)
return coupled, coupled_choices, derender_choices
@pytest.mark.asyncio
async def test_stream_parity_logprobs_reasoning(client):
"""Per-chunk logprobs on the parser path match the coupled stream across
the reasoning -> content transition."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
client, messages, include_reasoning=True, max_tokens=256
)
if not coupled["reasoning"]:
pytest.skip("Model did not emit a <think> block")
assert any(
ch["delta"].get("reasoning") and ch.get("logprobs") for ch in derender_choices
)
@pytest.mark.asyncio
async def test_stream_parity_logprobs_tool_call(client):
"""Chunks the parser swallows (tool call markup) still carry their
logprobs, on both sides, so no token's logprobs are lost."""
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
client,
messages,
tools=TOOLS,
tool_choice=FORCE_WEATHER_TOOL,
include_reasoning=True,
max_tokens=512,
)
if not coupled["tool_calls"]:
pytest.skip("Model did not emit a tool call")
swallowed = [
ch
for ch in derender_choices
if ch.get("logprobs")
and not ch["delta"].get("content")
and not ch["delta"].get("reasoning")
and not ch["delta"].get("tool_calls")
]
assert swallowed, "expected at least one buffered-markup chunk with logprobs"
@pytest.mark.asyncio
async def test_stream_parity_logprobs_hidden_reasoning(client):
"""With `include_reasoning=False` and a parser configured, neither side
returns logprobs on any chunk; derender still emits the content."""
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
extra = {"logprobs": True, "max_tokens": 256}
# Token ids come from a shown-reasoning run: the coupled server hides
# them along with the logprobs when reasoning is hidden.
shown_choices, prompt_token_ids = await _coupled_stream(
client, messages, include_reasoning=True, **extra
)
hidden_choices, _ = await _coupled_stream(
client, messages, require_token_ids=False, include_reasoning=False, **extra
)
assert all(ch.get("logprobs") is None for ch in hidden_choices)
assert assemble_stream(hidden_choices)["reasoning"] is None
finish_reason = shown_choices[-1]["finish_reason"]
derender_choices: list[dict] = []
disagg = await stream_chat_derender(
client,
[tid for ch in shown_choices for tid in ch["token_ids"]],
[len(ch["token_ids"]) for ch in shown_choices],
{"model": MODEL, "messages": messages, "include_reasoning": False, **extra},
len(prompt_token_ids),
prompt_token_ids,
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
logprobs=[_sampled_generate_logprobs(ch) for ch in shown_choices],
raw_choices=derender_choices,
)
assert all(ch.get("logprobs") is None for ch in derender_choices)
shown = assemble_stream(shown_choices)
assert disagg["reasoning"] is None
assert disagg["content"] == shown["content"]
# ---------------------------------------------------------------------------
# Inline text level: generate(output_mode="text") == derender(generate(...))
# ---------------------------------------------------------------------------
@@ -1427,6 +1427,159 @@ class TestDerenderChatStreamParsed:
assert delta.reasoning is None
assert delta.content == "c"
@pytest.mark.asyncio
async def test_logprobs_attached_to_each_parsed_delta(
self, parsed_derenderer, tokenizer
):
"""A chunk's logprobs ride on whatever delta the parser emits for it
(reasoning, content or tool call), with token ids decoded, as on
`/v1/chat/completions`."""
chunks = [
[_FakeParser.REASON],
[_FakeParser.CONTENT],
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
]
state = None
seen = []
for ids in chunks:
chunk, state = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
ids, logprobs=_generate_logprobs(ids)
),
state=state,
chat_request=_chat_request(),
prompt_token_ids=[1, 2],
)
choice = chunk.choices[0]
assert choice.logprobs is not None and choice.logprobs.content is not None
assert [e.token for e in choice.logprobs.content] == (
convert_ids_list_to_tokens(tokenizer, ids)
)
seen.append(choice.delta)
assert seen[0].reasoning == "r"
assert seen[1].content == "c"
assert (
seen[2].tool_calls and seen[2].tool_calls[0].function.name == "get_weather"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_kwargs", "expected_candidates"),
[({"logprobs": True, "top_logprobs": 2}, 2), ({}, 4)],
)
async def test_top_logprobs_follow_the_chat_cut(
self, parsed_derenderer, request_kwargs, expected_candidates
):
"""The parsed path trims generate's candidate list to the request's
`top_logprobs` like the plain path does, and keeps every candidate
when the request does not ask for logprobs."""
ids = [_FakeParser.CONTENT]
logprobs = _generate_logprobs(ids)
logprobs["content"][0]["top_logprobs"] = [
{"token_id": tid, "logprob": -float(rank)}
for rank, tid in enumerate([ids[0], 11, 12, 13])
]
chunk, _ = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(ids, logprobs=logprobs),
chat_request=_chat_request(**request_kwargs),
prompt_token_ids=[1, 2],
)
content = chunk.choices[0].logprobs.content
assert len(content) == 1
assert len(content[0].top_logprobs) == expected_candidates
@pytest.mark.asyncio
async def test_logprobs_kept_when_parser_swallows_chunk(
self, parsed_derenderer, tokenizer
):
"""A chunk the parser emits nothing for (buffered markup) still
carries its logprobs on an empty delta, so no token's logprobs are
lost. Mirrors the coupled endpoint, which emits a chunk whenever
logprobs are present even if the delta is empty."""
swallowed = [_FakeParser.CONTENT + 1] # unknown to _FakeParser -> None
chunk, _ = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
swallowed, logprobs=_generate_logprobs(swallowed)
),
chat_request=_chat_request(),
prompt_token_ids=[1, 2],
)
choice = chunk.choices[0]
assert choice.delta.content is None and not choice.delta.tool_calls
assert [e.token for e in choice.logprobs.content] == (
convert_ids_list_to_tokens(tokenizer, swallowed)
)
@pytest.mark.asyncio
async def test_include_reasoning_false_drops_logprobs_on_every_chunk(
self, parsed_derenderer
):
"""Hidden reasoning suppresses logprobs on reasoning *and* content
chunks (decoded logprob tokens would leak the reasoning), the same
rule `/v1/chat/completions` applies with a parser configured."""
state = None
for ids in ([_FakeParser.REASON], [_FakeParser.CONTENT]):
chunk, state = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
ids, logprobs=_generate_logprobs(ids)
),
state=state,
chat_request=_chat_request(include_reasoning=False),
prompt_token_ids=[1, 2],
)
assert chunk.choices[0].logprobs is None
assert chunk.choices[0].delta.content == "c"
@pytest.mark.asyncio
async def test_logprobs_multibyte_across_chunks(self, parsed_derenderer, tokenizer):
"""Byte-fallback correction crosses chunk boundaries on the parser
path too, via the carried `logprob_context_token_ids`, which starts
empty like the engine's and never includes the prompt."""
token_ids = tokenizer.encode("👍", add_special_tokens=False)
if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]):
pytest.skip("Tokenizer does not byte-split this character")
state = None
last_chunk = None
for tid in token_ids:
last_chunk, state = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
[tid], logprobs=_generate_logprobs([tid])
),
state=state,
chat_request=_chat_request(),
prompt_token_ids=[1, 2, 3, 4, 5],
)
assert last_chunk is not None and state is not None
final_entry = last_chunk.choices[0].logprobs.content[0]
assert not final_entry.token.endswith("�"), final_entry.token
assert state.logprob_context_token_ids == token_ids[-4:]
@pytest.mark.asyncio
async def test_logprob_context_advances_without_logprobs(self, parsed_derenderer):
"""The context tail tracks sampled ids whether or not the chunk
carries logprobs, so a later chunk that does can still be corrected."""
ids = [_FakeParser.REASON, _FakeParser.CONTENT, _FakeParser.CONTENT]
_, state = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(ids),
chat_request=_chat_request(),
prompt_token_ids=[1, 2],
)
assert state.logprob_context_token_ids == ids
_, state = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk([], finish_reason="stop"),
state=state,
chat_request=_chat_request(),
prompt_token_ids=[1, 2],
)
assert state.logprob_context_token_ids == ids
# ---------------------------------------------------------------------------
# Harmony / GPT-OSS replay — unit, no server
@@ -52,6 +52,8 @@ async def stream_chat_derender(
prompt_token_ids: list[int],
finish_reason: str = "stop",
on_chunk: Callable[[list[dict]], None] | None = None,
logprobs: list[dict | None] | None = None,
raw_choices: list[dict] | None = None,
) -> dict:
"""Feed `output_ids` through the streaming chat derender endpoint in
the given `chunk_sizes`, threading `stream_state` across calls and
@@ -64,7 +66,13 @@ async def stream_chat_derender(
`tool_calls` assembled so far, letting callers assert properties of the
intermediate deltas (e.g. monotonic argument growth) rather than only
the final assembled result.
`logprobs` supplies one `GenerateLogProbs` payload (or `None`) per chunk
and `raw_choices`, if given, collects every derendered choice so callers
can compare per-chunk fields such as `logprobs`.
"""
if logprobs is not None:
assert len(logprobs) == len(chunk_sizes)
state = None
choices: list[dict] = []
pos = 0
@@ -84,6 +92,7 @@ async def stream_chat_derender(
"index": 0,
"token_ids": tids,
"finish_reason": finish_reason if is_last else None,
"logprobs": logprobs[i] if logprobs is not None else None,
}
],
},
@@ -101,6 +110,8 @@ async def stream_chat_derender(
assert data["chunk"]["choices"][0]["token_ids"] == tids
state = data["stream_state"]
choices.extend(data["chunk"]["choices"])
if raw_choices is not None:
raw_choices.extend(data["chunk"]["choices"])
if on_chunk is not None:
on_chunk(assemble_stream(choices)["tool_calls"])
@@ -1,12 +0,0 @@
model_name: "deepseek-ai/DeepSeek-R1"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--data-parallel-size 8
--enable-expert-parallel
--attention-backend=TRITON_ATTN
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -1,12 +0,0 @@
model_name: "deepseek-ai/DeepSeek-R1"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 8
--enable-expert-parallel
--attention-backend=TRITON_ATTN
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -1,12 +0,0 @@
model_name: "deepseek-ai/DeepSeek-V3.2"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--data-parallel-size 8
--enable-expert-parallel
--attention-backend=TRITON_ATTN
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -1,12 +0,0 @@
model_name: "deepseek-ai/DeepSeek-V3.2"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 8
--enable-expert-parallel
--attention-backend=TRITON_ATTN
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -12,4 +12,5 @@ server_args: >-
--enable-expert-parallel
--mamba-backend triton
--moe-backend triton
--attention-backend ROCM_AITER_FA
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -12,4 +12,5 @@ server_args: >-
--enable-expert-parallel
--mamba-backend triton
--moe-backend triton
--attention-backend ROCM_AITER_FA
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -1,8 +0,0 @@
DeepSeek-R1-TP_MI325.yaml
DeepSeek-R1-DP_MI325.yaml
DeepSeek-V3.2-TP_MI325.yaml
DeepSeek-V3.2-DP_MI325.yaml
Qwen3-30B-A3B-NVFP4.yaml
Qwen3-30B-A3B-NVFP4-quark.yaml
Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
-40
View File
@@ -353,46 +353,6 @@ def test_silu_and_mul_with_clamp(
)
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
@torch.inference_mode()
def test_masked_situ_and_mul(
default_vllm_config,
linear_beta: float,
dtype: torch.dtype,
) -> None:
"""Masked SITU computes valid expert rows and preserves padded zeros."""
device = CUDA_DEVICES[0]
num_experts, max_num_tokens, d = 4, 7, 512
beta = 1.5
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
torch.ops._C.masked_situ_and_mul(
output, input, expert_num_tokens, beta, linear_beta
)
gate, up = input.float().chunk(2, dim=-1)
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
if linear_beta > 0:
up = linear_beta * torch.tanh(up / linear_beta)
expected = (expected * up).to(dtype)
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
torch.testing.assert_close(
output[expert, :num_tokens],
expected[expert, :num_tokens],
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
opcheck(
torch.ops._C.masked_situ_and_mul,
(output, input, expert_num_tokens, beta, linear_beta),
)
MOE_ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
pytest.param(
+112
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
import pytest
import torch
import torch.nn.functional as F
@@ -358,3 +360,113 @@ def test_causal_conv1d_varlen(
)
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
@pytest.mark.parametrize("itype", [torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [True])
@pytest.mark.parametrize("has_bias", [True])
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("dim", [64, 2048])
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
@pytest.mark.skipif(
current_platform.is_cpu(),
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
)
def test_causal_conv1d_update_ragged_varlen(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
mk_inputs = lambda: _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
)
inputs, state = mk_inputs()
actual = causal_conv1d_update(conv_state=state, **inputs)
ref_inputs, ref_state = mk_inputs()
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
for i, ref_out in enumerate(ref_outs):
start = int(inputs["query_start_loc"][i])
end = int(inputs["query_start_loc"][i + 1])
entry = inputs["conv_state_indices"][i : i + 1]
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
assert torch.equal(state[entry], ref_state[entry])
def _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
device = DEVICE
set_random_seed(0)
batch_size = len(q_lens_cfg)
max_query_len = 5
assert max(q_lens_cfg) <= max_query_len
state_len = width - 1 + max_query_len
total_entries = 10 * batch_size
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
query_start_loc = torch.cat(
[
torch.zeros(1, dtype=torch.int32, device=device),
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
]
)
total_tokens = int(q_lens.sum())
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
conv_state = torch.randn(
total_entries, state_len, dim, device=device, dtype=itype
).transpose(1, 2)
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
activation = None if not silu_activation else "silu"
conv_state_indices = (
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
).to(torch.int32)
num_accepted_tokens = torch.randint(
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
)
inputs = dict(
x=x,
weight=weight,
bias=bias,
activation=activation,
conv_state_indices=conv_state_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=max_query_len,
)
return inputs, conv_state
def _create_ragged_varlen_ref_outs(
x,
conv_state,
weight,
bias,
activation,
conv_state_indices,
num_accepted_tokens,
query_start_loc,
max_query_len,
) -> Iterator[torch.Tensor]:
for seq_idx in range(len(conv_state_indices)):
start = int(query_start_loc[seq_idx])
end = int(query_start_loc[seq_idx + 1])
entry = conv_state_indices[seq_idx : seq_idx + 1]
if start == end:
yield x[start:end]
continue
out_ref = causal_conv1d_update(
x[start:end],
conv_state,
weight,
bias,
activation=activation,
conv_state_indices=entry,
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
query_start_loc=query_start_loc.new_tensor([0, end - start]),
max_query_len=max_query_len,
)
yield out_ref
+47
View File
@@ -6,6 +6,10 @@ import torch
import torch.nn.functional as F
from einops import rearrange, repeat
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
_chunk_scan_fwd,
_chunk_scan_fwd_kernel,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -578,3 +582,46 @@ def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
rtol=rtol,
msg=lambda x, i=i: f"seq{i} state " + x,
)
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
"""The autotuner picks a config by speed only, so every config in the list
must give the same finite output. On sm90, configs that pipeline a wgmma
with `cb` in registers wrote NaN into whole rows (#61603)."""
from triton.runtime.errors import OutOfResources
set_random_seed(0)
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
seqlen, nchunks = 181, 2
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
ref = None
for config in list(_chunk_scan_fwd_kernel.configs):
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
for _ in range(10):
out = torch.empty_like(x)
try:
_chunk_scan_fwd(
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
)
except OutOfResources:
# The autotuner skips configs that do not fit this device.
break
assert torch.isfinite(out).all(), f"non-finite output with {config}"
if ref is None:
ref = out
torch.testing.assert_close(
out,
ref,
atol=2e-2,
rtol=2e-2,
msg=lambda m, config=config: f"{config}: {m}",
)
+13 -1
View File
@@ -10,7 +10,6 @@ import torch.nn.functional as F
import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.layers.mhc as mhc_layers
from vllm.model_executor.kernels.mhc.tilelang import (
_torch_hc_prenorm_gemm,
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
@@ -1002,6 +1001,19 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
torch.testing.assert_close(actual, expected, atol=5e-2, rtol=1e-2)
def _torch_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
) -> None:
assert out.shape[0] == 1
assert sqrsum.shape[0] == 1
x_float = x.float()
out[0].copy_(x_float @ fn.t())
sqrsum[0].copy_(x_float.square().sum(dim=-1))
@pytest.mark.skipif(
not HAS_TILELANG_MHC,
reason="TileLang MHC support required",
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import os
import time
import traceback
from collections.abc import Callable
from typing import Any, Concatenate
@@ -23,6 +24,8 @@ from vllm.utils.network_utils import get_open_port
P = ParamSpec("P")
LAUNCH_TIMEOUT_S = 900
@dataclasses.dataclass
class ProcessGroupInfo:
@@ -148,7 +151,7 @@ def parallel_launch_with_config(
*args: P.args,
**kwargs: P.kwargs,
) -> None:
spawn(
context = spawn(
_worker_parallel_launch,
args=(
world_size,
@@ -162,5 +165,16 @@ def parallel_launch_with_config(
)
+ args,
nprocs=world_size,
join=True,
join=False,
)
# A worker stuck in communicator setup never exits, so bound the wait.
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
if time.monotonic() >= deadline:
for process in context.processes:
process.kill()
for process in context.processes:
process.join()
raise TimeoutError(
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
)
+11 -1
View File
@@ -127,6 +127,14 @@ if has_nixl_ep():
BACKENDS += ["nixl_ep"]
DEEPEP_BACKENDS = {"deepep_high_throughput", "deepep_low_latency"}
# These tests run on one node, where NVSHMEM reaches peers over NVLink, so
# keep it off the NICs to avoid depending on IB setup on shared CI hosts.
DEEPEP_SINGLE_NODE_NVSHMEM_ENV = {
"NVSHMEM_REMOTE_TRANSPORT": "none",
# DeepEP forces IBGDA on, so also hide the NICs from it.
"NVSHMEM_HCA_LIST": "none",
}
MORI_BACKENDS = {"mori_high_throughput", "mori_low_latency"}
QUANT_METHODS = [
@@ -2187,6 +2195,8 @@ def test_moe_layer(
rocm_deepep = current_platform.is_rocm() and backend in DEEPEP_BACKENDS
parallel_worker = _parallel_worker_rocm_deepep if rocm_deepep else _parallel_worker
cuda_deepep = current_platform.is_cuda() and backend in DEEPEP_BACKENDS
env_dict = DEEPEP_SINGLE_NODE_NVSHMEM_ENV if cuda_deepep else None
launch_failures: list[str] = []
try:
@@ -2197,7 +2207,7 @@ def test_moe_layer(
world_size,
parallel_worker,
vllm_config,
None,
env_dict,
test_config_batch,
verbosity,
failure_report_path=failure_report_path,
+14
View File
@@ -582,6 +582,20 @@ def test_fused_topk_bias(
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
@pytest.mark.parametrize("scoring_func", [0, 1])
def test_fused_grouped_topk_accepts_idle_rank(scoring_func):
"""Zero-token routing must leave CUDA ready for the next batch."""
if not current_platform.is_cuda():
pytest.skip("CUDA required")
from vllm import _custom_ops as ops
scores = torch.empty((0, 8), device="cuda", dtype=torch.float32)
bias = torch.zeros(8, device="cuda")
weights, ids = ops.grouped_topk(scores, 2, 1, 2, True, 1.0, bias, scoring_func)
torch.testing.assert_close(weights, scores.new_empty((0, 2)))
torch.testing.assert_close(ids, scores.new_empty((0, 2), dtype=torch.int32))
@pytest.mark.parametrize("m,k", MK_S)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize(
@@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
@pytest.mark.parametrize("hidden_size", [256, 2048])
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
def test_silu_quant_skips_expert_padding_on_graph_replay(
counts: list[int],
group_size: int,
@@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
x *= 16
clean_x = x.clone()
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
output_dtype = (
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
def set_counts(values):
positions = []
@@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
output.view(torch.uint8).fill_(91)
graph.replay()
reference_q, reference_s = quantize(x)
assert actual_q.dtype == reference_q.dtype
torch.testing.assert_close(
actual_q.view(torch.uint8)[live],
reference_q.view(torch.uint8)[live],
+57 -15
View File
@@ -3,11 +3,11 @@
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
Supported (hidden_size, num_experts) pairs:
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)
Correctness baseline: F.linear in float32. Every M in [1, 32] is covered so
Correctness baseline: F.linear in float32. Every supported M is covered so
all tuned geometries (wide-block, experts-per-block, token-group; boundaries
at M=4/5, odd/even, M=15/16) are exercised on Blackwell, and the legacy
at M=4/5, M=9/10, M=12/13, odd/even, M=15/16) are exercised on Blackwell, and the legacy
128/1 geometry everywhere else.
"""
@@ -18,8 +18,12 @@ from vllm._custom_ops import fp32_router_gemm
from vllm.platforms import current_platform
# (hidden_size, num_experts)
SHAPES = [(3072, 256), (6144, 128)]
ALL_M = list(range(1, 33))
SHAPES = [(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)]
CASES = [
(m, hidden, experts)
for hidden, experts in SHAPES
for m in range(1, 49 if hidden == 5120 else 33)
]
# Absolute tolerance for fp32 kernel vs float64 reference
ATOL_FP32 = 2e-4
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
@@ -40,8 +44,7 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", ALL_M)
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""fp32 activation → fp32 output should match reference closely."""
_requires_sm90()
@@ -58,8 +61,7 @@ def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", ALL_M)
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""bf16 activation → fp32 output should match reference within bf16 error."""
_requires_sm90()
@@ -124,11 +126,11 @@ def test_topk_routing_consistency(num_tokens: int, hidden_dim: int, num_experts:
)
def test_zero_tokens_returns_empty():
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
def test_zero_tokens_returns_empty(hidden_dim, num_experts):
"""M=0 is a graceful no-op returning an empty [0, E] tensor."""
_requires_sm90()
device = torch.device("cuda")
hidden_dim, num_experts = SHAPES[0]
mat_a = torch.empty(0, hidden_dim, dtype=torch.float32, device=device)
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
out = fp32_router_gemm(mat_a, mat_b)
@@ -136,17 +138,19 @@ def test_zero_tokens_returns_empty():
assert out.dtype == torch.float32
def test_rejects_invalid_inputs():
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
def test_rejects_invalid_inputs(hidden_dim, num_experts):
"""The entry must fail loudly, never compute silently wrong results."""
_requires_sm90()
device = torch.device("cuda")
hidden_dim, num_experts = SHAPES[0]
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
# num_tokens > 32 (beyond the instantiated range)
# Beyond the instantiated range for this shape.
invalid_tokens = 49 if hidden_dim == 5120 else 33
with pytest.raises(Exception, match="num_tokens"):
fp32_router_gemm(
torch.randn(33, hidden_dim, dtype=torch.float32, device=device), mat_b
torch.randn(invalid_tokens, hidden_dim, dtype=torch.float32, device=device),
mat_b,
)
# unsupported (hidden_dim, num_experts) pair
@@ -173,3 +177,41 @@ def test_rejects_invalid_inputs():
fp32_router_gemm(
torch.randn(4, hidden_dim, dtype=torch.float16, device=device), mat_b
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize(
"tokens", [*range(1, 17), 17, 24, 32, 33, 40, 48, 56, 64, 65, 128]
)
@pytest.mark.parametrize(
"hidden_dim,num_experts", [(5120, 256), (4096, 352), (8192, 256)]
)
@torch.inference_mode()
def test_router_retains_fp32_weights_and_logits(
default_vllm_config, dist_init, tokens, hidden_dim, num_experts
):
"""Low-latency routing must retain FP32 weight precision and logits."""
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("SM100 required")
torch.manual_seed(42)
router = GateLinear(
hidden_dim, num_experts, params_dtype=torch.float32, out_dtype=torch.float32
).cuda()
router.weight.normal_(std=0.01)
x = torch.randn(tokens, hidden_dim, device="cuda", dtype=torch.bfloat16)
expected = (x.double() @ router.weight.double().T).float()
actual, bias = router(x)
assert bias is None and actual.dtype == torch.float32
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-5)
torch.testing.assert_close(actual.topk(8).indices, expected.topk(8).indices)
if tokens in (1, 16, 32, 33, 48, 56):
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured, _ = router(x)
x.normal_()
graph.replay()
expected = (x.double() @ router.weight.double().T).float()
torch.testing.assert_close(captured, expected, rtol=2e-5, atol=2e-5)
+25 -1
View File
@@ -566,7 +566,7 @@ def test_invalid_output_dtype():
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
ll_bf16_gemm(a, b, output_dtype=torch.bfloat16)
ll_bf16_gemm(a, b, output_dtype=torch.float16)
def test_cache_miss_compiles_dotprod():
@@ -581,5 +581,29 @@ def test_cache_miss_compiles_dotprod():
_assert_close(out, _ref(a, b), context="cache miss dotprod")
@pytest.mark.parametrize("M", [1, 2, 8, 16])
@pytest.mark.parametrize("N", [1792, 3584])
@torch.inference_mode()
def test_bf16_output_and_fp32_cache_coexist(M, N):
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(42)
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
for dtype in (torch.float32, torch.bfloat16):
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
assert actual.dtype == dtype
reference = (x.double() @ w.double().T).to(dtype)
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
x.normal_()
graph.replay()
reference = (x.double() @ w.double().T).bfloat16()
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -4,7 +4,7 @@
import json
import sys
from collections import defaultdict
from contextlib import nullcontext
from contextlib import contextmanager, nullcontext
from inspect import signature
from pathlib import Path
from types import ModuleType, SimpleNamespace
@@ -12,8 +12,10 @@ from typing import Any
from unittest.mock import Mock, call, patch
import pytest
import torch
from vllm.model_executor.warmup import kernel_warmup as warmup
import vllm.model_executor.warmup.kernel_warmup as warmup
import vllm.utils.flashinfer as fi_utils
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
@@ -45,7 +47,12 @@ def _make_runner(modules, *, max_tokens: int = 8192):
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=None),
parallel_config=SimpleNamespace(data_parallel_rank=0),
parallel_config=SimpleNamespace(
enable_elastic_ep=False,
nnodes=1,
pipeline_parallel_size=1,
data_parallel_rank=0,
),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
@@ -133,6 +140,258 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
]
@pytest.mark.parametrize(
"has_v2,elastic_ep,node_count",
[(False, False, 1), (True, False, 1), (True, True, 1), (True, False, 2)],
ids=["v1", "v2", "elastic-v1", "multi-node-v1"],
)
@pytest.mark.parametrize(
"hisparse,world_size,rank,skip_ops,failure",
[
(False, 1, 0, [], None),
(False, 2, 0, [], None),
(False, 2, 1, [], None),
(True, 2, 0, ["fp4_gemm", "bmm_fp8"], None),
(False, 2, 1, ["fp4_gemm"], "dummy"),
(False, 2, 1, [], "replayssm"),
(True, 2, 0, [], "kimi"),
],
)
def test_flashinfer_autotune_lifecycle(
monkeypatch,
tmp_path,
has_v2,
elastic_ep,
node_count,
hisparse,
world_size,
rank,
skip_ops,
failure,
):
use_v2 = has_v2 and not elastic_ep and node_count == 1 and world_size == 1
runner = _make_runner([])
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
runner.vllm_config.attention_config.hisparse_config = object() if hisparse else None
model = runner.get_model()
cache_path = tmp_path / "autotune.json"
rank_cache_path = cache_path.with_name(f"autotune_dp0_rank{rank}.json")
rank_cache_path.write_text(json.dumps({"_metadata": {}, "op": 1}))
events = Mock()
events.tuner.load_configs.return_value = True
events.tuner._dirty = True
world = SimpleNamespace(
rank_in_group=rank,
world_size=world_size,
cpu_group=object(),
broadcast_object=events.broadcast,
barrier=events.barrier,
)
@contextmanager
def autotune_context(**kwargs):
assert torch.is_inference_mode_enabled()
events.enter()
try:
yield
finally:
events.exit()
events.autotune.side_effect = autotune_context
if failure is not None:
getattr(events, failure).side_effect = RuntimeError("warmup failed")
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
SimpleNamespace(
AutoTuner=SimpleNamespace(get=lambda: events.tuner),
set_autotune_process_group=events.set_group,
),
)
monkeypatch.setitem(
sys.modules,
"flashinfer",
SimpleNamespace(autotune_v2=events.autotune, autotune_v2_reload=events.reload),
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_world_group", lambda: world
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_pp_group",
lambda: SimpleNamespace(world_size=1),
)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: has_v2)
monkeypatch.setattr(fi_utils, "autotune", events.autotune)
monkeypatch.setattr(warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", skip_ops)
resolve_file = Mock(return_value=cache_path)
resolve_root = Mock(return_value=str(tmp_path))
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_file", resolve_file)
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_v2_root", resolve_root)
def gather(out, value, *, group):
out[:] = [value] * world_size
monkeypatch.setattr(torch.distributed, "all_gather_object", gather)
monkeypatch.setattr(warmup, "_run_flashinfer_autotune_dummy_runs", events.dummy)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", events.replayssm)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", events.kimi)
monkeypatch.setattr(
warmup, "autotune_hisparse_flashinfer_attention", events.hisparse
)
if failure is None:
warmup.flashinfer_autotune(runner)
else:
with pytest.raises(RuntimeError, match="warmup failed"):
warmup.flashinfer_autotune(runner)
kwargs = {"skip_ops": set(skip_ops)} if skip_ops else {}
expected = []
if use_v2:
resolve_file.assert_not_called()
resolve_root.assert_called_once_with()
expected.append(call.autotune(mode="tune", cache_root=str(tmp_path), **kwargs))
else:
resolve_root.assert_not_called()
resolve_file.assert_called_once_with(runner)
expected.extend(
[
call.tuner.load_configs(str(rank_cache_path)),
call.autotune(tune_mode=True, **kwargs),
]
)
expected.extend(
[call.set_group(world.cpu_group if world_size > 1 else None), call.enter()]
)
if hisparse:
expected.append(call.hisparse(runner))
for name, arg, kwargs in [
("dummy", runner, {"skip_attn": hisparse}),
("replayssm", runner, {}),
("kimi", model, {}),
]:
expected.append(getattr(call, name)(arg, **kwargs))
if failure == name:
break
expected.append(call.exit())
expected.append(call.set_group(None))
if failure is None:
if world_size > 1:
expected.append(call.barrier())
if use_v2 and world_size > 1:
expected.extend([call.reload(), call.barrier()])
elif not use_v2:
expected.append(call.tuner.save_configs(str(rank_cache_path)))
assert events.mock_calls == expected
@pytest.mark.parametrize(
"node_count,elastic_ep,pp_size",
[(1, False, 1), (2, False, 1), (1, True, 1), (1, False, 2)],
)
@pytest.mark.parametrize("world_size,rank", [(1, 0), (2, 0), (2, 1)])
@pytest.mark.parametrize("model_runner_v2,max_reqs", [(False, 2), (True, 1), (True, 2)])
def test_sparse_autotune_respects_cache_selection(
monkeypatch,
tmp_path,
node_count,
elastic_ep,
pp_size,
world_size,
rank,
model_runner_v2,
max_reqs,
):
import vllm.model_executor.warmup.flashinfer_sparse_mla_warmup as sparse
runner = _make_runner([])
runner.max_num_reqs = max_reqs
runner.vllm_config.use_v2_model_runner = model_runner_v2
runner.vllm_config.kernel_config = SimpleNamespace(enable_flashinfer_autotune=True)
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
runner.vllm_config.parallel_config.pipeline_parallel_size = pp_size
worker = SimpleNamespace(
model_runner=runner,
vllm_config=runner.vllm_config,
execute_model=Mock(),
sample_tokens=Mock(),
)
world = SimpleNamespace(
rank_in_group=rank,
world_size=world_size,
broadcast_object=Mock(return_value=b"cached"),
barrier=Mock(),
)
managed = Mock(return_value=nullcontext())
legacy = Mock(return_value=nullcontext())
reload = Mock()
tuner = Mock()
monkeypatch.setitem(
sys.modules,
"flashinfer",
SimpleNamespace(autotune_v2=managed, autotune_v2_reload=reload),
)
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
SimpleNamespace(AutoTuner=SimpleNamespace(get=lambda: tuner)),
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_world_group", lambda: world
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: True)
monkeypatch.setattr(sparse, "has_flashinfer", lambda: True)
monkeypatch.setattr(
sparse.current_platform, "is_device_capability_family", lambda _: True
)
monkeypatch.setattr(
sparse, "_flashinfer_sparse_mla_decode_label", lambda *_: "DSv3.2"
)
monkeypatch.setattr(sparse, "resolve_flashinfer_autotune_v2_root", lambda: tmp_path)
cache_path = tmp_path / "autotune.json"
cache_path.write_bytes(b"cached")
monkeypatch.setattr(
sparse, "resolve_flashinfer_autotune_file", lambda _: cache_path
)
monkeypatch.setattr(sparse, "flashinfer_autotune", legacy)
def mixed_warmup(*args, mixed_step_context=None, **kwargs):
context = (
mixed_step_context if mixed_step_context is not None else nullcontext()
)
with context:
assert torch.is_inference_mode_enabled()
return True
mixed = Mock(side_effect=mixed_warmup)
monkeypatch.setattr(sparse, "run_mixed_prefill_decode_warmup", mixed)
assert sparse._run_flashinfer_sparse_mla_decode_autotune(worker, 16, frozenset())
if node_count == 1 and not elastic_ep and pp_size == 1 and world_size == 1:
managed.assert_called_once_with(mode="tune", cache_root=tmp_path)
legacy.assert_not_called()
world.broadcast_object.assert_not_called()
tuner.load_configs.assert_not_called()
reload.assert_not_called()
world.barrier.assert_not_called()
else:
managed.assert_not_called()
reload.assert_not_called()
assert legacy.call_count == (1 if rank == 0 else 0)
world.broadcast_object.assert_called_once_with(
b"cached" if rank == 0 else None, src=0
)
tuner.load_configs.assert_called_once_with(str(cache_path))
assert mixed.call_count == (1 if model_runner_v2 and max_reqs >= 2 else 0)
assert runner._dummy_run.call_count == (0 if mixed.called else 1)
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
"""The drafter's M can exceed the pass size; buckets must include it."""
runner = _make_runner([])
@@ -266,7 +525,9 @@ class _AutotuneRun:
for rank in range(self.pp * self.tp):
self.rank = rank
self.tuners[rank] = _AutotuneTuner(self)
flashinfer_autotune(_make_runner([]))
runner = _make_runner([])
runner.vllm_config.parallel_config.pipeline_parallel_size = self.pp
flashinfer_autotune(runner)
return self
def assert_collectives_match(self):
@@ -283,6 +544,8 @@ def autotune_run(monkeypatch, tmp_path):
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp, tmp_path)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: pp > 1)
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
@@ -168,23 +168,28 @@ def _warmup_store_kernel(out, VALUE: tl.constexpr):
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
def test_registered_warmup_overlaps_kernels_and_runtime_stays_serial(
monkeypatch,
) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2))
def make_store(value):
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=value)
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler.
# Initialize Triton's binder before wrapping its compiler.
store.get_warmup_keys()
return store
stores = [make_store(value) for value in (1, 2)]
caller = threading.get_ident()
barrier = threading.Barrier(2, timeout=10)
compiled_on = []
@@ -200,14 +205,15 @@ def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
registry = JitWarmupRegistry(None)
with registry.activate():
store.register_warmup()
for store in stores:
store.register_warmup()
registry.warmup()
assert len(compiled_on) == 2
assert all(thread != caller for thread in compiled_on)
assert active_mode.get() is None
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
for value in (1, 2):
for store, value in zip(stores, (1, 2)):
store(out, value)
assert out.item() == value
assert len(compiled_on) == 2
@@ -220,3 +220,44 @@ def test_real_triton_kernel_override_end_to_end() -> None:
"out_ptr",
"BLOCK",
)
def test_cpu_slot_mapping_override_forwards_kernel_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm.utils import cpu_triton_utils
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
calls: list[tuple[Any, ...]] = []
fake_torch = types.SimpleNamespace(
ops=types.SimpleNamespace(
_C=types.SimpleNamespace(
compute_slot_mapping_kernel_impl=lambda *args: calls.append(args)
)
)
)
monkeypatch.setattr(cpu_triton_utils, "torch", fake_torch)
override = KernelOverride(
ComputeSlotMappingKernel.kernel,
cpu_triton_utils._compute_slot_mapping_kernel_impl,
)
override[(2,)](
num_tokens=2,
max_num_tokens=4,
query_start_loc_ptr="query_start_loc",
positions_ptr="positions",
block_table_ptr="block_table",
block_table_stride=128,
slot_mapping_ptr="slot_mapping",
KV_CACHE_BLOCK_SIZE=16,
TOTAL_CP_WORLD_SIZE=1,
TOTAL_CP_RANK=0,
CP_KV_CACHE_INTERLEAVE_SIZE=1,
PAD_ID=-1,
BLOCK_SIZE=1024,
)
assert calls == [
("query_start_loc", "positions", "block_table", "slot_mapping", 16)
]
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
Adaptive verification splits the draft budget evenly on the host, then trims
each verify request on device. The KDA spec path must therefore plan from the
device offsets, never from host lengths that merely look uniform.
"""
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.models.kimi_k3.amd.kda_metadata import (
KimiK3ROCmKDABackend,
KimiK3ROCmKDAMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
NUM_SPEC = 3
def _config(adaptive: bool):
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=NUM_SPEC,
enable_adaptive_verification=adaptive,
)
vllm_config.cache_config.mamba_cache_mode = "none"
return vllm_config
def _spec() -> MambaSpec:
return MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=NUM_SPEC,
)
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
return KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=_spec(),
layer_names=["layer.0"],
vllm_config=_config(adaptive),
device=DEVICE,
)
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
)
common = common.replace(
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
)
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
def test_adaptive_verification_plans_from_device_offsets():
"""Equal host lengths would select the fixed-length recurrent kernel,
which places sequence i at i * L and ignores the device cu_seqlens."""
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
assert bound(_config(adaptive=False), _spec()) is None
meta = _trimmed_verify_batch(_builder(adaptive=True))
assert meta.num_spec_decodes == 2
assert meta.uniform_spec_sequence_length is None
assert torch.equal(
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
)
# Without adaptive verification the host lengths are exact and kept.
meta = _trimmed_verify_batch(_builder(adaptive=False))
assert meta.uniform_spec_sequence_length == 3
@@ -482,7 +482,7 @@ def _spec_for(monkeypatch, **attrs) -> MambaSpec:
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
for name, value in dict(
use_prefill_checkpoint=True,
use_fused_chunk=True,
kda_prefill_backend="fused",
num_spec=0,
use_safe_gate=True,
).items():
@@ -506,7 +506,8 @@ def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
"attr,value",
[
("use_prefill_checkpoint", False),
("use_fused_chunk", False),
("kda_prefill_backend", "triton"),
("kda_prefill_backend", "gluon"),
("use_safe_gate", False),
],
)
+23 -2
View File
@@ -77,7 +77,7 @@ def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
use_fused_chunk=use_fused,
backend=kw.pop("backend", "fused" if use_fused else "triton"),
out=out,
**kw,
)
@@ -112,6 +112,27 @@ def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize(
"seqlens",
[[1], [64], [1024], [4096], [513, 64, 1, 1200], [512] * 6],
)
def test_gluon_chunk_matches_triton(seqlens: list[int]) -> None:
"""AITER's Gluon chunk_kda must match the Triton chunk path too."""
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_kda_gluon_prefill_available():
pytest.skip("AITER Gluon chunk_kda is not available here")
inp = _inputs(seqlens, seed=len(seqlens))
o_ref, state_ref = _run(inp, use_fused=False)
o_gluon, state_gluon = _run(inp, use_fused=False, backend="gluon")
torch.testing.assert_close(o_gluon.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_gluon, state_ref, rtol=5e-3, atol=5e-3)
state_rel = (state_gluon - state_ref).norm() / state_ref.norm()
assert state_rel < 5e-3, f"final state relative L2 error {state_rel:.2e}"
@pytest.mark.parametrize("use_fused", [False, True])
def test_out_argument_receives_the_result(use_fused: bool) -> None:
"""``out`` is a promise about where the result lands.
@@ -648,7 +669,7 @@ def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
has_initial_state=warm,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
use_fused_chunk=use_fused,
backend="fused" if use_fused else "triton",
**kw,
)
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="AITER fused conv1d is ROCm-only"
)
WIDTH = 4 # Kimi-K3 short_conv_kernel_size
STATE_LEN = WIDTH - 1
HEAD_DIM = 128
LOCAL_HEADS = 4 # per-TP-rank heads; projection = 512, packed dim = 1536
def _fused_fn():
gdn = pytest.importorskip("aiter.ops.triton.gated_delta_net")
return gdn.causal_conv1d_split_qkv_triton_tile_fn
def _make_inputs(seqlens, has_initial_state, seed=0, device="cuda"):
torch.manual_seed(seed)
proj = HEAD_DIM * LOCAL_HEADS
dim = 3 * proj
total = sum(seqlens)
n_seqs = len(seqlens)
# Token-major activations, as produced by the in_proj GEMM.
mixed_qkv = torch.randn(total, dim, dtype=torch.bfloat16, device=device)
weight = torch.randn(dim, WIDTH, dtype=torch.bfloat16, device=device) * 0.3
# More cache lines than sequences, assigned in a shuffled order. Line 0 is
# vLLM's null block (NULL_BLOCK_ID), which causal_conv1d_fn skips, so real
# sequences never map to it.
n_lines = n_seqs + 2
conv_state = torch.randn(
n_lines, dim, STATE_LEN, dtype=torch.bfloat16, device=device
)
cache_indices = (torch.randperm(n_lines - 1, device=device)[:n_seqs] + 1).to(
torch.int32
)
query_start_loc = torch.tensor(
[0] + list(torch.tensor(seqlens).cumsum(0).tolist()),
dtype=torch.int32,
device=device,
)
has_initial = torch.full(
(n_seqs,), has_initial_state, dtype=torch.bool, device=device
)
return mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
def _reference(
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
):
"""fp32 per-sequence depthwise causal conv + SiLU.
Returns (out[T, dim], new_conv_state).
"""
x = mixed_qkv.float().t() # (dim, T)
w = weight.float()
dim = x.shape[0]
out = torch.empty_like(x)
new_state = conv_state.float().clone()
qsl = query_start_loc.tolist()
for s in range(len(qsl) - 1):
start, end = qsl[s], qsl[s + 1]
ci = int(cache_indices[s])
xs = x[:, start:end]
if bool(has_initial[s]):
xin = torch.cat([conv_state[ci].float(), xs], dim=1)
else:
xin = F.pad(xs, (STATE_LEN, 0))
y = F.conv1d(xin.unsqueeze(0), w.unsqueeze(1), None, groups=dim)[0]
out[:, start:end] = F.silu(y)
new_state[ci] = xin[:, -STATE_LEN:]
return out.t(), new_state
def _three_launch(
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
):
"""The pre-fusion KDA prefill path: one causal_conv1d_fn per projection."""
proj = mixed_qkv.shape[1] // 3
outs = []
for x, state, w in zip(
mixed_qkv.split(proj, dim=-1),
conv_state.split(proj, dim=-2),
weight.split(proj, dim=0),
):
outs.append(
causal_conv1d_fn(
x.transpose(0, 1),
w,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial,
cache_indices=cache_indices,
query_start_loc=query_start_loc,
).transpose(0, 1)
)
return outs
@pytest.mark.parametrize(
"seqlens", [[1024], [8, 1, 300], [64, 64, 64], [513, 64, 1, 1200]]
)
@pytest.mark.parametrize("has_initial_state", [False, True])
@pytest.mark.parametrize("block_m", [8, 128])
def test_fused_split_qkv_conv1d_matches_reference(seqlens, has_initial_state, block_m):
fused_fn = _fused_fn()
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial = _make_inputs(
seqlens, has_initial_state
)
proj = mixed_qkv.shape[1] // 3
ref_out, ref_state = _reference(
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial
)
state_3 = conv_state.clone()
q3, k3, v3 = _three_launch(
mixed_qkv, weight, state_3, cache_indices, qsl, has_initial
)
state_f = conv_state.clone()
# Exactly the call made by KimiK3DeltaAttention._forward.
qf, kf, vf = fused_fn(
mixed_qkv.transpose(0, 1),
weight,
None,
state_f,
qsl,
proj,
proj,
cache_indices=cache_indices,
has_initial_state=has_initial,
activation="silu",
block_m=block_m,
metadata=None,
preserve_input_dtype=True,
)
fused = torch.cat([qf, kf, vf], dim=-1)
assert fused.shape == mixed_qkv.shape and fused.dtype == mixed_qkv.dtype
assert all(t.is_contiguous() for t in (qf, kf, vf))
tol = dict(atol=2e-2, rtol=2e-2)
torch.testing.assert_close(fused.float(), ref_out, **tol)
torch.testing.assert_close(fused, torch.cat([q3, k3, v3], dim=-1), **tol)
# conv_state writeback: assigned lines updated, the others untouched.
torch.testing.assert_close(state_f.float(), ref_state, **tol)
torch.testing.assert_close(state_f, state_3, **tol)
untouched = torch.ones(conv_state.shape[0], dtype=torch.bool, device="cuda")
untouched[cache_indices.long()] = False
assert untouched[NULL_BLOCK_ID]
assert torch.equal(state_f[untouched], conv_state[untouched])
+18
View File
@@ -85,6 +85,24 @@ def test_image_media_io_rgba_custom_background(tmp_path):
assert green_numpy[0][0][2] == 0 # B
@pytest.mark.parametrize("background_color", [(255, 255, 255), (12, 34, 56)])
def test_image_media_io_rgb_png_transparency(tmp_path, background_color):
image = Image.new("RGB", (2, 1), (255, 0, 0))
image.putpixel((1, 0), (0, 0, 255))
path = tmp_path / "rgb_transparency.png"
image.save(path, transparency=(255, 0, 0))
with Image.open(path) as source:
reference = Image.alpha_composite(
Image.new("RGBA", source.size, (*background_color, 255)),
source.convert("RGBA"),
).convert("RGB")
image_io = ImageMediaIO(rgba_background_color=background_color)
converted = image_io.load_file(path)
np.testing.assert_array_equal(np.asarray(converted), np.asarray(reference))
def test_image_media_io_no_mode_conversion(tmp_path):
"""image_mode=None skips conversion and preserves the original mode."""
# RGBA image: opaque black pixel on a fully transparent background
+10 -2
View File
@@ -237,8 +237,16 @@ def _rgba_png_bytes() -> bytes:
return buf.getvalue()
def test_hash_collision_media_io_config():
data = _rgba_png_bytes()
def _rgb_trns_png_bytes() -> bytes:
image = Image.new("RGB", (8, 8), (255, 0, 0))
buf = BytesIO()
image.save(buf, format="PNG", transparency=(255, 0, 0))
return buf.getvalue()
@pytest.mark.parametrize("png_bytes", [_rgba_png_bytes, _rgb_trns_png_bytes])
def test_hash_collision_media_io_config(png_bytes):
data = png_bytes()
white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data)
white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
+65
View File
@@ -5,6 +5,7 @@ from io import BytesIO
from pathlib import Path
import numpy as np
import pybase64
import pytest
from PIL import Image, ImageChops
@@ -13,6 +14,8 @@ from vllm.multimodal.image import (
convert_image_mode,
normalize_image,
)
from vllm.multimodal.media import MediaConnector
from vllm.multimodal.parse import MultiModalDataParser
pytestmark = pytest.mark.cpu_test
@@ -69,6 +72,68 @@ def test_palette_with_trns_to_rgb():
assert (r, g, b) == (255, 0, 0)
def _reopen(image: Image.Image, image_format: str, **save_kwargs) -> Image.Image:
buffer = BytesIO()
image.save(buffer, image_format, **save_kwargs)
return Image.open(buffer)
def _rgb_png() -> Image.Image:
image = Image.new("RGB", (2, 1), (255, 0, 0))
image.putpixel((1, 0), (0, 0, 255))
return _reopen(image, "PNG", transparency=(255, 0, 0))
def _gif() -> Image.Image:
# The transparent index 0 and the opaque index 1 are both black.
image = Image.new("P", (2, 1))
image.putpalette([0] * 768)
image.putpixel((1, 0), 1)
return _reopen(image, "GIF", transparency=0)
def _rgba_png() -> Image.Image:
image = Image.new("RGBA", (2, 1), (255, 0, 0, 0))
image.putpixel((1, 0), (0, 0, 255, 255))
return _reopen(image, "PNG")
@pytest.mark.parametrize(
"make_image",
[
_rgb_png,
lambda: _rgb_png().convert("RGB"),
_gif,
lambda: _gif().convert("RGB"),
_rgba_png,
],
ids=["rgb_png", "rgb_png_to_rgb", "gif", "gif_to_rgb", "rgba_png"],
)
def test_transparency_offline_matches_online(make_image):
"""A PIL image passed offline and the same image sent online as a PNG
end up with the same pixels, matching Pillow's own RGBA conversion.
After ``convert("RGB")``, Pillow keeps the transparent color in ``info``
and saves it as a tRNS chunk, so every pixel of that color is
transparent in both pipelines, including opaque GIF pixels that share it.
"""
image = make_image()
buffer = BytesIO()
image.save(buffer, "PNG")
url = f"data:image/png;base64,{pybase64.b64encode(buffer.getvalue()).decode()}"
parser = MultiModalDataParser()
offline = parser.parse_mm_data({"image": image})["image"].get(0)
online = parser.parse_mm_data({"image": MediaConnector().fetch_image(url)})
online = online["image"].get(0)
np.testing.assert_array_equal(np.asarray(offline), np.asarray(online))
expected = Image.alpha_composite(
Image.new("RGBA", image.size, (255, 255, 255, 255)), image.convert("RGBA")
).convert("RGB")
np.testing.assert_array_equal(np.asarray(online), np.asarray(expected))
def test_l_mode_no_trns_to_rgb():
"""L-mode without transparency should convert directly."""
img = Image.new("L", (4, 4), 128)
+121
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import json
import pytest
@@ -15,11 +16,14 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.minimax_m2 import (
THINK_END,
TOOL_CALL_END,
TOOL_CALL_START,
MinimaxM2Parser,
_minimax_m2_arg_value_extender,
minimax_m2_config,
)
@@ -238,6 +242,123 @@ class TestStreaming:
assert collect_tool_arguments(results) == ""
WRITE_CALL_HEAD = (
'<minimax:tool_call>\n<invoke name="write_file">\n'
'<parameter name="path">/repo/app.py</parameter>\n'
'<parameter name="mode">644</parameter>\n'
'<parameter name="content">'
)
WRITE_CALL_TAIL = "</parameter>\n</invoke>\n</minimax:tool_call>"
# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
# the value, including prefixes of the closing and next-parameter tags.
CONTENT_LINE = 'if a < b > c: print("x\\y") # </p> </param <parameter\té\n\n'
class TestArgValueExtender:
"""The open parameter value streams without re-running the converter."""
@pytest.fixture
def write_tool(self):
return ChatCompletionToolsParam(
function=FunctionDefinition(
name="write_file",
parameters={
"type": "object",
"properties": {
"path": {"type": "string"},
"mode": {"type": "integer"},
"content": {"type": "string"},
},
},
),
)
@pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
def test_long_call_streams_non_streaming_arguments(
self, mock_tokenizer, mock_request, write_tool, use_schema
):
mock_request.tools = [write_tool] if use_schema else []
text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
results = simulate_tool_streaming(
MinimaxM2Parser(mock_tokenizer),
mock_request,
[text[i : i + 7] for i in range(0, len(text), 7)],
)
result = MinimaxM2Parser(mock_tokenizer).extract_tool_calls(text, mock_request)
arguments = collect_tool_arguments(results)
assert arguments == result.tool_calls[0].function.arguments
assert json.loads(arguments)["content"] == CONTENT_LINE * 200
assert ('"mode": 644,' if use_schema else '"mode": "644",') in arguments
def test_conversions_do_not_grow_with_argument_length(
self, mock_tokenizer, mock_request, write_tool
):
mock_request.tools = [write_tool]
base = minimax_m2_config()
converter = base.arg_converter
assert converter is not None
def streaming_conversions(lines: int) -> int:
calls = 0
def counting_converter(raw_args: str, partial: bool) -> str:
nonlocal calls
calls += partial
return converter(raw_args, partial)
parser = ParserEngine(
mock_tokenizer,
parser_engine_config=dataclasses.replace(
base, arg_converter=counting_converter
),
)
text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
simulate_tool_streaming(
parser,
mock_request,
[text[i : i + 4] for i in range(0, len(text), 4)],
)
return calls
assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
def test_extender_stops_at_split_end_tag(self):
extend = _minimax_m2_arg_value_extender('<parameter name="k">')
assert extend is not None
deltas = ['a"\n', "</param", "eter>"]
assert [extend(d) for d in deltas] == ['a\\"\\n', "</param", None]
def test_extender_stops_at_split_next_parameter(self):
extend = _minimax_m2_arg_value_extender("<parameter name='k'>x")
assert extend is not None
assert [extend(d) for d in ["< parameter", " name ="]] == ["< parameter", None]
@pytest.mark.parametrize(
"raw_args",
[
'<parameter name="k">x</parameter>',
'<parameter name="k',
(
'<parameter name="k">1</parameter><parameter name="j">2</parameter>'
'<parameter name="k">3'
),
'<parameter name="a<parameter name=k>x',
'<parameter name="k>x',
],
ids=[
"closed_value",
"open_name",
"repeated_key_not_last",
"earlier_unmatched_opener",
"bare_name_opens_quote",
],
)
def test_no_extender_unless_open_value_is_stable(self, raw_args):
assert _minimax_m2_arg_value_extender(raw_args) is None
class TestReasoning:
def test_extract_reasoning_without_start_token(self, parser, mock_request):
reasoning, content = parser.extract_reasoning(
@@ -380,6 +380,40 @@ class TestDelegatingPromptDetection:
assert delta.reasoning is None
assert delta.content == "answer"
def test_prior_turn_think_end_keeps_non_streaming_reasoning(
self, mock_tokenizer, mock_request
):
parser = _Qwen3DelegatingParser(mock_tokenizer)
parser.set_prompt_token_ids([_THINK_END_ID, _IM_END_ID, _IM_START_ID, _TEXT_ID])
output_ids = [ord("a"), _THINK_END_ID, ord("b")]
reasoning, content, _ = parser.parse(
mock_tokenizer.decode(output_ids),
mock_request,
model_output_token_ids=output_ids,
)
assert reasoning == "a"
assert content == "b"
assert parser.count_reasoning_tokens(output_ids) > 0
def test_think_end_in_current_turn_parses_output_as_content(
self, mock_tokenizer, mock_request
):
parser = _Qwen3DelegatingParser(mock_tokenizer)
parser.set_prompt_token_ids([_IM_START_ID, _THINK_START_ID, _THINK_END_ID])
output_ids = [ord("a"), ord("b")]
reasoning, content, _ = parser.parse(
mock_tokenizer.decode(output_ids),
mock_request,
model_output_token_ids=output_ids,
)
assert reasoning is None
assert content == "ab"
assert parser.count_reasoning_tokens(output_ids) == 0
class TestStreaming:
def test_basic_streaming(self, parser):
+747
View File
@@ -0,0 +1,747 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import pytest
from openai.types.responses import FunctionTool, ToolChoiceFunction
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.config.reasoning import ReasoningConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.entrypoints.openai.responses.protocol import (
ResponsesRequest,
ResponsesResponse,
)
from vllm.exceptions import VLLMValidationError
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
from vllm.parser.parser_manager import ParserManager
from vllm.parser.response_template import ResponseTemplateParser
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
# The `response_template` of the Beam checkpoints.
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
"defaults": {"role": "assistant"},
"start_anchor_pattern": r"<\|start\|>assistant",
"fields": {
"thinking": {
"open": "<|channel|>analysis<|message|>",
"close": "<|end|><|start|>assistant",
"content": "text",
},
"content": {
"open": "<|channel|>final<|message|>",
"close": ["<|return|>", "<|end|>", "<|call|>"],
"content": "text",
},
"tool_calls": {
"open_pattern": (
r" to=(?P<name>[^\s<]+)"
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
),
"close": ["<|end|><|start|>assistant", "<|call|>"],
"repeats": True,
"content": "xml-inline",
"content_args": {
"tag_pattern": (
r'<rfl:parameter name="(?P<key>[^"]+)">'
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
),
"value_parser": {"name": "text", "args": {"strip": False}},
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
},
}
VOCAB = {
"<|start|>": 200,
"<|channel|>": 201,
"<|message|>": 202,
"<|end|>": 203,
"<|call|>": 204,
"<|return|>": 205,
"<|constrain|>": 206,
}
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
PROMPT_TOKEN_IDS = [200, 100]
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
FINAL = "<|channel|>final<|message|>"
NEXT_MESSAGE = "<|end|><|start|>assistant"
def _tool(name, properties, *, strict=None):
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": {"type": "object", "properties": properties},
"strict": strict,
},
)
TOOLS = [
_tool("get_weather", {"city": {"type": "string"}}),
_tool("set_count", {"count": {"type": "integer"}}),
]
STRICT_TOOLS = [
_tool("get_weather", {"city": {"type": "string"}}),
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
]
class BeamTokenizer:
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
self.response_template = response_template
self.init_kwargs = {}
self.name_or_path = ""
self.all_special_tokens = list(VOCAB)
self.all_special_ids = list(VOCAB.values())
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
return "".join(
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
for token_id in token_ids
)
def get_vocab(self):
return VOCAB
def encode(self, text, add_special_tokens=False):
del add_special_tokens
return list(text.encode())
def request(**overrides):
values = dict(
tool_choice="auto",
include_reasoning=True,
tools=TOOLS,
parallel_tool_calls=True,
)
values.update(overrides)
return SimpleNamespace(**values)
def parameter(name, value):
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
def call(name, body, *, terminator="<|call|>"):
return (
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
)
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
parser_cls = ParserManager.get_parser(
tool_parser_name="beam",
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
assert parser_cls is not None
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
parser.set_prompt_token_ids(prompt_token_ids)
return parser
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
reasoning, content, calls = make_parser(prompt_token_ids).parse(
text, request(), enable_auto_tools=True
)
return reasoning, content, normalize_calls(calls)
def normalize_calls(calls):
return [(call.name, json.loads(call.arguments)) for call in calls or []]
def test_beam_parsers_are_registered():
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
def test_parser_manager_selects_beam_parser():
parser_cls = ParserManager.get_parser(
tool_parser_name="beam",
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
assert parser_cls is not None
assert issubclass(parser_cls, BeamParser)
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
assert parser_cls.tool_parser_cls is BeamToolParser
assert parser_cls._enable_auto_tools
@pytest.mark.parametrize("other", ["hermes", "hf"])
def test_beam_parser_rejects_other_parsers(other):
with pytest.raises(TypeError, match="cannot be combined"):
ParserManager.get_parser(
tool_parser_name=other,
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
def test_beam_parser_requires_checkpoint_metadata_at_startup():
with pytest.raises(
TypeError, match="--reasoning-parser beam requires `response_template`"
):
ParserManager.get_parser(
reasoning_parser_name="beam",
tokenizer=BeamTokenizer(response_template=None),
)
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
del template["fields"]["tool_calls"]
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
ParserManager.get_parser(
tool_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(template),
)
def test_reasoning_and_final_answer():
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
def test_prefilled_reasoning_continues_from_prompt():
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
assert parse(text, prompt) == (" more.", "Done.", [])
def test_parallel_calls_are_coerced_to_their_schemas():
text = (
THOUGHT
+ call(
"get_weather",
parameter("city", "Paris"),
terminator="<|end|><|start|>assistant",
)
+ call("set_count", parameter("count", "7"))
)
assert parse(text) == (
"plan",
None,
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
)
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
def test_output_cut_before_the_next_message_start_ends_its_region(end):
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
its reasoning or call and lacks only the next message's start."""
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
assert parse(THOUGHT + weather) == (
"plan",
None,
[("get_weather", {"city": "Paris"})],
)
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
FINAL_IDS = [201, 102, 202]
@pytest.mark.parametrize(
("prompt_token_ids", "token_ids", "count"),
[
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
],
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
)
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
parser = make_parser(prompt_token_ids)
assert parser.count_reasoning_tokens(token_ids) == count
def test_reasoning_tokens_are_counted_as_the_generation_grows():
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
parser = make_parser()
assert [
parser.count_reasoning_tokens(token_ids[:end])
for end in range(1, len(token_ids) + 1)
] == [
make_parser().count_reasoning_tokens(token_ids[:end])
for end in range(1, len(token_ids) + 1)
]
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
def test_call_stop_token_ends_the_answer():
assert parse(FINAL + "done<|call|>") == (None, "done", [])
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
def test_streaming_matches_non_streaming(chunk_size):
text = (
THOUGHT
+ call(
"get_weather",
parameter("city", "Paris"),
terminator="<|end|><|start|>assistant",
)
+ call("set_count", parameter("count", "7"))
)
parser = make_parser()
reasoning, calls = [], []
for start in range(0, len(text), chunk_size):
end = min(start + chunk_size, len(text))
delta = parser.parse_delta(
text[start:end], [], request(), finished=end == len(text)
)
if delta is not None:
reasoning.append(delta.reasoning or "")
calls.extend(delta.tool_calls)
assert "".join(reasoning) == "plan"
assert [
(call.function.name, json.loads(call.function.arguments)) for call in calls
] == parse(text)[2]
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
WEATHER = [("get_weather", {"city": "Paris"})]
def chat_request(tools=STRICT_TOOLS, **body):
return ChatCompletionRequest(
model="beam",
messages=[{"role": "user", "content": "Weather?"}],
tools=tools and [tool.model_dump() for tool in tools],
**body,
)
def named(name):
return {"type": "function", "function": {"name": name}}
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
return req
def accepts(req, text):
grammar = Grammar.from_structural_tag(
req.extract_structured_outputs().structural_tag
)
return _is_grammar_accept_string(grammar, text)
@pytest.mark.parametrize(
("tool_choice", "text", "calls"),
[
("required", THOUGHT + WEATHER_CALL, WEATHER),
("required", WEATHER_CALL, None),
(
"required",
THOUGHT
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
+ call("set_count", parameter("count", "7")),
[*WEATHER, ("set_count", {"count": 7})],
),
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
(
"required",
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
None,
),
("required", THOUGHT + FINAL + "Paris<|return|>", None),
(named("set_count"), THOUGHT + WEATHER_CALL, None),
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
(
named("get_weather"),
THOUGHT
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
+ WEATHER_CALL,
None,
),
],
)
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
"""From the message start, the grammar opens with the reasoning, and
accepts only calls the parser reads back."""
req = adjust(chat_request(tool_choice=tool_choice))
assert req._grammar_from_parser
assert accepts(req, text) is (calls is not None)
if calls is not None:
assert parse(text)[2] == calls
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
req = adjust(
chat_request(tool_choice="required"),
chat_template_kwargs={"enable_thinking": False},
)
assert accepts(req, WEATHER_CALL)
assert accepts(req, THOUGHT + WEATHER_CALL)
def test_parallel_tool_calls_false_allows_a_single_call():
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
assert accepts(req, THOUGHT + WEATHER_CALL)
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
@pytest.mark.parametrize("strict", [False, True])
@pytest.mark.parametrize(
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
)
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
"""A model that skips a parameter's opener must not write whitespace until
`max_tokens`."""
write = _tool(
"write",
{"path": {"type": "string"}, "content": {"type": "string"}},
strict=strict,
)
req = adjust(chat_request([write], tool_choice="required"))
body = parameter("path", "a.py") + separator + parameter("content", "x")
assert accepts(req, THOUGHT + call("write", body)) is accepted
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
def test_forced_calls_in_prefilled_reasoning_start_after_it():
req = adjust(
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
)
assert not req._grammar_from_parser
assert accepts(req, WEATHER_CALL)
def test_forced_calls_cannot_follow_a_prefilled_answer():
with pytest.raises(VLLMValidationError, match="required tool choice"):
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
def test_prefilled_answer_can_only_be_finished():
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
assert accepts(req, "Sunny.<|return|>")
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
def test_forced_call_with_unencodable_tool_name_is_rejected():
tools = [_tool("get weather", {"city": {"type": "string"}})]
with pytest.raises(VLLMValidationError, match="tool name"):
adjust(chat_request(tools, tool_choice="required"))
def test_string_formats_and_patterns_are_held_in_raw_text():
"""Like XGrammar, a strict string keeps its format or pattern, unless a
match could end the value early."""
contact = _tool(
"set_contact",
{
"email": {"type": "string", "format": "email"},
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
"n": {"type": "integer"},
},
strict=True,
)
contact.function.parameters["required"] = ["email", "code", "n"]
req = adjust(chat_request([contact], tool_choice="required"))
def contact_call(email, code, *rest):
body = parameter("email", email) + parameter("code", code) + "".join(rest)
return THOUGHT + call("set_contact", body)
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
assert accepts(req, text)
assert normalize_calls(calls) == [
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
]
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
# Line breaks could end the value.
dotted = _tool(
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
)
with pytest.raises(VLLMValidationError, match="required tool choice"):
adjust(chat_request([dotted], tool_choice="required"))
@pytest.mark.parametrize(
("text", "accepted"),
[
(THOUGHT + WEATHER_CALL, True),
(THOUGHT + FINAL + "Sunny.<|return|>", True),
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
(THOUGHT + call("set_count", parameter("count", "x")), False),
],
)
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
assert accepts(adjust(chat_request()), text) is accepted
@pytest.mark.parametrize(
("text", "accepted"),
[
(THOUGHT + FINAL + "Sunny.<|return|>", True),
(FINAL + "Sunny.<|return|>", False),
(WEATHER_CALL, False),
(THOUGHT + WEATHER_CALL, False),
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
],
)
def test_tool_choice_none_allows_only_an_answer(text, accepted):
req = adjust(chat_request(TOOLS, tool_choice="none"))
assert req._grammar_from_parser
assert accepts(req, text) is accepted
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
req = adjust(chat_request(tools, tool_choice=tool_choice))
count_call = THOUGHT + call("set_count", parameter("count", "3"))
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
assert not accepts(req, FINAL + "Sunny.<|return|>")
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
assert not accepts(req, THOUGHT + "Sunny.")
assert accepts(req, count_call) is (tool_choice == "auto")
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
@pytest.mark.parametrize("strict", [None, True])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
def test_automatic_choice_lets_calls_name_undeclared_tools(
strict, tool_choice, response_format
):
"""Unless a forced choice limits calls to the declared tools. The parser
passes such calls on, so the harness can tell the model the tool does not
exist."""
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
req = adjust(
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
)
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
parser = make_parser(tools=[bash])
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
assert accepts(req, undeclared) is (tool_choice == "auto")
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
@pytest.mark.parametrize(
("name", "body", "accepted"),
[
("set_count", parameter("count", "1"), True),
("set_count", parameter("count", "x"), False),
("get_weather", parameter("town", "Paris"), False),
# The grammar cannot enforce `minProperties`.
("find", parameter("x", "1") + parameter("x", "y"), True),
("search_files", parameter("x", "1") + parameter("x", "y"), True),
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
],
)
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
name, body, accepted
):
find = _tool("find", {"q": {"type": "string"}})
find.function.parameters["minProperties"] = 1
req = adjust(chat_request([*TOOLS, find]))
assert accepts(req, THOUGHT + call(name, body)) is accepted
def test_value_ends_before_an_opener_on_the_next_line():
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
req = adjust(chat_request([write], tool_choice="required"))
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
text = THOUGHT + call("write", body)
assert accepts(req, text)
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
assert normalize_calls(calls) == [
("write", {"content": "print(1)\n", "path": "a.py"})
]
def test_always_constrained_output_can_be_disabled(monkeypatch):
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
assert adjust(chat_request(TOOLS)).structured_outputs is None
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
dotted = _tool(
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
)
assert adjust(chat_request([dotted])).structured_outputs is None
def test_generic_parser_does_not_derive_the_template_grammar():
req = chat_request(TOOLS, tool_choice="required")
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
with pytest.raises(VLLMValidationError, match="required tool choice"):
parser.adjust_request(req)
@pytest.mark.parametrize("tools", [None, TOOLS])
def test_structured_output_is_framed_as_the_answer(tools):
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
req = adjust(
chat_request(
tools,
response_format={
"type": "json_schema",
"json_schema": {"name": "answer", "schema": schema},
},
)
)
assert req.response_format is None
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
def test_responses_named_tool_choice_follows_the_template_grammar():
req = adjust(
ResponsesRequest(
input="Weather?",
tools=[
FunctionTool(
type="function",
name="get_weather",
parameters={
"type": "object",
"properties": {"city": {"type": "string"}},
},
strict=True,
)
],
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
text={"format": {"type": "json_object"}},
)
)
assert req._grammar_from_parser
assert req.text.format.type == "json_object"
assert accepts(req, THOUGHT + WEATHER_CALL)
assert not accepts(req, FINAL + "{}<|return|>")
@pytest.mark.parametrize("status", ["in_progress", "completed"])
@pytest.mark.parametrize("inside_reasoning", [False, True])
def test_responses_preserves_text_config_with_template_grammar(
status, inside_reasoning
):
text = {
"format": {
"type": "json_schema",
"name": "answer",
"strict": True,
"schema": {
"type": "object",
"properties": {"a": {"type": "integer"}},
"required": ["a"],
"additionalProperties": False,
},
},
"verbosity": "low",
}
req = ResponsesRequest(input="Answer", text=text)
expected_text = req.text.model_dump(by_alias=True)
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
adjust(req, prompt_token_ids=prompt)
params = req.to_sampling_params(default_max_tokens=128)
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
prefix = "" if inside_reasoning else THOUGHT
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
assert response.model_dump(by_alias=True)["text"] == expected_text
@pytest.mark.parametrize("reverse_alternatives", [False, True])
def test_reasoning_budget_initializes_from_checkpoint_pattern(
monkeypatch, reverse_alternatives
):
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
alternatives = [
r"^<\|channel\|>analysis<\|message\|>",
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
]
if reverse_alternatives:
alternatives.reverse()
template["fields"]["thinking"] = {
"open_pattern": "(?:" + "|".join(alternatives) + ")",
"close": "<|end|>",
"content": "text",
}
tokenizer = BeamTokenizer(template)
monkeypatch.setattr(
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
)
config = ReasoningConfig(reasoning_parser="beam")
config.initialize_token_ids(SimpleNamespace())
assert config.enabled
# The marker must work both after the prompt's assistant header and in
# subsequent complete assistant messages, regardless of regex branch order.
assert config.reasoning_start_token_ids == tokenizer.encode(
"<|channel|>analysis<|message|>"
)
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
thinking = template["fields"]["thinking"]
del thinking["open"]
thinking["open_pattern"] = pattern
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
+179
View File
@@ -9,6 +9,9 @@ import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.parser_manager import ParserManager
from vllm.parser.qwen3 import qwen3_config
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
@@ -417,3 +420,179 @@ def test_parse_auto_tools_no_calls_returns_none(tokenizer):
assert reasoning is None
assert content == PLAIN_TEXT
assert tool_calls is None
# --------------------------------------------------------------------------
# Prompt that already closes the reasoning block
# --------------------------------------------------------------------------
CONTINUE_MESSAGES = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "Sure,"},
]
REASONED_OUTPUT = "<think>\nlet me think\n</think>\n\n" + PLAIN_TEXT
TOOL_TURN_MESSAGES = [
{"role": "user", "content": "Weather in Paris?"},
{
"role": "assistant",
"content": "<think>\nneed a tool\n</think>\n\n",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
def reasoning_parser(tokenizer, name="qwen3"):
parser_cls = ParserManager.get_parser(
reasoning_parser_name=name, tokenizer=tokenizer
)
return parser_cls(tokenizer, None)
def encode(tokenizer, text):
return tokenizer.encode(text, add_special_tokens=False)
def prompt_case(tokenizer, case):
"""(rendered prompt ids, request, model output) for each prompt shape."""
if case == "continue_final_message":
# The template renders an empty, closed think block before "Sure,".
prompt = tokenizer.apply_chat_template(
CONTINUE_MESSAGES,
tokenize=False,
add_generation_prompt=False,
continue_final_message=True,
)
request = make_request(
messages=CONTINUE_MESSAGES,
add_generation_prompt=False,
continue_final_message=True,
)
return encode(tokenizer, prompt), request, " here is the answer"
if case == "template_closed_thinking":
# The template switches thinking off, but the parser is not told
# through chat_template_kwargs (#53284).
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "hi"}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
return encode(tokenizer, prompt), make_request(), PLAIN_TEXT
if case == "earlier_turn_closed_reasoning":
# Only a previous assistant turn closes reasoning; this one is open.
prompt = tokenizer.apply_chat_template(
TOOL_TURN_MESSAGES,
tools=[{"type": "function", "function": {"name": "get_weather"}}],
tokenize=False,
add_generation_prompt=True,
)
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
assert case == "reasoning_open"
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "hi"}],
tokenize=False,
add_generation_prompt=True,
)
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"])
def test_parse_prompt_closed_reasoning_is_content(tokenizer, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
parser = reasoning_parser(tokenizer)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning is None
assert content == output
assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0
@pytest.mark.parametrize(
("parser_name", "case"),
[
("qwen3", "reasoning_open"),
("qwen3", "earlier_turn_closed_reasoning"),
# BaseThinkingReasoningParser scans the prompt without turn
# boundaries, so it must not decide non-streaming classification.
("deepseek_r1", "earlier_turn_closed_reasoning"),
],
)
def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
parser = reasoning_parser(tokenizer, parser_name)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning.strip() == "let me think"
assert content.strip() == PLAIN_TEXT
assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0
def test_parse_prompt_ignored_without_turn_boundaries(tokenizer):
"""Without turn boundaries, an earlier turn's </think> looks like this
turn's, so the engine must not start the output in content."""
prompt_ids, request, output = prompt_case(
tokenizer, "earlier_turn_closed_reasoning"
)
parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True))
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning.strip() == "let me think"
assert content.strip() == PLAIN_TEXT
@pytest.mark.parametrize(
"case",
[
"continue_final_message",
"template_closed_thinking",
"reasoning_open",
"earlier_turn_closed_reasoning",
],
)
def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
output_ids = encode(tokenizer, output)
parser = reasoning_parser(tokenizer)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=output_ids
)
non_streaming = (reasoning or "", content or "")
non_streaming_count = parser.count_reasoning_tokens(output_ids)
parser = reasoning_parser(tokenizer)
streamed_reasoning = streamed_content = ""
for i, token_id in enumerate(output_ids):
delta = parser.parse_delta(
tokenizer.decode([token_id]),
[token_id],
request,
prompt_token_ids=prompt_ids,
finished=i == len(output_ids) - 1,
)
if delta is not None:
streamed_reasoning += delta.reasoning or ""
streamed_content += delta.content or ""
assert (streamed_reasoning, streamed_content) == non_streaming
assert parser.count_reasoning_tokens(output_ids) == non_streaming_count
@@ -0,0 +1,492 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from copy import deepcopy
from typing import Any
import pytest
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.parser.chat_parsing.response_templates import load_response_template
from vllm.parser.response_template_grammar import (
answer_rest_structural_tag,
tool_call_structural_tag,
)
TEMPLATE: dict[str, Any] = {
"start_anchor": "<assistant>",
"fields": {
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
"content": {"content": "text"},
"tool_calls": {
"open_pattern": r'<call name="(?P<name>[\w.-]+)">',
"close": "</call>",
"repeats": True,
"content": "xml-inline",
"content_args": {
"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)</arg>'
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
},
}
ANSWER_TEMPLATE = deepcopy(TEMPLATE)
ANSWER_TEMPLATE["fields"]["content"] = {
"open": "<answer>",
"close": "</answer>",
"content": "text",
}
def tool(parameters, name="f", strict=True) -> dict[str, Any]:
return {"name": name, "parameters": parameters, "strict": strict}
WEATHER = tool(
{
"type": "object",
"properties": {"city": {"type": "string"}, "days": {"type": "integer"}},
"required": ["city"],
},
name="get.weather",
)
PING = {"name": "ping"}
CALL = '<call name="get.weather"><arg name="city">Paris</arg></call>'
def build(
template=TEMPLATE,
functions=(WEATHER, PING),
*,
parallel=True,
thinking=True,
answer=False,
answer_format=None,
sorted_arguments=False,
):
fields = load_response_template(template).fields
return tool_call_structural_tag(
fields["tool_calls"],
list(functions),
parallel_tool_calls=parallel,
thinking=fields["thinking"] if thinking else None,
answer=fields["content"] if answer else None,
answer_format=answer_format,
sorted_arguments=sorted_arguments,
)
def accepts(tag, text):
return _is_grammar_accept_string(Grammar.from_structural_tag(json.dumps(tag)), text)
def call(*arguments):
body = "".join(f'<arg name="{key}">{value}</arg>' for key, value in arguments)
return f'<call name="f">{body}</call>'
@pytest.mark.parametrize(
("text", "accepted"),
[
("<think>plan</think>" + CALL, True),
(CALL, True),
(CALL.removesuffix("</call>"), True),
(
'<call name="get.weather"><arg name="city">Paris</arg>\n'
'<arg name="days">3</arg></call>',
True,
),
(
'<call name="get.weather"><arg name="city">Paris</arg>'
'<arg name="days">soon</arg></call>',
False,
),
('<call name="get.weather"><arg name="days">3</arg></call>', False),
(CALL + '<call name="ping"><arg name="any">x</arg></call>', True),
('<call name="other"></call>', False),
("<think>plan</think>The answer.", False),
],
)
def test_forced_calls_follow_the_template_format(text, accepted):
assert accepts(build(), text) is accepted
def test_single_call_when_parallel_calls_are_disabled():
tag = build(parallel=False)
assert accepts(tag, CALL)
assert not accepts(tag, CALL + CALL)
def test_thinking_is_rejected_when_not_requested():
assert not accepts(build(thinking=False), "<think>plan</think>" + CALL)
@pytest.mark.parametrize(
("text", "accepted"),
[
("<think>plan</think><answer>Sunny.", True),
("<answer>Sunny.</answer>", True),
("<think>plan</think>" + CALL, True),
("<answer>Sunny.</answer>" + CALL, False),
('<call name="get.weather"><arg name="days">soon</arg></call>', False),
],
)
def test_answer_field_lets_the_model_reply_instead(text, accepted):
assert accepts(build(ANSWER_TEMPLATE, answer=True), text) is accepted
@pytest.mark.parametrize("required", [False, True])
@pytest.mark.parametrize(
("text", "optional_only"),
[
("<think>plan</think>" + CALL, False),
("<think>\nplan</think><answer>Sunny.", False),
("<think> \n</think><answer>Sunny.", False),
(CALL, True),
("<answer>Sunny.", True),
("<think></think><answer>Sunny.", True),
("<think><</think><answer>Sunny.", True),
],
)
def test_required_reasoning_opens_the_message_with_nonempty_reasoning(
required, text, optional_only
):
fields = load_response_template(ANSWER_TEMPLATE).fields
tag = tool_call_structural_tag(
fields["tool_calls"],
[WEATHER],
parallel_tool_calls=True,
thinking=fields["thinking"],
answer=fields["content"],
message_start=["<assistant>"],
requires_reasoning=required,
)
assert accepts(tag, text) is (not required or not optional_only)
def test_answer_rest_finishes_the_open_answer():
answer = load_response_template(ANSWER_TEMPLATE).fields["content"]
tag = answer_rest_structural_tag(answer)
assert accepts(tag, "Sunny.</answer>")
assert accepts(tag, "Sunny.")
assert not accepts(tag, "Sunny.</answer>" + CALL)
def test_answer_without_literal_opener_is_not_derived():
assert build(answer=True) is None
def test_answer_holds_the_answer_format():
schema = {"type": "json_schema", "json_schema": {"type": "object"}}
with_tools = build(ANSWER_TEMPLATE, answer=True, answer_format=schema)
answer_only = build(ANSWER_TEMPLATE, (), answer=True, answer_format=schema)
for tag in (with_tools, answer_only):
assert accepts(tag, '<think>plan</think><answer>{"a": 1}</answer>')
assert not accepts(tag, "<answer>Sunny.</answer>")
assert accepts(with_tools, CALL)
assert not accepts(answer_only, CALL)
def test_json_arguments_follow_the_schema():
template = deepcopy(TEMPLATE)
template["fields"]["tool_calls"].update(content="json", content_args={})
tag = build(template, [WEATHER])
assert accepts(tag, '<call name="get.weather">{"city": "Paris"}</call>')
assert not accepts(tag, '<call name="get.weather">{"days": 3}</call>')
END_X = {"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)X'}
@pytest.mark.parametrize(
"change",
[
{"content": "json", "content_args": {"unquoted_keys": True}},
{
"transform": {
"type": "function",
"function": {"name": "fixed", "arguments": "{content}"},
}
},
{"content": "kv-lines", "content_args": {}},
# Framing in which argument text can spell a closer, where the parser
# would end the call.
{"close": ["}X", "</call>"], "content_args": END_X},
{
"close": ["}X", "</call>"],
"content_args": {**END_X, "value_parser": {"name": "json"}},
},
{"close": ["7X", "</call>"], "content_args": END_X},
{"close": [">X", "</call>"]},
{"close": ["></call>", "</call>"]},
{"close": ["><a", "</call>"]},
{"close": ["}</call>", "</call>"], "content": "json", "content_args": {}},
],
)
def test_underivable_tool_formats_are_not_derived(change):
template = deepcopy(TEMPLATE)
template["fields"]["tool_calls"].update(change)
assert build(template) is None
def test_non_literal_thinking_delimiters_are_not_derived():
template = deepcopy(TEMPLATE)
del template["fields"]["thinking"]["open"]
template["fields"]["thinking"]["open_pattern"] = r"<think(?: mode=\w+)?>"
assert build(template) is None
assert build(template, thinking=False) is not None
LEVEL = {"type": "object", "properties": {"level": {"$ref": "#/$defs/level"}}}
@pytest.mark.parametrize(
("schema", "value", "accepted"),
[
({"type": "integer", "nullable": True}, "null", True),
({"type": "integer", "nullable": True}, "soon", False),
({"type": ["integer", "null"]}, "null", True),
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "true", True),
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "soon", False),
({"type": ["string", "integer"]}, "soon", True),
({"enum": ["celsius", "fahrenheit"]}, "celsius", True),
({"enum": ["celsius", "fahrenheit"]}, "kelvin", False),
({"type": "string", "const": "on"}, "off", False),
(LEVEL, '{"level": 2}', True),
(LEVEL, '{"level": "high"}', False),
],
)
def test_argument_values_follow_how_the_parser_reads_them(schema, value, accepted):
parameters = {
"type": "object",
"properties": {"v": schema},
"required": ["v"],
"$defs": {"level": {"type": "integer"}},
}
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
@pytest.mark.parametrize(
("schema", "strict", "derivable"),
[
# The parser reads values without a type as text.
({"enum": [1, 2]}, True, False),
({"$ref": "#/$defs/level"}, True, False),
({"$ref": "#/$defs/level"}, False, True),
# Raw text cannot hold its closer.
({"enum": ["a</arg>b"]}, True, False),
({"type": "object", "properties": {"x": {"$ref": "#/x"}}}, True, False),
(False, True, False),
# A match could write the start of the value's end.
({"type": "string", "pattern": "^[a-z<]+$"}, True, False),
({"type": "string", "pattern": "^[a-z<]+$"}, False, True),
({"type": "string", "pattern": "^a.b$"}, True, False),
# XGrammar holds length bounds only in JSON.
({"type": "string", "minLength": 3}, True, False),
({"type": "string", "format": "ipv6"}, True, False),
],
)
def test_schemas_text_values_cannot_hold_are_not_derived(schema, strict, derivable):
parameters = {
"type": "object",
"properties": {"v": schema},
"$defs": {"level": {"type": "integer"}},
}
assert (build(functions=[tool(parameters, strict=strict)]) is not None) is derivable
@pytest.mark.parametrize(
("schema", "value", "accepted"),
[
({"type": "string", "format": "email"}, "a.b@x.co", True),
({"type": "string", "format": "email"}, "ab", False),
({"type": "string", "format": "date-time"}, "2026-10-06T12:00:00Z", True),
({"type": "string", "format": "date-time"}, "2026-13-06T12:00:00Z", False),
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABC", True),
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABCD", False),
# Like XGrammar, the format overrides the pattern and the length bounds.
(
{"type": "string", "format": "uuid", "pattern": "^x$", "maxLength": 1},
"123e4567-e89b-12d3-a456-426614174000",
True,
),
],
)
def test_text_values_hold_string_formats_and_patterns(schema, value, accepted):
parameters = {"type": "object", "properties": {"v": schema}, "required": ["v"]}
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
def test_values_cannot_hold_a_closer():
template = deepcopy(TEMPLATE)
template["fields"]["tool_calls"]["close"] = ["</call>", "STOP"]
text = {"type": "object", "properties": {"v": {"type": "string"}}}
strings = {"type": "array", "items": {"type": "string"}}
json_text = {"type": "object", "properties": {"v": strings}}
assert accepts(build(template, [tool(text)]), call(("v", "ab")))
assert not accepts(build(template, [tool(text)]), call(("v", "aSTOPb")))
assert not accepts(
build(template, [tool(None, strict=False)]), call(("v", "aSTOPb"))
)
assert accepts(build(template, [tool(json_text)]), call(("v", '["ab"]')))
assert not accepts(build(template, [tool(json_text)]), call(("v", '["aSTOPb"]')))
def test_extra_arguments_need_additional_properties():
parameters = {"type": "object", "properties": {"city": {"type": "string"}}}
extra = call(("city", "Paris"), ("zip", "75001"))
assert not accepts(build(functions=[tool(parameters)]), extra)
# Next to declared keys, an extra one could repeat them unchecked.
allowed = {**parameters, "additionalProperties": True}
assert build(functions=[tool(allowed)]) is None
free_form = {"type": "object", "additionalProperties": True}
assert accepts(build(functions=[tool(free_form)]), extra)
def test_only_strict_tools_follow_their_schema():
off_schema = call(("days", "soon"))
parameters = WEATHER["parameters"]
assert accepts(build(functions=[tool(parameters, strict=False)]), off_schema)
assert not accepts(build(functions=[tool(parameters)]), off_schema)
assert accepts(build(functions=[tool(None)]), call())
assert not accepts(build(functions=[tool(None)]), call(("a", "x")))
def test_annotations_keep_strict_arguments_to_their_schema():
parameters = {
**WEATHER["parameters"],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "weather",
}
tag = build(functions=[tool(parameters)])
assert accepts(tag, call(("city", "Paris"), ("days", "3")))
assert not accepts(tag, call(("city", "Paris"), ("days", "soon")))
@pytest.mark.parametrize(
("arguments", "accepted", "accepted_sorted"),
[
((("path", "a"), ("content", "b")), True, True),
((("content", "b"), ("path", "a")), False, True),
((("path", "a"),), True, True),
((("content", "b"),), False, False),
((("path", "a"), ("path", "c")), False, False),
((("path", "a"), ("mode", "c")), False, False),
],
)
def test_strict_arguments_follow_declared_or_sorted_order(
arguments, accepted, accepted_sorted
):
parameters = {
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
"required": ["path"],
}
tags = [
build(functions=[tool(parameters)], sorted_arguments=sorted_arguments)
for sorted_arguments in (False, True)
]
assert [accepts(tag, call(*arguments)) for tag in tags] == [
accepted,
accepted_sorted,
]
LOOKAHEAD_TEMPLATE = deepcopy(TEMPLATE)
LOOKAHEAD_TEMPLATE["fields"]["tool_calls"]["content_args"]["tag_pattern"] = (
r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))'
)
@pytest.mark.parametrize(
("body", "accepted_loose", "accepted_strict"),
[
('<arg name="city">Paris\n</arg>\n<arg name="days">3\n</arg>', True, True),
('<arg name="city">Paris\n<arg name="days">3\n</arg>', True, True),
('<arg name="days">3\n<arg name="city">Paris\n</arg>', True, False),
('<arg name="city">Paris\n<arg name="city">Lyon\n</arg>', True, False),
('<arg name="city">Paris\n<arg name="hours">3\n</arg>', True, False),
('<arg name="city">Paris\n<arg name="', False, False),
('<arg name="city">Paris\n<arg name="days\n</arg>', False, False),
('<arg name="city">a <arg name="b"> c\n</arg>', True, True),
],
)
def test_values_end_where_a_lookahead_end_does(body, accepted_loose, accepted_strict):
parameters = WEATHER["parameters"]
tags = [
build(LOOKAHEAD_TEMPLATE, [tool(parameters, strict=strict)])
for strict in (False, True)
]
text = f'<call name="f">{body}</call>'
assert [accepts(tag, text) for tag in tags] == [accepted_loose, accepted_strict]
@pytest.mark.parametrize("lookahead", [False, True])
@pytest.mark.parametrize(
("keys", "accepted"),
[
("b", True),
("ab", True),
("bc", True),
("bd", True),
("be", True),
("abde", True),
("bcde", True),
("", False),
("a", False),
("acd", False),
("bdc", False),
("bcc", False),
],
)
def test_last_parameter_follows_every_required_one(keys, accepted, lookahead):
parameters = {
"properties": {key: {"type": "string"} for key in "abcde"},
"required": ["b"],
}
tag = build(LOOKAHEAD_TEMPLATE, [tool(parameters)])
ends = ["\n" if lookahead else "\n</arg>"] * (len(keys) - 1) + ["\n</arg>"]
body = "".join(f'<arg name="{k}">x{e}' for k, e in zip(keys, ends))
assert accepts(tag, f'<call name="f">{body}</call>') is accepted
def test_required_argument_without_schema_is_still_required():
parameters = {"type": "object", "properties": {}, "required": ["a"]}
tag = build(functions=[tool(parameters)])
assert accepts(tag, call(("a", "x")))
assert not accepts(tag, call())
assert build(functions=[tool({**parameters, "required": ["a", 1]})]) is None
@pytest.mark.parametrize(
("function", "match"),
[
(tool({}, name="get weather"), "tool name"),
(tool({"properties": {'a"b': {"type": "string"}}}), "parameter name"),
],
)
def test_unencodable_names_are_rejected(function, match):
with pytest.raises(ValueError, match=match):
build(functions=[function])
+27
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
from collections.abc import Sequence
from dataclasses import dataclass
@@ -376,6 +377,32 @@ class TestRenderPrompt:
assert results[0]["prompt_token_ids"] == tokens
assert results[0]["prompt"] == "[1, 2, 3, 4]"
@pytest.mark.parametrize(
("truncation_side", "expected"),
[("left", [106, 107, 108, 109]), ("right", [100, 101, 102, 103])],
)
def test_detokenization_matches_truncated_tokens(self, truncation_side, expected):
"""Echoed prompt text must match the truncated tokens sent to the engine."""
renderer = _build_renderer(MockModelConfig())
params = TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=4,
truncation_side=truncation_side,
needs_detokenization=True,
)
def render():
return renderer.render_prompts(
_preprocess_prompt(renderer.model_config, list(range(100, 110)))
)
results = renderer.tokenize_prompts(render(), params)
async_results = asyncio.run(renderer.tokenize_prompts_async(render(), params))
for result in (*results, *async_results):
assert result["prompt_token_ids"] == expected
assert result["prompt"] == str(expected)
def test_explicit_side_tokenizer_unbounded(self):
renderer = _build_renderer(MockModelConfig())
+8
View File
@@ -48,6 +48,14 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
assert "VLLM_API_KEY" not in envs.compile_factors()
def test_ray_bundle_indices_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
# Engines sharing a placement group get different bundle indices but
# compile the same graphs, so they must share a cache key.
monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", "4,5,6,7")
assert "VLLM_RAY_BUNDLE_INDICES" not in envs.compile_factors()
def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False)
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False)
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
block_table.stride(0),
block_table.stride(1),
cu_num_tokens,
cu_num_tokens[1:] - cu_num_tokens[:-1],
KERNEL_BLOCK_SIZE=kernel_block_size,
BLOCK_SIZE=1024,
)
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
bt.stride(0),
bt.stride(1),
cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024,
)
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
bt.stride(0),
bt.stride(1),
cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024,
)
+144 -7
View File
@@ -10,6 +10,7 @@ Known Issues:
import sys
from types import MethodType, SimpleNamespace
from typing import Literal
import pytest
import torch
@@ -47,6 +48,9 @@ from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
from vllm.v1.attention.backends.mla.prefill.selector import (
MLAPrefillSelectorConfig,
)
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
is_cute_dsl_mla_prefill_supported,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
from vllm.v1.kv_cache_interface import (
@@ -418,6 +422,11 @@ for backend in BACKENDS_TO_TEST:
if query_len_support != QueryLenSupport.SINGLE_ONLY:
SPEC_DECODE_BACKENDS.append(backend)
# Backends whose decode pipeline replays device-trimmed verify batches.
ADAPTIVE_VARLEN_BACKENDS = [
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
]
BACKEND_BLOCK_SIZES = {}
for backend in BACKENDS_TO_TEST:
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
@@ -470,6 +479,10 @@ BATCH_SPECS = {
"spec_decode_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
),
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
"spec_decode_ragged_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
),
"spec_decode_medium": BatchSpec(
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
),
@@ -1474,6 +1487,7 @@ def run_attention_backend(
# Force the prefill backend selection (None means auto-select).
vllm_config.attention_config.mla_prefill_backend = prefill_backend
mla_attention_module.backend_supports_prefill_query_quantization.cache_clear()
# Set the current vllm config so that get_current_vllm_config() works
# in the backend implementations
@@ -1581,6 +1595,42 @@ def run_attention_backend(
return output
def _with_even_cpu_query_split(
common_attn_metadata: CommonAttentionMetadata,
) -> CommonAttentionMetadata:
"""Host query lengths as adaptive verification leaves them.
The device keeps the trimmed per-request lengths; the host sees the same
total spread evenly, the first requests taking the remainder
(AdaptiveVerificationManager.compact_batch).
"""
num_reqs = common_attn_metadata.num_reqs
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
cpu_lens[: total % num_reqs] += 1
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
return common_attn_metadata.replace(
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=int(
(
common_attn_metadata.query_start_loc[1:]
- common_attn_metadata.query_start_loc[:-1]
)
.max()
.item()
),
)
def _fp8_cache_roundtrip(tensor: torch.Tensor, scale: float) -> torch.Tensor:
"""Reproduce cache storage rounding for the SDPA reference."""
fp8_dtype = current_platform.fp8_dtype()
limits = torch.finfo(fp8_dtype)
quantized = (tensor.float() / scale).clamp(limits.min, limits.max).to(fp8_dtype)
return (quantized.float() * scale).to(tensor.dtype)
def _run_backend_correctness(
default_vllm_config,
dist_init,
@@ -1595,6 +1645,9 @@ def _run_backend_correctness(
qk_nope_head_dim: int,
v_head_dim: int,
chunked_prefill_workspace_size: int | None = None,
adaptive_verification: bool = False,
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"] = "trtllm-gen",
use_prefill_query_quantization: bool = False,
):
"""Test that all backends produce similar outputs to a reference implementation
using torch.nn.functional.scaled_dot_product_attention.
@@ -1626,6 +1679,10 @@ def _run_backend_correctness(
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
# CUTLASS_MLA does not support non-1 Q/K scales
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
if adaptive_verification:
backends_to_test = [
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
]
if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
@@ -1670,18 +1727,25 @@ def _run_backend_correctness(
hf_config_override["num_attention_heads"]
)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.attention_config.trtllm_ragged_backend = trtllm_ragged_backend
vllm_config.attention_config.use_prefill_query_quantization = (
use_prefill_query_quantization
)
# For spec decode tests, add a speculative_config to set the reorder_batch_threshold
if is_spec_decode_test:
from vllm.config import SpeculativeConfig
# Get the query length from the batch spec (they should all be uniform)
query_len = batch_spec.query_lens[0]
# Get the query length from the batch spec (they should all be uniform,
# except under adaptive verification, where the longest is k+1)
query_len = max(batch_spec.query_lens)
# Set num_speculative_tokens to query_len - 1
# (since threshold is 1 + num_spec_tokens)
# Use ngram method which doesn't require a draft model
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=query_len - 1
method="ngram",
num_speculative_tokens=query_len - 1,
enable_adaptive_verification=adaptive_verification,
)
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -1756,6 +1820,19 @@ def _run_backend_correctness(
# K_PE (rope component): [s_len, 1, qk_rope_head_dim]
k_pe_full = torch.randn(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
kv_c_cached, k_pe_cached = kv_c_full, k_pe_full
kv_c_prefill, k_pe_prefill = kv_c_full, k_pe_full
if kv_cache_dtype.startswith("fp8"):
kv_c_cached = _fp8_cache_roundtrip(kv_c_full, k_scale)
k_pe_cached = _fp8_cache_roundtrip(k_pe_full, k_scale)
# Prefill reads cached context and unquantized new tokens.
kv_c_prefill = torch.cat(
[kv_c_cached[:context_len], kv_c_full[context_len:]]
)
k_pe_prefill = torch.cat(
[k_pe_cached[:context_len], k_pe_full[context_len:]]
)
# Determine if this sequence uses the decode pipeline or prefill
# pipeline for each backend
# NOTE: For spec decode tests with uniform query_len > 1, backends that
@@ -1805,10 +1882,10 @@ def _run_backend_correctness(
q_mqa = torch.cat([ql_nope, q_pe], dim=-1)
# K: [s_len, kv_lora_rank + qk_rope_head_dim]
# (broadcasted to all heads)
k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1)
k_mqa = torch.cat([kv_c_cached, k_pe_cached.squeeze(1)], dim=-1)
k_mqa = k_mqa.unsqueeze(1).expand(-1, num_q_heads, -1)
# V: [s_len, kv_lora_rank] (broadcasted to all heads)
v_mqa = kv_c_full.unsqueeze(1).expand(-1, num_q_heads, -1)
v_mqa = kv_c_cached.unsqueeze(1).expand(-1, num_q_heads, -1)
# Create custom attention mask for decode path:
# - Query tokens can attend to all context tokens
@@ -1837,13 +1914,18 @@ def _run_backend_correctness(
#######################################################
# Prefill path: MHA-style attention with full sequence
# Apply kv_b_proj to the full kv_c tensor
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_full, kv_b_proj_weight)
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_prefill, kv_b_proj_weight)
k_nope_full, v_full = kv_nope_full.split([qk_nope_head_dim, v_head_dim], dim=-1)
# Build attention inputs for full sequence
q_mha = torch.cat([q_nope, q_pe], dim=-1) # [q_len, num_heads, total_dim]
k_pe_full_expanded = k_pe_full.expand(-1, num_q_heads, -1)
k_pe_full_expanded = k_pe_prefill.expand(-1, num_q_heads, -1)
k_full = torch.cat([k_nope_full, k_pe_full_expanded], dim=-1)
if use_prefill_query_quantization:
fp8_dtype = current_platform.fp8_dtype()
q_mha = q_mha.to(fp8_dtype).to(dtype)
k_full = k_full.to(fp8_dtype).to(dtype)
v_full = v_full.to(fp8_dtype).to(dtype)
# Create custom attention mask:
# - Query tokens can attend to all context tokens
@@ -1951,6 +2033,13 @@ def _run_backend_correctness(
)
kv_cache_per_block_size[block_size] = kv_cache
if adaptive_verification:
# Only after the cache is filled: prepopulation reads the context
# lengths off the host query lengths.
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
common_attn_metadata
)
# 4. Run vLLM backends and compare
rtol = 1e-2
atol = {
@@ -2111,6 +2200,8 @@ def test_backend_correctness(
_prefill_backend_dimension_params(),
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("trtllm_ragged_backend", ["trtllm-gen", "cute-dsl"])
@pytest.mark.parametrize("use_prefill_query_quantization", [False, True])
def test_chunked_context_backend_correctness(
default_vllm_config,
dist_init,
@@ -2119,8 +2210,26 @@ def test_chunked_context_backend_correctness(
qk_nope_head_dim: int,
v_head_dim: int,
kv_cache_dtype: str,
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"],
use_prefill_query_quantization: bool,
):
"""Split, packed, and context-free requests match the SDPA reference."""
if prefill_backend != MLAPrefillBackendEnum.TRTLLM_RAGGED and (
trtllm_ragged_backend == "cute-dsl" or use_prefill_query_quantization
):
pytest.skip("TRTLLM ragged-specific test configuration")
if use_prefill_query_quantization and kv_cache_dtype == "auto":
pytest.skip("FP8 prefill queries require an FP8 KV cache")
if trtllm_ragged_backend == "cute-dsl":
query_dtype = (
current_platform.fp8_dtype()
if use_prefill_query_quantization
else torch.bfloat16
)
if not is_cute_dsl_mla_prefill_supported(
qk_nope_head_dim + 64, v_head_dim, query_dtype
):
pytest.skip("Unsupported CuTe DSL prefill dimensions or query dtype")
_run_backend_correctness(
default_vllm_config,
dist_init,
@@ -2135,4 +2244,32 @@ def test_chunked_context_backend_correctness(
qk_nope_head_dim=qk_nope_head_dim,
v_head_dim=v_head_dim,
chunked_prefill_workspace_size=1024,
trtllm_ragged_backend=trtllm_ragged_backend,
use_prefill_query_quantization=use_prefill_query_quantization,
)
@pytest.mark.skipif(
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
)
def test_adaptive_varlen_decode_correctness(
default_vllm_config, dist_init, workspace_init
):
"""Device query lengths 1..k+1 against an even host split, as adaptive
verification hands them to the target model's attention. TP 8 gives 16
heads, the padded Kimi-K3 count."""
_run_backend_correctness(
default_vllm_config,
dist_init,
workspace_init,
"spec_decode_ragged_small",
"deepseek-ai/DeepSeek-R1",
8,
"fp8",
1.0,
1.0,
None,
128,
128,
adaptive_verification=True,
)
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
stride_0,
stride_1,
paged_kv_indptr,
num_real_tokens,
*,
KERNEL_BLOCK_SIZE,
BLOCK_SIZE,
CLAMP_TAIL=False,
):
# Both block-table strides are passed explicitly: the kernel no longer
# assumes the column stride is 1. This stand-in indexes the tensor
@@ -145,6 +147,7 @@ def _builder(
)
),
_mtp_decode_qlen=mtp_decode_qlen,
_varlen_decode_qlen=0,
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
impl.qk_rope_head_dim = 64
impl.scale = head_dim**-0.5
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=1,
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
)
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=qlen,
paged_kv_indptr=torch.tensor(
[0, local_kv.shape[0]], dtype=torch.int32, device=device
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
)
assert metadata.has_persistent_metadata
def _adaptive_config(
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
) -> SimpleNamespace:
return SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
enable_adaptive_verification=adaptive,
parallel_drafting=False,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
use_v2_model_runner=True,
)
def test_varlen_cudagraph_bound_follows_adaptive_verification():
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_adaptive_config(), SimpleNamespace()) == 8
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
# DCP routes plan per-row windows from host lengths.
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
# Kimi-K3 DSpark draft layers share the target's group; causality is
# decided per batch, so the merged non-causal spec keeps the bound.
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
assert bound(_adaptive_config(), shared_draft_group) == 8
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for KVCacheConfigBuilder resolution."""
from unittest.mock import MagicMock, patch
import pytest
from vllm.platforms import Platform
from vllm.v1.core.kv_cache_config_builder import (
KVCacheConfigBuilder,
get_kv_cache_config_builder,
)
from vllm.v1.core.kv_cache_planning import (
DefaultKVCacheConfigBuilder,
get_kv_cache_groups_from_builder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
def _make_vllm_config(supports_grouping: bool = False) -> MagicMock:
"""Create a minimal mock VllmConfig for builder resolution tests.
``supports_grouping`` selects whether the resolved model class declares
``SupportsKVCacheGrouping`` (exposing a ``get_kv_cache_groups`` classmethod).
"""
cfg = MagicMock()
cfg.model_config.supports_kv_cache_grouping = supports_grouping
cfg.attention_config.hisparse_config = None
if supports_grouping:
cfg.model_config.registry.resolve_model_cls.return_value = (
_GroupedModel,
"arch",
)
return cfg
CUSTOM_PATH = "tests.v1.core.test_kv_cache_config_builder.CustomBuilder"
DEFAULT_PATH = "vllm.v1.core.kv_cache_planning.DefaultKVCacheConfigBuilder"
class CustomBuilder(DefaultKVCacheConfigBuilder):
"""A test builder subclass."""
pass
class _GroupedModel:
"""A test model class with a ``get_kv_cache_groups`` classmethod."""
@classmethod
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
return []
class _PartialGroupedModel:
"""A test model class whose grouping omits layers."""
@classmethod
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
return [KVCacheGroupSpec(["a"], kv_cache_spec["a"])]
class TestPlatformHookResolution:
"""The platform hook selects only the placement builder."""
def test_default_hook_returns_default_builder(self):
cfg = _make_vllm_config()
assert Platform.get_kv_cache_config_builder_cls(cfg) == DEFAULT_PATH
class TestBuilderResolution:
@patch("vllm.platforms.current_platform")
@pytest.mark.parametrize(
("builder_cls_path", "expected_cls"),
[
(None, DefaultKVCacheConfigBuilder),
(CUSTOM_PATH, CustomBuilder),
],
)
def test_resolves_hook_selected_builder(
self, mock_platform, builder_cls_path, expected_cls
):
mock_platform.get_kv_cache_config_builder_cls.return_value = (
builder_cls_path or DEFAULT_PATH
)
cfg = _make_vllm_config()
assert type(get_kv_cache_config_builder(cfg)) is expected_cls
@patch("vllm.platforms.current_platform")
def test_resolves_fresh_instance_per_call(self, mock_platform):
mock_platform.get_kv_cache_config_builder_cls.return_value = CUSTOM_PATH
cfg = _make_vllm_config()
assert get_kv_cache_config_builder(cfg) is not get_kv_cache_config_builder(cfg)
def test_public_interface_declares_two_placement_hooks(self):
# get_kv_cache_configs lives in kv_cache_planning and is Core-owned;
# the builder only declares the placement hooks.
assert KVCacheConfigBuilder.__abstractmethods__ == {
"get_kv_cache_groups",
"get_kv_cache_config_from_groups",
}
assert issubclass(DefaultKVCacheConfigBuilder, KVCacheConfigBuilder)
assert (
"pool_bytes_per_block"
not in DefaultKVCacheConfigBuilder.__abstractmethods__
)
assert (
"max_memory_usage_bytes"
not in DefaultKVCacheConfigBuilder.__abstractmethods__
)
class TestModelGroupingPriority:
"""A model class with a get_kv_cache_groups classmethod owns grouping."""
def test_model_classmethod_owns_grouping(self):
cfg = _make_vllm_config(supports_grouping=True)
assert DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {}) == []
def test_model_grouping_rejects_hisparse(self):
cfg = _make_vllm_config(supports_grouping=True)
cfg.attention_config.hisparse_config = object()
with pytest.raises(ValueError, match="HiSparse"):
DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {})
class TestCoreChecksGrouping:
"""Core checks the grouping whether a model or a platform produced it."""
_SPEC = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
)
def test_model_grouping_is_checked(self):
cfg = _make_vllm_config(supports_grouping=True)
cfg.model_config.registry.resolve_model_cls.return_value = (
_PartialGroupedModel,
"arch",
)
with pytest.raises(AssertionError, match="miss layers"):
get_kv_cache_groups_from_builder(
DefaultKVCacheConfigBuilder(), cfg, {"a": self._SPEC, "b": self._SPEC}
)
@pytest.mark.parametrize(
("groups", "match"),
[
([KVCacheGroupSpec(["a"], _SPEC)], "miss layers"),
(
[KVCacheGroupSpec(["a", "b"], _SPEC), KVCacheGroupSpec(["b"], _SPEC)],
"overlaps",
),
],
)
def test_platform_grouping_is_checked(self, groups, match):
class FixedGroupsBuilder(DefaultKVCacheConfigBuilder):
def get_kv_cache_groups(self, vllm_config, kv_cache_spec):
return groups
with pytest.raises(AssertionError, match=match):
get_kv_cache_groups_from_builder(
FixedGroupsBuilder(),
_make_vllm_config(),
{"a": self._SPEC, "b": self._SPEC},
)
class TestPlatformCustomPriority:
"""A vendor platform can override the hook to customize priority."""
def test_platform_builder_wins_over_model_declaration(self):
class PlatformFirstPlatform(Platform):
@classmethod
def get_kv_cache_config_builder_cls(cls, vllm_config):
return CUSTOM_PATH
cfg = _make_vllm_config()
assert PlatformFirstPlatform.get_kv_cache_config_builder_cls(cfg) == CUSTOM_PATH
with patch("vllm.platforms.current_platform", PlatformFirstPlatform):
assert isinstance(get_kv_cache_config_builder(cfg), CustomBuilder)
class TestProjectionPreservesGroupMetadata:
"""PP projection must carry host_resident/role/enable_kv_transfer."""
def test_partial_projection_preserves_metadata(self):
from vllm.v1.core.kv_cache_planning import _project_kv_cache_groups_to_worker
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupRole,
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
)
group = MagicMock()
group.layer_names = ["layer1", "layer2", "layer3"]
group.kv_cache_spec = spec
group.is_eagle_group = False
group.host_resident = True
group.enable_kv_transfer = False
group.role = KVCacheGroupRole.HISPARSE_SOURCE
projected = _project_kv_cache_groups_to_worker(
[group], {"layer1": spec, "layer2": spec}
)
assert len(projected) == 1
p = projected[0]
assert p.host_resident is True
assert p.enable_kv_transfer is False
assert p.role is KVCacheGroupRole.HISPARSE_SOURCE
assert p.layer_names == ["layer1", "layer2"]
class TestBytesPerBlockProbe:
"""The single-block probe must not be polluted by num_gpu_blocks_override."""
def test_probe_suppresses_override(self):
import torch
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
from vllm.v1.core.kv_cache_planning import (
DefaultKVCacheConfigBuilder,
_pool_bytes_per_block,
_pool_bytes_per_block_from_builder,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.float16
)
vc = VllmConfig(
model_config=ModelConfig(max_model_len=100),
cache_config=CacheConfig(block_size=16),
)
resolve_kv_cache_layout(vc, [["BLHNC"]], [spec])
groups = [
KVCacheGroupSpec(["a"], spec),
KVCacheGroupSpec(["b"], spec),
]
pure_c = _pool_bytes_per_block(vc, groups)
vc.cache_config.num_gpu_blocks_override = 500
probed_c = _pool_bytes_per_block_from_builder(
DefaultKVCacheConfigBuilder(), vc, groups
)
assert probed_c == pure_c
assert vc.cache_config.num_gpu_blocks_override == 500
@pytest.mark.parametrize("declares_pool_bytes", [False, True])
def test_several_device_allocations_need_declared_pool_bytes(
self, declares_pool_bytes
):
"""A placement with one allocation per group cannot be sized from one
backing; the builder must declare its per-block bytes."""
from vllm.v1.core.kv_cache_planning import _pool_bytes_per_block_from_builder
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
class PerGroupAllocationBuilder(DefaultKVCacheConfigBuilder):
def get_kv_cache_config_from_groups(self, vllm_config, groups, num_blocks):
tensors = [
KVCacheTensor(
size=page * num_blocks,
layers=group.layer_names,
layer_stride=page * num_blocks,
block_stride=page,
)
for group, page in zip(groups, (64, 32))
]
return KVCacheConfig(num_blocks, tensors, groups)
def _get_pool_bytes_per_block(self, vllm_config, groups):
return 64 + 32 if declares_pool_bytes else None
vc = _make_vllm_config()
vc.cache_config.num_gpu_blocks_override = None
groups = [KVCacheGroupSpec([name], MagicMock()) for name in "ab"]
builder = PerGroupAllocationBuilder()
if declares_pool_bytes:
assert _pool_bytes_per_block_from_builder(builder, vc, groups) == 96
else:
with pytest.raises(ValueError, match="_get_pool_bytes_per_block"):
_pool_bytes_per_block_from_builder(builder, vc, groups)
+53 -33
View File
@@ -18,6 +18,7 @@ from vllm.inputs import PromptType
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import EngineCoreRequestType
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.core_client import DPLBAsyncMPClient
from vllm.v1.metrics.loggers import StatLoggerBase
@@ -336,12 +337,12 @@ async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
@pytest.mark.asyncio
async def test_dp_pause_late_request_does_not_block_drain():
"""A request arriving after pause must not leave the coordinator believing
"""A request arriving after pause must not make the coordinator believe
the engines are running.
Paused engines discard START_DP_WAVE, so nothing can report wave
completion afterwards; if forwarding the wake also marks the engines as
running, drain has no way back and can only end in a timeout.
Paused engines start no wave for it, so nothing could report wave
completion afterwards; a running flag set on its behalf would leave drain
no way back but a timeout.
MoE only: wave coordination is enabled iff the model is MoE, so a dense
model never reaches the coordinator state this exercises.
@@ -365,11 +366,9 @@ async def test_dp_pause_late_request_does_not_block_drain():
# A design that never reports the engines as running would satisfy
# every drain assertion below while destroying the signal, so pin it
# down first. The front-end sets its own copy optimistically when it
# forwards the wake, so sample only after several coordinator
# publishes (every 100ms while stats change) have overwritten it.
# Rank 0 reports the wave only after its first step, which can outlast
# the 2s on a cold device, so wait for the report rather than sample it.
# down first. Rank 0 reports the wave only after its first step, which
# can outlast the 2s on a cold device, so wait for the report rather
# than sample it.
await asyncio.sleep(2)
assert not long_request.done(), "the warmup request was too short to sample"
assert await _poll_flag(engine, True, timeout=30), (
@@ -382,24 +381,12 @@ async def test_dp_pause_late_request_does_not_block_drain():
await engine.pause_generation(mode="abort")
await engine.wait_for_requests_to_drain(drain_timeout=30)
# Awaiting add_request guarantees the new-request notification has been
# sent to the coordinator - the message that used to latch the flag.
collector = await engine.add_request(
request_id="late",
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=5),
)
# The front-end marks the engines running off the back of that
# notification. This is what makes the test non-vacuous: it is the
# path that used to leave the coordinator stuck.
assert await _poll_flag(engine, True, timeout=5), (
"the late request did not notify the coordinator"
)
# It must settle back by itself. Unfixed it never does, because the
# paused engines discard the wake and so never report wave completion.
assert await _poll_flag(engine, False, timeout=60)
assert not await _poll_flag(engine, True, timeout=5)
# The late request was held rather than dropped: it completes on resume.
await engine.resume_generation()
@@ -409,6 +396,46 @@ async def test_dp_pause_late_request_does_not_block_drain():
break
@pytest.mark.asyncio
async def test_dp_request_reaching_rank_after_resume_wakes_peers():
"""A request can reach its rank only after resume (e.g. from another API
server), with no other work anywhere. That rank must wake its peers itself,
or it steps alone. Modelled here by holding the request's ADD on the client
side across pause and resume."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(
_get_dp_pause_engine_args(expert_parallel=True)
)
after.callback(engine.shutdown)
client = engine.engine_core
params = SamplingParams(max_tokens=5)
await _consume(engine.generate(DP_PAUSE_PROMPT, params, "warmup"))
await engine.pause_generation(mode="abort")
assert await _poll_flag(engine, False, timeout=30)
held: list[tuple] = []
send_input = client._send_input
def hold_add(request_type, request, engine_id=None):
if request_type == EngineCoreRequestType.ADD:
held.append((request_type, request, engine_id))
return asyncio.sleep(0)
return send_input(request_type, request, engine_id)
client._send_input = hold_add
task = asyncio.create_task(
_consume(engine.generate(DP_PAUSE_PROMPT, params, "late"))
)
await asyncio.sleep(2)
assert held
await engine.resume_generation()
client._send_input = send_input
await send_input(*held[0])
await asyncio.wait_for(task, timeout=60)
@pytest.mark.asyncio
async def test_dp_sleep_late_request_does_not_block_drain():
"""The same latch, reached through sleep rather than pause.
@@ -441,14 +468,7 @@ async def test_dp_sleep_late_request_does_not_block_drain():
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=5),
)
assert await _poll_flag(engine, True, timeout=5), (
"the request did not notify the coordinator"
)
# Sleeping engines cannot report wave completion, so a coordinator that
# marked them running when it forwarded the wake never hears otherwise.
assert await _poll_flag(engine, False, timeout=60)
assert not await _poll_flag(engine, True, timeout=5)
await engine.wake_up()
assert not await engine.is_sleeping()
@@ -628,7 +648,7 @@ async def test_dp_pause_keep_race_staggered_engines():
@pytest.mark.asyncio
async def test_dp_pause_barrier_request_deadlock():
"""Test that start_dp_wave is ignored while paused.
"""Test that a request sent while paused does not wake any engine.
Sequence:
1. Pause all engines (PAUSED_ALL).
@@ -697,7 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
# Yield so generate() preprocessing completes and
# add_request_async is called (which, in buggy code,
# would send FIRST_REQ and wake engine 1).
# would wake engine 1).
for _ in range(200):
await asyncio.sleep(0)
@@ -725,7 +745,7 @@ async def test_dp_pause_barrier_request_deadlock():
for t in mid_barrier_tasks:
t.cancel()
pytest.fail(
"Staggered barrier deadlocked — FIRST_REQ sent while "
"Staggered barrier deadlocked — a request sent while "
"paused caused collective-op mismatch between engines"
)
@@ -12,6 +12,7 @@ from typing import Any
import pytest
import torch
import vllm.envs as envs
from tests.conftest import HfRunner, VllmRunner
from tests.utils import (
large_gpu_mark,
@@ -168,9 +169,16 @@ def test_with_eagle3_spec_decoding(
reason=("XPU matmul/attention kernels are not batch-invariant"),
)
def test_with_ngram_gpu_spec_decoding(
vllm_runner: type[VllmRunner], hf_runner: type[HfRunner]
vllm_runner: type[VllmRunner],
hf_runner: type[HfRunner],
monkeypatch: pytest.MonkeyPatch,
):
"""Check ngram accuracy and acceptance across scheduling configurations."""
if current_platform.is_rocm():
# hipBLASLt picks GEMM solutions by batch shape, so sync and async runs
# can flip near-tie greedy tokens and shift ngram acceptance.
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
ngram_gpu_config = {
"method": "ngram_gpu",
"num_speculative_tokens": 3,
@@ -8,6 +8,7 @@ import pytest
from tests.models.utils import check_logprobs_close
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt
from vllm.platforms import current_platform
pytestmark = pytest.mark.cpu_model
@@ -112,3 +113,24 @@ def test_prefix_cache_hit_matches_cold_cache(full_prefill_refs):
name_0="cold_cache",
name_1="warm_cache",
)
def test_prefix_cache_hit_reaches_last_full_block():
"""A repeated prompt must hit every full block of the hybrid block size.
The workers raise the block size to fit the GDN state pages, and the
engine core reruns its config checks after its handshake; the raised size
must survive them, or the prefill's state checkpoint misses the block
boundaries and nothing is reused. One chunk holds the whole prompt.
"""
llm = _make_llm(enable_prefix_caching=True, max_num_batched_tokens=2048)
block_size = llm.llm_engine.vllm_config.cache_config.block_size
# Ends mid-block: only the last full block's state can be reused.
num_tokens = block_size + block_size // 2
assert num_tokens <= 2048
prompt = TokensPrompt(prompt_token_ids=[1000 + i for i in range(num_tokens)])
sp = SamplingParams(max_tokens=1)
hits = [llm.generate(prompt, sp)[0].num_cached_tokens for _ in range(2)]
del llm
assert hits == [0, block_size]
+49 -1
View File
@@ -25,7 +25,7 @@ from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine import EngineCoreRequest, EngineCoreRequestType
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor
from vllm.v1.executor.uniproc_executor import UniProcExecutor
@@ -689,6 +689,54 @@ def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
assert not core.pending_pause
@pytest.mark.parametrize(
"request_wave,engines_running,pause_state,announces",
[
pytest.param(3, False, PauseState.UNPAUSED, True, id="current-wave"),
pytest.param(2, False, PauseState.UNPAUSED, True, id="stale-wave"),
pytest.param(4, False, PauseState.UNPAUSED, True, id="newer-wave"),
pytest.param(3, True, PauseState.UNPAUSED, False, id="running"),
pytest.param(3, False, PauseState.PAUSED_ALL, False, id="paused"),
],
)
def test_idle_dp_rank_announces_wave_for_new_work(
request_wave, engines_running, pause_state, announces
):
"""An idle rank handed work announces the wave itself, since only engines
start waves; a paused rank waits for resume."""
core = object.__new__(DPEngineCoreProc)
core.has_coordinator = True
core.current_wave = 3
core.engines_running = engines_running
core.scheduler = MagicMock(pause_state=pause_state)
core.output_queue = MagicMock()
with patch.object(EngineCore, "add_request"):
core.add_request(MagicMock(), request_wave)
if announces:
assert core.engines_running
_, outputs = core.output_queue.put_nowait.call_args.args[0]
assert outputs.start_wave == max(request_wave, 3)
else:
core.output_queue.put_nowait.assert_not_called()
@pytest.mark.parametrize("paused", [False, True])
def test_paused_dp_rank_ignores_start_wave(paused):
"""A START_DP_WAVE still in flight when the pause completes must not wake
the paused rank."""
core = object.__new__(DPEngineCoreProc)
core.ignore_start_dp_wave = paused
core.engine_index = 0
core.current_wave = 3
core.engines_running = False
core._handle_client_request(EngineCoreRequestType.START_DP_WAVE, (3, 1))
assert core.engines_running != paused
def _pausable_engine_core_proc() -> EngineCoreProc:
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
core = object.__new__(EngineCoreProc)
+3 -3
View File
@@ -314,7 +314,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
client.current_wave = 0
client.resources = SimpleNamespace(stats_update_task=None)
client.stats_update_address = "inproc://scale-down-stats"
client.first_req_sock_addr = "inproc://scale-down-first-request"
client.scale_sock_addr = "inproc://scale-down-notify"
pause_started = asyncio.Event()
async def pause_scheduler(*args, **kwargs):
@@ -326,11 +326,11 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
with (
zmq.asyncio.Context() as ctx,
ctx.socket(zmq.XPUB) as coordinator,
ctx.socket(zmq.PAIR) as first_req_socket,
ctx.socket(zmq.PAIR) as scale_socket,
):
client.ctx = ctx
coordinator.bind(client.stats_update_address)
first_req_socket.bind(client.first_req_sock_addr)
scale_socket.bind(client.scale_sock_addr)
client._ensure_stats_update_task()
scale_task = None
try:
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Widen the dummy weight init for the MoRIIO dummy-weight P/D test.
The dummy loader fills float weights in +-1e-3, which keeps every activation so
small that outputs barely depend on the input, so a wrong KV prefix would go
unnoticed. The P/D test names ``WideDummyWeights`` as each server's
``--worker-extension-cls``. Workers import it before they load the model, which
re-registers the ``dummy`` load format in every worker process with a loader
that fills 16/32-bit float weights in +-``MORIIO_DUMMY_WEIGHT_RANGE``; 8-bit
weights and scales keep the default init. The format keeps the name ``dummy``
so every config path that special-cases dummy weights still applies.
The test process itself never imports this module.
"""
import os
import torch
from torch import nn
from vllm.config import ModelConfig
from vllm.model_executor.model_loader import register_model_loader
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
from vllm.model_executor.model_loader.weight_utils import (
initialize_single_dummy_weight,
)
_RANGE = float(os.environ["MORIIO_DUMMY_WEIGHT_RANGE"])
@register_model_loader("dummy")
class WideDummyModelLoader(DummyModelLoader):
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
# DummyModelLoader.load_weights with a wider init for 16/32-bit floats.
for layer in model.modules():
info = get_layerwise_info(layer)
if info.can_load():
self._process_online_quant_layer(layer, info)
continue
for name, tensor in get_layer_tensors(layer).items():
if name in layer._non_persistent_buffers_set:
continue
if tensor.is_floating_point() and torch.finfo(tensor.dtype).bits >= 16:
initialize_single_dummy_weight(
tensor, low=-_RANGE, high=_RANGE, name=name
)
else:
initialize_single_dummy_weight(tensor, name=name)
class WideDummyWeights:
"""Worker extension that only exists to import this module in workers."""
@@ -0,0 +1,736 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Single-node MoRIIO WRITE P/D checks on dummy weights.
Each case serves a real model architecture with ``--load-format dummy``, widened
by ``moriio_wide_dummy`` (see ``DUMMY_WEIGHT_RANGE``): the
config keeps the real hidden size, head geometry, MLA ranks and recurrent-state
shapes, so the connector registers and transfers exactly the production cache
layout, but the layer stack is cut to a few layers that still contain every
cache-group kind and the MoE is cut to a handful of experts. Dummy weights are
seeded, so the prefill, decode and reference servers hold identical weights.
Cases marked ``HMA_off`` run every server with
``--disable-hybrid-kv-cache-manager``, so all layers, sliding-window ones
included, share a single cache group.
Prefill runs on one GPU, decode on another, and MoRI moves the KV over xGMI.
Either way decode computes the last prompt token itself. For hybrid models
prefill computes only the first ``n - 1`` tokens and transfers the KV and the
recurrent state after them. For attention-only models prefill computes and
transfers all ``n``, and decode's scheduler, seeing the whole prompt cached,
recomputes the last token. A direct serve splits the same prompt the same way
when the first ``n - 1`` tokens end on a chunk boundary that both servers share,
so the prompts are chosen to make that happen: one is a single chunk plus one
token, the others are a whole number of hybrid cache blocks plus one token
(prefix caching ends chunks on block boundaries) or, for attention-only models,
a whole number of chunks plus one token. With the split matched, a correct
transfer reproduces the direct serve's first-token log-probabilities exactly,
so the comparison needs no noise allowance. The
servers compile with Inductor's deterministic mode, since otherwise each one
benchmarks its own reduction launch configs and separately started servers
can drift apart by a few bf16 logit steps. Guards:
* decode must report an external prefix-cache hit for every transferred token
of every prompt, so the match is not a local recompute;
* changing the first half of a prompt must move the direct serve's log-probs by
at least ``MIN_CONTEXT_EFFECT``, so a wrong prefix would be visible;
* ``test_corrupted_prefill_state_is_detected`` makes prefill compute its KV with
a different norm epsilon (same shapes, different values) and requires every
prompt to fail the comparison.
"""
import atexit
import contextlib
import json
import os
import random
import shutil
import signal
import subprocess
import sys
import tempfile
import time
from collections.abc import Callable, Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
_ROOT = Path(__file__).resolve().parents[4]
_PROXY = (
_ROOT / "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
)
# Max allowed difference between centered first-token log-probabilities of the
# P/D and direct outputs, over the direct serve's top-k tokens. The split is
# matched, so correct transfers produce identical logits; this only absorbs the
# fp32 rounding of the log-softmax (~1e-7), far below one bf16 logit step.
TOLERANCE = float(os.getenv("MORIIO_DUMMY_E2E_TOL", "1e-5"))
# Several bf16 steps of the logits at DUMMY_WEIGHT_RANGE.
MIN_CONTEXT_EFFECT = 0.05
# Half-width of the uniform dummy init for 16/32-bit float weights, applied in
# every server worker by moriio_wide_dummy.py. The loader default (1e-3) shrinks
# every activation so far that a context change moves the first-token log-probs
# by ~1e-4; at 0.1 it moves them by >0.1 and a corrupted prefill changes the
# top-k set entirely.
DUMMY_WEIGHT_RANGE = 0.1
_WIDE_DUMMY_DIR = Path(__file__).resolve().parent
TOP_K = 20
MAX_TOKENS = 4
MAX_MODEL_LEN = 4096
# Prefill token budget of every server. Prompts longer than it are chunked, so
# the final-chunk save path runs; the short prompt fills exactly one chunk.
PREFILL_CHUNK = 299
SHORT_PROMPT_LEN = PREFILL_CHUNK + 1
# Hybrid blocks per aligned prompt, before the trailing token.
ALIGNED_BLOCKS = (1, 2)
# Prefill chunks per long attention-only prompt, before the trailing token.
ALIGNED_CHUNKS = (2, 4)
def _mori_available() -> bool:
try:
import mori.io # noqa: F401
except ImportError:
return False
return True
pytestmark = [
pytest.mark.skipif(
not current_platform.is_rocm() or not _mori_available(),
reason="MoRIIO P/D needs ROCm and MoRI",
),
pytest.mark.skipif(
current_platform.is_rocm() and current_platform.device_count() < 2,
reason="needs one GPU for prefill and one for decode",
),
]
# ---------------------------------------------------------------------------
# Model architectures
def _kimi_linear_overrides(cfg) -> dict[str, Any]:
# 3 KDA layers then 1 MLA layer; layer 0 stays dense (first_k_dense_replace).
linear = dict(cfg.linear_attn_config)
linear["kda_layers"] = [1, 2, 3]
linear["full_attn_layers"] = [4]
groups = max(1, cfg.num_expert_group)
return {
"num_hidden_layers": 4,
"linear_attn_config": linear,
"num_experts": 2 * groups,
"num_experts_per_token": 2,
"topk_group": min(cfg.topk_group, groups),
}
def _qwen3_5_moe_overrides(cfg) -> dict[str, Any]:
# 3 Gated DeltaNet layers then 1 full-attention layer.
return {
"text_config": {
"num_hidden_layers": 4,
"layer_types": ["linear_attention"] * 3 + ["full_attention"],
"num_experts": 4,
"num_experts_per_tok": 2,
}
}
def _qwen3_dense_overrides(cfg) -> dict[str, Any]:
# Attention-only: decode gets the KV for the whole prompt.
return {"num_hidden_layers": 4}
def _deepseek_v3_overrides(cfg) -> dict[str, Any]:
# MLA attention-only: layer 0 dense, layer 1 MoE, no MTP layer.
return {
"num_hidden_layers": 2,
"first_k_dense_replace": 1,
"n_routed_experts": 8,
"num_experts_per_tok": 2,
"n_group": 1,
"topk_group": 1,
"num_nextn_predict_layers": 0,
}
def _gemma3_overrides(cfg) -> dict[str, Any]:
# Five sliding-window layers then one global layer, as in the real
# 5:1 pattern. Without the hybrid KV cache manager vLLM gives the sliding
# layers full-attention cache specs, so all six share one group.
return {
"num_hidden_layers": 6,
"layer_types": ["sliding_attention"] * 5 + ["full_attention"],
}
@dataclass(frozen=True)
class ArchCase:
model: str
overrides: Callable[[Any], dict[str, Any]]
# Norm-epsilon override that perturbs every computed KV/state value
# without changing a single shape. Used by the corruption control.
corrupt_overrides: dict[str, Any]
env: dict[str, str] = field(default_factory=dict)
args: tuple[str, ...] = ()
# Has recurrent-state (mamba-style) layers, so prefill stops at n - 1.
hybrid: bool = True
ARCHS: dict[str, ArchCase] = {
"kimi_linear": ArchCase(
model="moonshotai/Kimi-Linear-48B-A3B-Instruct",
overrides=_kimi_linear_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "1"},
args=("--trust-remote-code", "--mamba-ssm-cache-dtype", "float32"),
),
"qwen3_5_moe": ArchCase(
model="Qwen/Qwen3.5-35B-A3B",
overrides=_qwen3_5_moe_overrides,
corrupt_overrides={"text_config": {"rms_norm_eps": 1.0}},
# AITER's GDN conv-update kernel rejects the DS conv-state layout that
# MoRIIO's recurrent-state transfer requires.
env={"VLLM_ROCM_USE_AITER": "0"},
args=("--language-model-only",),
),
"qwen3_dense": ArchCase(
model="Qwen/Qwen3-32B",
overrides=_qwen3_dense_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "0"},
hybrid=False,
),
"deepseek_v3": ArchCase(
model="deepseek-ai/DeepSeek-V3",
overrides=_deepseek_v3_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "1"},
hybrid=False,
),
# Text-only Gemma 3. The multimodal Gemma3ForConditionalGeneration takes
# the inputs_embeds path on KV consumers (enable_mm_embeds is forced on
# there) and computes differently from a direct serve without any transfer.
"gemma3": ArchCase(
model="google/gemma-3-1b-it",
overrides=_gemma3_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "0"},
hybrid=False,
),
}
def _arch_overrides(arch: ArchCase) -> dict[str, Any]:
from vllm.transformers_utils.config import get_config
return arch.overrides(get_config(arch.model, trust_remote_code=True))
def _merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]:
merged = dict(base)
for key, value in extra.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
merged[key] = _merge(merged[key], value)
else:
merged[key] = value
return merged
# ---------------------------------------------------------------------------
# Serving
@dataclass(frozen=True)
class Case:
arch: str
attention_backend: str
prefill_graph: str
decode_graph: str
# Run every server of the case without the hybrid KV cache manager.
hma_off: bool = False
@property
def extra_args(self) -> tuple[str, ...]:
return ("--disable-hybrid-kv-cache-manager",) if self.hma_off else ()
@property
def id(self) -> str:
return (
f"{self.arch}-{self.attention_backend}-"
f"P_{self.prefill_graph}-D_{self.decode_graph}"
+ ("-HMA_off" if self.hma_off else "")
)
_DEFAULT = "FULL_AND_PIECEWISE"
CASES = [
# Kimi-Linear: KDA + MLA groups.
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT),
Case("kimi_linear", "ROCM_AITER_MLA", "NONE", "NONE"),
Case("kimi_linear", "ROCM_AITER_MLA", "PIECEWISE", "PIECEWISE"),
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, "FULL_DECODE_ONLY"),
Case("kimi_linear", "TRITON_MLA", _DEFAULT, _DEFAULT),
# Qwen3.5: Gated DeltaNet + full-attention groups.
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, _DEFAULT),
Case("qwen3_5_moe", "TRITON_ATTN", "NONE", "NONE"),
Case("qwen3_5_moe", "TRITON_ATTN", "PIECEWISE", "PIECEWISE"),
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, "FULL_DECODE_ONLY"),
# ROCM_ATTN keeps a (2, num_blocks, ...) KV layout and opts out of KV
# connectors, so the second ROCm attention backend is the unified one.
Case("qwen3_5_moe", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
# Attention-only: prefill transfers the whole prompt.
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT),
Case("qwen3_dense", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
# Without the hybrid KV cache manager: one cache group per model.
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
Case("deepseek_v3", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT, hma_off=True),
Case("gemma3", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
]
def _graph_args(mode: str) -> list[str]:
if mode == "NONE":
return ["--enforce-eager"]
# Inductor otherwise picks reduction launch configs by timing them, so each
# separately compiled server may sum in a different order and drift by a
# few bf16 logit steps; deterministic mode fixes the choice.
config = {
"cudagraph_mode": mode,
"inductor_compile_config": {"deterministic": True},
}
return ["--compilation-config", json.dumps(config)]
def _visible_devices() -> tuple[str, list[str]]:
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
selector = next((k for k in selectors if k in os.environ), selectors[0])
visible = os.getenv(selector)
devices = (
visible.split(",")
if visible
else [str(i) for i in range(current_platform.device_count())]
)
return selector, devices
def _device_env(selector: str, device: str) -> dict[str, str]:
# vLLM mirrors HIP_VISIBLE_DEVICES into CUDA_VISIBLE_DEVICES at import, so
# the test process carries both; pin every selector or the server refuses
# the mismatch. ROCR narrows the set first, so HIP/CUDA then index into it.
env = {selector: device}
for key in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
if key != selector:
env[key] = "0" if selector == "ROCR_VISIBLE_DEVICES" else device
return env
# KV consumers force enable_mm_embeds on, which moves multimodal-capable models
# (Qwen3.5) from input_ids to inputs_embeds, but the AOT compile-cache key does
# not cover it, so a consumer can load a producer's graph and crash in
# profile_run. Keep each role's compile cache apart until
# vllm-project/vllm#54642 lands (vllm-project/vllm#50891).
_CACHE_ROOT = Path(tempfile.mkdtemp(prefix="moriio_dummy_e2e_cache_"))
atexit.register(shutil.rmtree, _CACHE_ROOT, ignore_errors=True)
def _server(
arch: ArchCase,
overrides: dict[str, Any],
backend: str,
graph: str,
device: str,
selector: str,
*,
port: int | None = None,
extra_args: tuple[str, ...] = (),
kv_transfer_config: dict[str, Any] | None = None,
) -> RemoteOpenAIServer:
port = port or get_open_port()
args = [
*arch.args,
*extra_args,
"--port",
str(port),
"--load-format",
"dummy",
"--worker-extension-cls",
"moriio_wide_dummy.WideDummyWeights",
"--hf-overrides",
json.dumps(overrides),
"--dtype",
"bfloat16",
"--attention-backend",
backend,
"--max-model-len",
str(MAX_MODEL_LEN),
"--max-num-batched-tokens",
str(PREFILL_CHUNK),
"--max-num-seqs",
"16",
"--kv-cache-memory-bytes",
str(2 * 1024**3),
"--gpu-memory-utilization",
"0.5",
"--enable-prefix-caching",
*_graph_args(graph),
]
role = "direct"
if kv_transfer_config is not None:
args += ["--kv-transfer-config", json.dumps(kv_transfer_config)]
role = kv_transfer_config["kv_role"]
env = {
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
**_device_env(selector, device),
"VLLM_HOST_IP": "127.0.0.1",
"VLLM_PORT": str(get_open_port()),
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
"VLLM_SSM_CONV_STATE_LAYOUT": "DS",
"VLLM_CACHE_ROOT": str(_CACHE_ROOT / role),
# Only this directory, so the source tree cannot shadow installed vllm.
"PYTHONPATH": os.pathsep.join(
filter(None, [str(_WIDE_DUMMY_DIR), os.environ.get("PYTHONPATH")])
),
"MORIIO_DUMMY_WEIGHT_RANGE": str(DUMMY_WEIGHT_RANGE),
**arch.env,
}
return RemoteOpenAIServer(
arch.model, args, env_dict=env, auto_port=False, max_wait_seconds=900
)
def _moriio_config(role: str, http_port: int, ping_port: int) -> dict[str, Any]:
return {
"kv_connector": "MoRIIOConnector",
"kv_role": role,
"kv_connector_extra_config": {
"backend": "xgmi",
"read_mode": False,
"host_ip": "127.0.0.1",
"proxy_ip": "127.0.0.1",
"proxy_ping_port": str(ping_port),
"http_port": str(http_port),
"handshake_port": str(get_open_port()),
"notify_port": str(get_open_port()),
},
}
@contextlib.contextmanager
def _proxy(ping_port: int) -> Iterator[str]:
# Free ports for both sockets, so cases from parallel shards on one host
# do not collide.
port = get_open_port()
process = subprocess.Popen(
[
sys.executable,
str(_PROXY),
"--port",
str(port),
"--service-discovery-port",
str(ping_port),
],
start_new_session=True,
)
try:
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
assert process.poll() is None, "MoRIIO proxy exited during startup"
try:
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
break
except requests.ConnectionError:
time.sleep(0.2)
else:
raise TimeoutError("MoRIIO proxy did not start")
yield f"http://127.0.0.1:{port}"
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=10)
@contextlib.contextmanager
def _pd_pair(
arch: ArchCase,
prefill_overrides: dict[str, Any],
decode_overrides: dict[str, Any],
case: Case,
) -> Iterator[tuple[str, RemoteOpenAIServer]]:
selector, devices = _visible_devices()
assert len(devices) >= 2, "needs one GPU for prefill and one for decode"
prefill_port, decode_port, ping_port = (get_open_port() for _ in range(3))
with _proxy(ping_port) as proxy_url, contextlib.ExitStack() as stack:
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
with ThreadPoolExecutor(2) as pool:
futures = [
pool.submit(
_server,
arch,
prefill_overrides,
case.attention_backend,
case.prefill_graph,
devices[0],
selector,
port=prefill_port,
extra_args=case.extra_args,
kv_transfer_config=_moriio_config(
"kv_producer", prefill_port, ping_port
),
),
pool.submit(
_server,
arch,
decode_overrides,
case.attention_backend,
case.decode_graph,
devices[1],
selector,
port=decode_port,
extra_args=case.extra_args,
kv_transfer_config=_moriio_config(
"kv_consumer", decode_port, ping_port
),
),
]
for future in futures:
servers.append(future.result())
_wait_for_registration(proxy_url, arch.model)
yield proxy_url, servers[1]
def _wait_for_registration(proxy_url: str, model: str) -> None:
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
# The proxy hands decode max_tokens - 1 (prefill emits the first
# token), so the probe needs at least 2 for decode to accept it.
try:
response = requests.post(
f"{proxy_url}/v1/completions",
json={"model": model, "prompt": [1000, 1001], "max_tokens": 2},
timeout=300,
)
# The proxy streams the decode body, so a decode-side error can
# surface as a truncated response rather than a status code.
if response.status_code == 200 and response.json()["choices"]:
return
except requests.RequestException:
pass
time.sleep(1)
raise TimeoutError("prefill/decode never registered with the MoRIIO proxy")
# ---------------------------------------------------------------------------
# Measurement
def _metric(server: RemoteOpenAIServer, name: str) -> list[Any]:
from prometheus_client.parser import text_string_to_metric_families
response = requests.get(server.url_for("metrics"), timeout=10)
response.raise_for_status()
return [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
if sample.name == name
]
def _block_size(server: RemoteOpenAIServer) -> int:
# Hybrid models pad the attention block to the recurrent-state page, so the
# effective size is only known once the server has started.
sizes = {
int(sample.labels["block_size"])
for sample in _metric(server, "vllm:cache_config_info")
}
assert len(sizes) == 1, f"expected one cache block size, got {sizes}"
return sizes.pop()
def _prompt_lengths(arch: ArchCase, server: RemoteOpenAIServer) -> list[int]:
# Hybrid chunks end on cache-block boundaries; attention-only chunks end on
# the prefill budget, so the last token is computed on its own either way.
if arch.hybrid:
unit, counts = _block_size(server), ALIGNED_BLOCKS
else:
unit, counts = PREFILL_CHUNK, ALIGNED_CHUNKS
aligned = [
k * unit + 1 for k in counts if k * unit + 1 + MAX_TOKENS <= MAX_MODEL_LEN
]
assert aligned, f"alignment unit {unit} leaves no aligned prompt"
return [SHORT_PROMPT_LEN, *aligned]
def _prompts(lengths: list[int]) -> list[list[int]]:
rng = random.Random(0)
return [[rng.randrange(1000, 30000) for _ in range(n)] for n in lengths]
def _first_step(url: str, model: str, prompt: list[int]) -> dict[str, float]:
response = requests.post(
f"{url}/v1/completions",
json={
"model": model,
"prompt": prompt,
"max_tokens": MAX_TOKENS,
"temperature": 0,
"logprobs": TOP_K,
"return_tokens_as_token_ids": True,
},
timeout=600,
)
response.raise_for_status()
top = response.json()["choices"][0]["logprobs"]["top_logprobs"]
assert top, "server returned no logprobs"
return top[0]
def _distance(ref: dict[str, float], got: dict[str, float]) -> float:
"""Max difference of centered log-probs over the shared top-k tokens.
Centering removes the log-sum-exp offset, leaving the logit differences the
KV actually determines.
"""
common = sorted(ref.keys() & got.keys())
if len(common) < len(ref) // 2:
return float("inf")
ref_mean = sum(ref[t] for t in common) / len(common)
got_mean = sum(got[t] for t in common) / len(common)
return max(abs((ref[t] - ref_mean) - (got[t] - got_mean)) for t in common)
def _external_hits(server: RemoteOpenAIServer) -> float:
samples = _metric(server, "vllm:external_prefix_cache_hits_total")
return sum(sample.value for sample in samples)
@dataclass
class Reference:
prompts: list[list[int]]
logprobs: list[dict[str, float]]
hybrid: bool
@property
def expected_external_hits(self) -> int:
# Hybrid prefill stops one token short; attention-only prefill hands
# decode the whole prompt.
return sum(len(p) - int(self.hybrid) for p in self.prompts)
def _reference(arch: ArchCase, overrides: dict[str, Any], case: Case) -> Reference:
"""Direct serve; also checks that the comparison can see a wrong prefix."""
selector, devices = _visible_devices()
# Prefill graphs run the chunks and the decode-only last step runs the same
# captured graph in every decode mode used here, so the prefill mode
# reproduces both halves of the P/D computation.
with _server(
arch,
overrides,
case.attention_backend,
case.prefill_graph,
devices[0],
selector,
extra_args=case.extra_args,
) as server:
url = server.url_root
lengths = _prompt_lengths(arch, server)
prompts = _prompts(lengths)
refs = [_first_step(url, arch.model, p) for p in prompts]
# Same tail, different early context: the comparison has to see it.
for prompt, ref in zip(prompts, refs):
altered = [t + 1 for t in prompt[: len(prompt) // 2]]
altered += prompt[len(prompt) // 2 :]
moved = _distance(ref, _first_step(url, arch.model, altered))
print(f"[sensitivity] len={len(prompt)} context-change distance={moved}")
assert moved >= MIN_CONTEXT_EFFECT, (
f"first-token log-probs barely depend on the context (moved "
f"{moved:.4g} for len {len(prompt)}); a wrong prefix would go "
"unnoticed"
)
# Determinism of the reference itself (served from its prefix cache).
again = _first_step(url, arch.model, prompts[-1])
print(f"[noise] repeat distance={_distance(refs[-1], again)}")
return Reference(prompts, refs, arch.hybrid)
def _pd_distances(
arch: ArchCase,
prefill_overrides: dict[str, Any],
decode_overrides: dict[str, Any],
case: Case,
ref: Reference,
) -> tuple[list[float], float]:
with _pd_pair(arch, prefill_overrides, decode_overrides, case) as (
proxy_url,
decode,
):
hits_before = _external_hits(decode)
distances = [
_distance(expected, _first_step(proxy_url, arch.model, prompt))
for prompt, expected in zip(ref.prompts, ref.logprobs)
]
hits = _external_hits(decode) - hits_before
lengths = [len(p) for p in ref.prompts]
print(f"[pd] {case.id} lens={lengths} distances={distances} external_hits={hits}")
return distances, hits
# ---------------------------------------------------------------------------
# Tests
@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id)
def test_write_matches_direct_serve(case: Case):
arch = ARCHS[case.arch]
overrides = _arch_overrides(arch)
ref = _reference(arch, overrides, case)
distances, hits = _pd_distances(arch, overrides, overrides, case, ref)
# Every transferred token of every prompt must come from prefill.
assert hits == ref.expected_external_hits, (
f"decode reported {hits} external prefix-cache hits, expected "
f"{ref.expected_external_hits}; it did not use all the transferred KV"
)
lengths = [len(p) for p in ref.prompts]
bad = {n: d for n, d in zip(lengths, distances) if not d <= TOLERANCE}
assert not bad, f"P/D first-token log-probs differ from direct serve: {bad}"
@pytest.mark.parametrize("arch_name", sorted(ARCHS))
def test_corrupted_prefill_state_is_detected(arch_name: str):
"""Negative control: KV computed with different values must fail."""
arch = ARCHS[arch_name]
default = next(c for c in CASES if c.arch == arch_name)
case = Case(
arch_name, default.attention_backend, _DEFAULT, _DEFAULT, default.hma_off
)
overrides = _arch_overrides(arch)
ref = _reference(arch, overrides, case)
corrupted = _merge(overrides, arch.corrupt_overrides)
distances, hits = _pd_distances(arch, corrupted, overrides, case, ref)
assert hits == ref.expected_external_hits, (
"decode did not use the transferred KV; the control proves nothing"
)
assert all(d > TOLERANCE for d in distances), (
f"P/D outputs with wrongly computed prefill KV still matched the "
f"reference (distances {distances}); the comparison cannot detect "
"transfer corruption"
)
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import runpy
from pathlib import Path
from unittest.mock import AsyncMock
import httpx
import pytest
pytestmark = pytest.mark.cpu_test
@pytest.mark.asyncio
@pytest.mark.parametrize("endpoint", ["/v1/completions", "/v1/chat/completions"])
@pytest.mark.parametrize("stream", [False, True])
async def test_response_media_type_matches_payload(monkeypatch, endpoint, stream):
"""Clients must select SSE parsing for streaming completion responses."""
path = (
Path(__file__).parents[4]
/ "examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py"
)
proxy = runpy.run_path(str(path))["_handle_completions"].__globals__
app = proxy["app"]
app.state.ready = asyncio.Event()
app.state.ready.set()
app.state.prefill_iterator = itertools.cycle([({}, 0)])
app.state.decode_iterator = itertools.cycle([0])
app.state.decode_clients = [{}]
payload = b'{"choices":[{"text":"hello"}]}'
body = b"data: " + payload + b"\n\ndata: [DONE]\n\n" if stream else payload
async def decode_response(*args, **kwargs):
yield body
monkeypatch.setitem(proxy, "send_request_to_service", AsyncMock())
monkeypatch.setitem(proxy, "stream_service_response", decode_response)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://proxy"
) as client:
response = await client.post(endpoint, json={"stream": stream})
expected_type = "text/event-stream" if stream else "application/json"
assert response.headers["content-type"].split(";")[0] == expected_type
assert response.content == body
@@ -146,6 +146,7 @@ def _write_consumer_scheduler_for_finished_request(tp_size: int = 2):
scheduler.is_producer = False
scheduler.mode = MoRIIOMode.WRITE
scheduler.tp_size = tp_size
scheduler._has_mamba = False
scheduler._reqs_need_recv = {}
scheduler.unmap_request_id = MagicMock()
return scheduler
@@ -158,6 +159,10 @@ def _write_producer_scheduler(block_size: int = 1) -> Any:
scheduler.mode = MoRIIOMode.WRITE
scheduler.is_producer = True
scheduler.block_size = block_size
scheduler._has_mamba = False
scheduler.kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=object())]
)
scheduler.blocks_per_sw = [0]
scheduler._group_block_sizes = [block_size]
scheduler.transfer_id_to_request_id = {}
@@ -944,6 +949,7 @@ def test_clamp_to_prompt_blocks_clamps_each_group_by_its_own_block_size():
# their own prompt-block count; a sliding-window group is left untouched.
scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler)
scheduler.blocks_per_sw = [0, 0, 3]
scheduler._has_mamba = False
scheduler._group_block_sizes = [16, 32, 16]
req = SimpleNamespace(num_prompt_tokens=40)
@@ -12,12 +12,13 @@ wiring.
import importlib
from dataclasses import asdict
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs
@@ -52,6 +53,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
attributes read by the method under test are set."""
def __init__(self, **attrs):
self.mode = MoRIIOMode.READ
self._mamba_group_ids: list[int] = []
self._attn_group_ids: list[int] = [0]
self._num_ssm_scratch_blocks = 0
@@ -287,7 +289,8 @@ def test_scheduler_rejects_mamba_group_without_two_states():
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
@pytest.mark.parametrize("read_mode", [True, False])
def test_scheduler_rejects_non_dspark_speculative_hybrid(read_mode):
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
@@ -298,7 +301,9 @@ def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(speculative_config=_spec_config("ngram")),
_gate_vllm_config(
read_mode=read_mode, speculative_config=_spec_config("ngram")
),
"engine",
config,
)
@@ -336,14 +341,16 @@ def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True):
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize("read_mode", [True, False])
@pytest.mark.parametrize(
"dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)]
)
def test_scheduler_accepts_dspark_with_wrapped_full_attention(
dcp_size, dcp_sharded, expected_tail
dcp_size, dcp_sharded, expected_tail, read_mode
):
config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded)
vllm_config = _gate_vllm_config(
read_mode=read_mode,
speculative_config=_spec_config("dspark"),
block_size=4,
num_lookahead_tokens=7,
@@ -384,19 +391,234 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
)
def test_scheduler_rejects_hybrid_write():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
],
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_clamps_attention_and_excludes_scratch_slots():
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
config.transfer_groups = tuple(config.kv_cache_groups)
assert sched._clamp_to_prompt_blocks(
SimpleNamespace(num_prompt_tokens=9), [[10, 11, 12, 13], [1, 2, 3, 4]]
) == [[11], [1, 2]]
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False), "engine", config
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_resolves_current_state_instead_of_cached_block_ids():
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
state = KVConnectorBlockState(
req_ids={"req"},
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
boundary_state_offloads={},
)
assert sched._get_write_blocks(
SimpleNamespace(request_id="req", num_prompt_tokens=9),
[[10, 11, 12], [1]],
SimpleNamespace(kv_connector_block_state=state),
) == [[51], [31, 32]]
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize("external_tokens", [0, 9])
def test_hybrid_write_notifies_grouped_slots_or_releases_a_complete_local_hit(
external_tokens,
):
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False), "engine", config
)
sched.send_notify_block = Mock()
sched._send_transfer_release = Mock()
req = _mk_request(
list(range(10)),
params={
"do_remote_prefill": True,
"transfer_id": "tx",
"remote_host": "127.0.0.2",
"remote_notify_port": 7000,
},
)
req.request_id = "req"
sched.update_state_after_alloc(
req, _FakeBlocks(([10, 11, 12], [1, 2, 3])), external_tokens
)
if external_tokens:
sched.send_notify_block.assert_called_once_with(
req_id="req",
transfer_id="tx",
block_notify_list=[[10], [1, 2, 3]],
host="127.0.0.2",
port=7000,
)
sched._send_transfer_release.assert_not_called()
else:
sched.send_notify_block.assert_not_called()
sched._send_transfer_release.assert_called_once_with("tx", "127.0.0.2", 7000)
assert not req.kv_transfer_params["do_remote_prefill"]
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_keeps_separate_draft_groups_and_their_block_spans():
config = _dspark_wrapped_attention_config()
# A disabled cache is absent from the wire. Target and draft may have
# different token spans, and neither can borrow the other's block IDs.
draft = SimpleNamespace(
enable_kv_transfer=True,
kv_cache_spec=FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=2,
dtype=torch.float32,
dcp_sharded=False,
),
)
disabled = SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object())
config.kv_cache_groups.insert(1, disabled)
config.kv_cache_groups.append(draft)
config.transfer_groups = tuple(
g for g in config.kv_cache_groups if g.enable_kv_transfer
)
config.select_transfer_block_ids = lambda blocks: (blocks[0], blocks[2], blocks[3])
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(
read_mode=False, dcp_size=2, speculative_config=_spec_config("dspark")
),
"engine",
config,
)
req = _mk_request(
list(range(9)),
params={
"do_remote_decode": True,
"transfer_id": "tx",
},
)
req.request_id = "req"
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
blocks = ([10, 11, 12, 13], [99], [20, 21, 22], [30, 31, 32, 33])
expected = [[11], [20, 21], [30, 31, 32]]
assert sched._clamp_to_prompt_blocks(req, blocks) == expected
# Exercise the connector entry point too: completion must not flatten
# a multi-group WRITE payload through READ's single-attention helper.
delay_free, params = _FakeConnector(sched).request_finished_all_groups(req, blocks)
assert delay_free
assert params["remote_block_ids"] == expected
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize(
"blocks,delay_free",
[(([0, 51, 52, 53], [31, 32, 33]), True), (([], []), False)],
)
def test_hybrid_write_request_finished_defers_free_only_with_blocks_to_send(
blocks, delay_free
):
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
sched.is_producer = True
req = _mk_request(
list(range(9)), params={"do_remote_decode": True, "transfer_id": "tx"}
)
req.request_id = "req"
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
assert sched.request_finished(req, blocks)[0] is delay_free
# The WRITE ACK, not the READ watchdog, releases the deferred blocks.
if delay_free:
assert sched._deferred_send_deadlines["req"][1] == "tx"
else:
assert "req" not in sched._deferred_send_deadlines
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_build_meta_saves_current_state_on_the_final_chunk():
"""Chunked prefill must not ship block ids cached at admission: align-mode
state slots move between chunks without appending a block."""
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
sched.is_producer = True
set_role(moriio_common.ROLE.PRODUCER)
params = {
"do_remote_decode": True,
"transfer_id": "tx",
"remote_engine_id": "decode",
"remote_block_ids": [],
"remote_host": "127.0.0.1",
"remote_handshake_port": 6000,
"remote_notify_port": 6001,
}
req = _mk_request(list(range(9)), params=params)
req.request_id = "req"
req.num_computed_tokens = 0
sched._reqs_need_save["req"] = (req, [[10, 11, 12], [1]])
sched._req_kv_params["req"] = dict(params)
def _output(cached_req_ids, new_block_ids, num_scheduled):
return SimpleNamespace(
scheduled_cached_reqs=SimpleNamespace(
req_ids=cached_req_ids, new_block_ids=new_block_ids
),
num_scheduled_tokens={"req": num_scheduled},
new_block_ids_to_zero=None,
kv_connector_block_state=KVConnectorBlockState(
req_ids={"req"},
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
boundary_state_offloads={},
),
)
# First chunk: not final, so nothing is saved yet.
meta = sched.build_connector_meta(_output([], [], 4))
assert not meta.reqs_to_save
assert "req" in sched._reqs_need_pending_save
# Final chunk: stale cached ids plus new ids are ignored in favour of
# the scheduler's current table, clamped to the prompt.
req.num_computed_tokens = 4
meta = sched.build_connector_meta(_output(["req"], [([77], [78])], 5))
assert meta.reqs_to_save["req"].local_block_ids == [[51], [31, 32]]
assert "req" not in sched._reqs_need_pending_save
@pytest.mark.parametrize(
"mode,deadlines,expected",
[
(MoRIIOMode.WRITE, {"req": (0.0, "tx")}, True),
(MoRIIOMode.WRITE, {}, False),
(MoRIIOMode.READ, {"req": (0.0, "tx")}, False),
],
)
def test_has_pending_push_work_tracks_write_deferred_frees(mode, deadlines, expected):
connector = _FakeConnector(_FakeScheduler(_deferred_send_deadlines=deadlines))
connector.mode = mode
assert connector.has_pending_push_work() is expected
def test_has_pending_push_work_is_false_on_worker_role():
connector = _FakeConnector(None)
connector.mode = MoRIIOMode.WRITE
assert connector.has_pending_push_work() is False
def test_loaded_kv_cache_groups_are_every_transfer_group():
# The connector restores the request's own blocks, not a hash-addressed
# prefix, so a transfer-enabled Mamba group counts as loaded too.
connector = _FakeConnector(None)
connector._kv_cache_config = SimpleNamespace(
transfer_group_ids=(0, 2), prefix_cacheable_group_ids=(0,)
)
assert connector.get_loaded_kv_cache_group_ids(SimpleNamespace()) == (0, 2)
def test_split_block_groups_rejects_non_hybrid_use():
@@ -728,11 +950,15 @@ def test_session_build_rejects_per_layer_region_count_mismatch():
worker._get_built_session("prefill")
def test_register_kv_caches_rejects_hybrid_write_before_registration():
worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"})
def test_register_kv_caches_rejects_invalid_hybrid_write_state_layout():
worker = _FakeWorker(
mode=MoRIIOMode.WRITE,
_transfer_layer_names={"kda.0"},
layer_to_spec={"kda.0": _mamba_spec(1)},
)
worker._is_mamba_layer = lambda _layer_name: True
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"):
worker.register_kv_caches({"kda.0": object()})
@@ -933,6 +1159,40 @@ def test_read_matches_only_requests_with_pending_remote_prefill(params, expected
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
@pytest.mark.parametrize(
"has_mamba,expected", [(False, (32, True)), (True, (31, True))]
)
def test_write_consumer_matches_full_prompt_unless_hybrid(has_mamba, expected):
"""Without a mamba group (attention-only, or HMA disabled) prefill pushed
KV for all N tokens; with one it pushed h(N-1)."""
scheduler = _FakeScheduler(
is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=has_mamba
)
request = SimpleNamespace(
num_prompt_tokens=32, kv_transfer_params={"do_remote_prefill": True}
)
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
@pytest.mark.parametrize(
"is_producer,has_mamba,expected_prompt_len",
[(True, True, 4), (True, False, 5), (False, True, 5)],
)
def test_on_new_request_truncates_only_hybrid_producers(
is_producer, has_mamba, expected_prompt_len
):
"""Attention-only prefill (including models run with HMA disabled)
computes all N prompt tokens; only a hybrid producer drops the last."""
scheduler = _FakeScheduler(is_producer=is_producer, _has_mamba=has_mamba)
request = _mk_request([10, 11, 12, 13, 14], params={"do_remote_decode": True})
scheduler.on_new_request(request)
assert request.num_prompt_tokens == expected_prompt_len
assert len(request.prompt_token_ids) == expected_prompt_len
def test_read_rematch_after_alloc_keeps_queued_transfer():
"""Clearing the flag blocks another external hit, not the queued READ."""
sched = _make_read_scheduler()
@@ -981,12 +1241,37 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
assert is_async is True
@pytest.mark.parametrize("params", [None, {}, {"do_remote_prefill": False}])
def test_write_does_not_wait_again_after_receive_or_for_local_requests(params):
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
req = _mk_request(list(range(10)))
req.kv_transfer_params = params
assert sched.get_num_new_matched_tokens(req, 0) == (0, False)
def test_write_never_declares_an_empty_async_load():
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False)
req = _mk_request(list(range(10)), params={"do_remote_prefill": True})
assert sched.get_num_new_matched_tokens(req, 10) == (0, False)
@pytest.mark.parametrize(
"prompt_len,computed,expected", [(16, 0, 15), (17, 16, 0), (1, 0, 0)]
)
def test_hybrid_write_advertises_only_the_state_computed_by_prefill(
prompt_len, computed, expected
):
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
req = _mk_request(list(range(prompt_len)), params={"do_remote_prefill": True})
assert sched.get_num_new_matched_tokens(req, computed) == (expected, expected > 0)
@pytest.mark.parametrize(
("mode", "num_computed_tokens", "expected", "is_async"),
[
(MoRIIOMode.READ, 0, 9, False),
(MoRIIOMode.READ, 10, 0, False),
(MoRIIOMode.WRITE, 2, 8, True),
(MoRIIOMode.WRITE, 2, 7, True),
],
)
def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
@@ -7,6 +7,7 @@ import threading
from collections import OrderedDict
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock
import pytest
import torch
@@ -106,7 +107,7 @@ def _write_task(layer_name: str, transfer_id: str = "xfer") -> Any:
request_id="req",
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[1, 3],
local_block_ids=[[1, 3]],
remote_block_ids_hint=None,
layer_name=layer_name,
event=None,
@@ -345,6 +346,8 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
calls: list[str] = []
class FakeWorker:
_has_mamba: bool
layer_to_group: dict[str, int]
kv_caches: dict[str, torch.Tensor]
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
@@ -364,11 +367,13 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
return ([call_id], [call_id + 10], [call_id + 20])
fake_worker = FakeWorker()
fake_worker._has_mamba = False
fake_worker.layer_to_group = dict.fromkeys(kv_caches, 0)
fake_worker.kv_caches = kv_caches
fake_worker.layer_name_to_local_kv_cache_metadata = {name: [] for name in kv_caches}
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
request_info = RemoteAllocInfo(block_ids=[4, 5])
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
remote_meta = _remote_meta()
dense0_plan = writer._prepare_transfer_plan(
@@ -395,7 +400,7 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
request_info = RemoteAllocInfo(block_ids=[4, 5])
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
@@ -411,6 +416,80 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
assert writer._sealed_writes["xfer"] == 2
def test_hybrid_write_attention_uses_its_own_group_not_the_first_group():
worker = SimpleNamespace(
_has_mamba=True,
kv_caches={"draft": torch.empty((8, 1, 4, 3))},
layer_to_group={"draft": 2},
_region_session_indices=lambda _: [5],
_compute_block_transfer_offsets=Mock(return_value=([1], [2], [3])),
)
writer = make_moriio_writer(worker)
task = _write_task("draft")
task.local_block_ids = [[10], [20], [30, 31]]
remote = RemoteAllocInfo(block_ids=[[40], [50], [60, 61, 62]])
meta = _remote_meta()
plan = writer._prepare_transfer_plan(task, remote, meta)
worker._compute_block_transfer_offsets.assert_called_once_with(
"draft", [30, 31], [60, 61], meta, remote_engine_id=task.dst_engine_id
)
assert plan.sess_idx == 5
def test_hybrid_write_posts_both_regions_before_layer_completion():
from unittest.mock import Mock
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorWorker,
)
wrapper = _wrapper_for_messages()
info = RemoteAllocInfo(block_ids=[[], [7], [9]])
wrapper.done_remote_allocate_req_dict["xfer"] = info
wrapper.write_remote_data = Mock(side_effect=["conv-status", "ssm-status"])
worker = SimpleNamespace(
moriio_wrapper=wrapper,
tp_rank=0,
get_engine_name_with_dp=lambda name, rank: name,
_get_built_session=lambda name: (["conv-session", "ssm-session"], None),
_is_mamba_layer=lambda name: True,
_mamba_payload_index_by_layer={"kda": 2},
_region_session_indices=lambda name: [0, 1],
_compute_mamba_transfer_offsets=Mock(
return_value=([10, 20, 30, 40], [50, 60, 70, 80], [2, 4, 8, 16], 3)
),
)
worker._mamba_blocks_for_layer = lambda name, blocks: (
MoRIIOConnectorWorker._mamba_blocks_for_layer(worker, name, blocks)
)
writer = make_moriio_writer(worker)
task = _write_task("kda")
task.local_block_ids = [[], [1], [3]]
task.remote_tp_size = 2
task.event = Mock()
writer._execute_write_task(task)
task.event.synchronize.assert_called_once()
worker._compute_mamba_transfer_offsets.assert_called_once_with(
"kda", [3], [9], remote_tp_size=2
)
assert wrapper.write_remote_data.call_args_list[0].args == (
[2, 4, 8],
[10, 20, 30],
[50, 60, 70],
"conv-session",
)
assert wrapper.write_remote_data.call_args_list[1].args == (
[16],
[40],
[80],
"ssm-session",
)
assert info.transfer_statuses == ["conv-status", "ssm-status"]
assert info.writes_done == 1
assert not info.completion_notified
def test_write_completion_notifies_once_after_all_sealed_writes_finish():
class FakeWrapper:
def __init__(self):
@@ -445,7 +524,7 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
)
wrapper = FakeWrapper()
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
request_info = RemoteAllocInfo(block_ids=[[4, 5]], writes_expected=2)
request_info.transfer_statuses.extend(["status-a", "status-b"])
request_info.completion_request_id = "req"
# Final notify port is resolved in _execute_write_task; finalize reads it
@@ -487,6 +566,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
wrapper.lock = threading.Lock()
wrapper._transfer_timeout = 1
wrapper.moriio_engine = object()
global_status = FakeStatus()
scoped_status = FakeStatus()
wrapper.transfer_status = [global_status]
@@ -500,7 +580,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
def test_write_failure_marks_terminal_and_clears_scheduled_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 2
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
@@ -518,7 +598,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 1
writer._scheduled_layers["xfer"] = {"dense0"}
@@ -610,7 +690,7 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
if expected == "remote_blocks":
request_info = wrapper.done_remote_allocate_req_dict["xfer"]
assert request_info.block_ids == [4, 5]
assert request_info.block_ids == [[4, 5]]
assert request_info.decode_dp_rank == 3
elif expected == "write_done":
assert wrapper.done_write_cache_req_ids == ["xfer"]
@@ -11,6 +11,8 @@ import importlib
import importlib.util
import threading
import time
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
@@ -127,3 +129,62 @@ def test_wait_blocks_until_terminal():
assert time.monotonic() - start >= 0.15
finally:
timer.cancel()
@pytest.mark.parametrize("timeout,expected_ms", [(0.05, 50), (0.0001, 1)])
def test_native_wait_receives_all_statuses_and_a_positive_deadline(
timeout, expected_ms
):
statuses = [_pending(), _ok()]
engine = SimpleNamespace(
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
)
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=timeout)
wrapper.waiting_for_transfer_complete(statuses)
engine.wait_all.assert_called_once_with(statuses, timeout_ms=expected_ms)
@pytest.mark.parametrize(
"result,statuses,message",
[
("ERR_RDMA_OP", [_pending(), _bad()], "SQ full"),
("IN_PROGRESS", [_pending(), _ok()], "timed out"),
("ERR_INVALID_ARGS", [_pending()], "ERR_INVALID_ARGS"),
],
)
def test_native_wait_reports_failure_or_timeout(result, statuses, message):
engine = SimpleNamespace(
wait_all=Mock(return_value=getattr(moriio_engine.StatusCode, result))
)
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
with pytest.raises(TransferError, match=message):
wrapper.waiting_for_transfer_complete(statuses)
def test_wait_falls_back_without_native_status_enum(monkeypatch):
monkeypatch.setattr(moriio_engine, "StatusCode", None)
engine = SimpleNamespace(wait_all=Mock())
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
with pytest.raises(TransferError, match="SQ full"):
wrapper.waiting_for_transfer_complete([_bad(), _ok()])
engine.wait_all.assert_not_called()
def test_nonblocking_poll_does_not_call_native_wait():
engine = SimpleNamespace(wait_all=Mock())
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
assert wrapper.poll_transfer_batch([_pending()]) is TransferBatchState.PENDING
wrapper.waiting_for_transfer_complete([])
engine.wait_all.assert_not_called()
def test_native_wait_drains_only_the_unscoped_status_snapshot():
statuses = [_pending(), _ok()]
engine = SimpleNamespace(
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
)
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
wrapper.transfer_status.extend(statuses)
wrapper.waiting_for_transfer_complete()
engine.wait_all.assert_called_once_with(statuses, timeout_ms=50)
assert wrapper.transfer_status == []
@@ -14,6 +14,8 @@ import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ROLE,
LayerTransferPlan,
MoRIIOError,
MoRIIOMode,
RemoteAllocInfo,
ReqMeta,
@@ -96,7 +98,7 @@ def _write_task(
request_id=request_id,
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[1, 3],
local_block_ids=[[1, 3]],
remote_block_ids_hint=None,
layer_name="layer0",
event=None,
@@ -249,6 +251,7 @@ def test_finalize_single_pod_port_uses_global_rank():
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = False
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
@@ -266,13 +269,14 @@ def test_write_blocks_for_req_forwards_routing_to_schedule_write():
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
assert captured["remote_dp_size_local"] == 8
assert captured["remote_ip"] == "10.0.0.1"
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
assert captured["local_block_ids"] == [[1]]
assert captured["remote_block_ids"] == [[2]]
assert not hasattr(worker, "multi_pod_hosts")
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = False
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
@@ -290,8 +294,8 @@ def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
assert captured["remote_dp_size_local"] == 4
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
assert captured["local_block_ids"] == [[1]]
assert captured["remote_block_ids"] == [[2]]
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
@@ -373,3 +377,73 @@ def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
assert results.failed_recving == {"req-1"}
assert wrapper.pop_finished_write_req_ids() == set()
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer():
from unittest.mock import Mock
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = True
worker.world_size = 2
worker.schedule_write_blocks = Mock()
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None)
worker.schedule_write_blocks.assert_not_called()
@pytest.mark.parametrize(
"notify,expected",
[
# Plain WRITE peers still send one flat attention list.
([7, 8], [[7, 8]]),
# Hybrid WRITE sends one list per transfer group, kept as-is even
# when the first group is empty (full attention hit, state only).
([[7], [9]], [[7], [9]]),
([[], [9]], [[], [9]]),
],
)
def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected):
wrapper = _real_wrapper()
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
wrapper._handle_remote_blocks_message(
{"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2}
)
info = wrapper.done_remote_allocate_req_dict["tA"]
assert info.block_ids == expected
assert info.decode_dp_rank == 2
def test_empty_layer_plan_posts_no_rdma_write():
from unittest.mock import Mock
wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock())
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
plan = LayerTransferPlan(
request_id="req",
transfer_id="tA",
layer_name="layer0",
sess_idx=0,
transfer_local_offsets=[],
transfer_remote_offsets=[],
transfer_sizes=[],
)
assert writer._do_layer_write(plan, ["session"]) == []
wrapper.write_remote_data.assert_not_called()
wrapper.write_remote_data_single.assert_not_called()
@pytest.mark.parametrize("local_state", [[], [1, 2]])
def test_hybrid_write_requires_exactly_one_running_state_slot(local_state):
worker = SimpleNamespace(
_mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1],
)
writer = make_moriio_writer(worker)
task = _write_task("tA", "10.0.0.1")
task.layer_name = "kda"
task.local_block_ids = [[], local_state]
with pytest.raises(MoRIIOError, match="one running-state slot"):
writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]]))
@@ -805,6 +805,23 @@ class TestMultiConnectorStats:
child_metrics.observe.assert_called_once_with({"transfer_duration": [1.5]}, 2)
@pytest.mark.parametrize("with_exporter", [False, True])
def test_prom_metrics_skips_stats_only_connectors(self, with_exporter):
"""Stats-only children must not prevent supported metrics from exporting."""
child_metrics = MagicMock()
metrics = object.__new__(MultiKVConnectorPromMetrics)
metrics._prom_metrics = (
{"MooncakeStoreConnector": child_metrics} if with_exporter else {}
)
payload = {"MooncakeConnector": {"bytes_transferred": [1024]}}
if with_exporter:
payload["MooncakeStoreConnector"] = {"load_get_bytes": [2048]}
metrics.observe(payload, engine_idx=2)
if with_exporter:
child_metrics.observe.assert_called_once_with({"load_get_bytes": [2048]}, 2)
def test_aggregate_same_connector(self):
"""Test aggregating stats from the same connector type."""
stats1 = MultiKVConnectorStats(
+82 -2
View File
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p=None)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topp_only(self, batch_size: int, vocab_size: int):
"""Test top-p only (k=None)."""
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k=None, p=p)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
"""Test combined top-k and top-p."""
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p)
@pytest.mark.parametrize("batch_size", [8, 48, 64])
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
"""Mixed batches at vocab sizes where the split top-p pipeline uses
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
k[1::4] = vocab_size
p[2::4] = 1.0
k[3::4] = vocab_size
p[3::4] = 1.0
self._compare_results(logits, k, p)
def test_both_disabled(self):
"""Test when both k and p are None (should be no-op)."""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
f"{kept_topk_topp.tolist()}"
)
@pytest.mark.parametrize("batch_size", [8, 48])
def test_skip_split_without_topp_only_rows(
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
):
"""has_topp_only_rows=False skips the split pipeline; the result must
not change when every top-p row also has a top-k, grammar rows
(k >= #finite logits) included."""
import vllm.v1.sample.ops.topk_topp_triton as tt
vocab_size = 32000
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
logits[1] = float("-inf")
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
logits[1, idx] = torch.randn(30, generator=self.generator)
k = torch.randint(
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
k[::4] = vocab_size
p[::4] = 1.0
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
monkeypatch.setattr(
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
)
actual = tt.apply_top_k_top_p_triton(
logits.clone(), k, p, has_topp_only_rows=False
)
assert torch.equal(actual, expected)
def test_topp_tie_break_deterministic(self):
"""Boundary-duplicate trimming must be deterministic across calls.
@@ -821,6 +871,36 @@ class TestTritonTopkTopp:
)
assert len(kept_idx) == pytorch_kept
def test_topp_ties_across_blocks(self):
"""The split top-p tie budget must carry across a slice's blocks.
Regression: the split top-p mask kernel restarted its count of
boundary-value copies at every 8192-entry block, so each block of a
slice could keep up to `numkeep` copies of its own.
"""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
batch_size, vocab_size, num_ties = 64, 151936, 4096
# 4096 tied tokens spread over many blocks, between 64 tokens above
# and a low tail; the p=0.75 boundary lands mid-copy in the tie group.
tie_idx = torch.arange(num_ties) * 37
logits = torch.full((batch_size, vocab_size), -8.0)
logits[:, tie_idx] = 0.0
logits[:, -64:] = 4.0
p = torch.full((batch_size,), 0.75, dtype=torch.float32)
kept = apply_top_k_top_p_triton(logits.clone(), None, p) != float("-inf")
expected = apply_top_k_top_p_pytorch(logits.clone(), None, p) != float("-inf")
num_kept, num_expected = kept.sum(dim=-1), expected.sum(dim=-1)
assert torch.equal(num_kept, num_expected), (
f"kept {num_kept.unique().tolist()}, "
f"expected {num_expected.unique().tolist()}"
)
# Kept copies of the boundary value are the first ones in index order.
kept_ties = kept[:, tie_idx]
num_kept_ties = kept_ties.sum(dim=-1, keepdim=True)
assert torch.equal(kept_ties, torch.arange(num_ties) < num_kept_ties)
# =============================================================================
# FlashInfer top-k/top-p robustness tests
@@ -245,6 +245,55 @@ def test_connector_reports_configured_cache_source(
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
@pytest.mark.parametrize("num_layers", [1, 2])
@pytest.mark.parametrize("backend", ["cpu", "disk"])
def test_connector_explicit_cpu_blocks_align_uneven_stages(
tmp_path, num_layers: int, backend: str
):
"""The same configured pool size must fit lighter and heavier PP stages."""
config = _make_kv_cache_config(num_blocks=16)
names = [f"layer_{i}" for i in range(num_layers)]
config.kv_cache_groups[0].layer_names = names
config.kv_cache_tensors[0].layers = names
config.kv_cache_tensors[0].size *= num_layers
vllm_config = _make_vllm_config()
extra = {
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8,
"num_cpu_blocks": 4,
"kv_offload_backend": backend,
}
if backend == "disk":
extra.update(
disk_path=str(tmp_path / "offload.bin"),
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
)
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = extra
connector = SimpleCPUOffloadConnector(
vllm_config, KVConnectorRole.SCHEDULER, config
)
assert connector.scheduler_manager is not None
assert connector.scheduler_manager.num_cpu_blocks == 4
assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4
@pytest.mark.parametrize(
"count, capacity_bytes",
[(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]]
+ [(1, _BYTES_PER_BLOCK // 2)],
)
def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes):
config = _make_kv_cache_config(num_blocks=16)
vllm_config = _make_vllm_config()
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = {
"cpu_bytes_to_use_per_rank": capacity_bytes,
"num_cpu_blocks": count,
}
with pytest.raises(ValueError, match="num_cpu_blocks"):
SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config)
def make_scheduler(
num_cpu_blocks: int = 8,
num_gpu_blocks: int = 16,
+63
View File
@@ -371,6 +371,69 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
}
@pytest.mark.parametrize("num_layers", [1, 2])
def test_explicit_cpu_blocks_bound_live_worker_allocation(monkeypatch, num_layers):
"""Different live KV strides must honor the same explicit CPU pool size."""
num_blocks = 4
names = [f"layer.{i}" for i in range(num_layers)]
raw = torch.zeros(
num_layers * num_blocks * BLOCK_BYTES, dtype=torch.int8, device="cuda"
)
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=raw.nbytes,
layers=names,
layer_stride=num_blocks * BLOCK_BYTES,
block_stride=BLOCK_BYTES,
)
],
kv_cache_groups=[],
)
caches = dict(zip(names, raw.view(num_layers, num_blocks, BLOCK_BYTES)))
worker = SimpleCPUOffloadWorker(None, config, 8 * BLOCK_BYTES, num_cpu_blocks=4)
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
try:
worker.register_kv_caches(caches)
assert worker.num_cpu_blocks == 4
assert worker.cpu_kv_caches is not None
assert all(t.shape[0] == 4 for t in worker.cpu_kv_caches.values())
assert sum(t.nbytes for t in worker.cpu_kv_caches.values()) <= 8 * BLOCK_BYTES
finally:
if worker._backend is not None:
worker._backend.shutdown()
@pytest.mark.parametrize(
"capacity_bytes, count, local_capacity",
[(8 * BLOCK_BYTES, 5, 4), (BLOCK_BYTES, 1, 0)],
)
def test_explicit_cpu_blocks_reject_heavier_worker_before_allocation(
capacity_bytes, count, local_capacity
):
"""A count that fits the scheduler's stage must fail on a smaller worker."""
names = ["layer.0", "layer.1"]
raw = torch.zeros(2 * 4 * BLOCK_BYTES, dtype=torch.int8, device="cuda")
config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=raw.nbytes,
layers=names,
layer_stride=4 * BLOCK_BYTES,
block_stride=BLOCK_BYTES,
)
],
kv_cache_groups=[],
)
worker = SimpleCPUOffloadWorker(None, config, capacity_bytes, num_cpu_blocks=count)
caches = dict(zip(names, raw.view(2, 4, BLOCK_BYTES)))
with pytest.raises(ValueError, match=f"local offload capacity {local_capacity}"):
worker.register_kv_caches(caches)
assert worker.cpu_kv_caches is None
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
num_blocks = 4
block_bytes = 32
@@ -1157,6 +1157,91 @@ def test_chunked_requests_match_full_batch(has_draft_logits: bool):
assert torch.equal(chunked_sampled[valid], sampled[valid])
@pytest.mark.parametrize("use_block_verification", [False, True])
@pytest.mark.parametrize("has_draft_logits", [False, True])
def test_adaptive_draft_lengths_match_and_bonus_oracle(
use_block_verification: bool, has_draft_logits: bool
):
torch.manual_seed(7)
device = "cuda"
K = 3
num_drafts = [3, 1, 0, 2, 3, 2]
temperatures = [1.0, 0.0, 0.0, 0.6, 0.0, 1.0]
greedy_num_matches = {1: 1, 2: 0, 4: 2}
num_reqs = len(num_drafts)
num_logits_per_req = torch.tensor(num_drafts, device=device) + 1
cu_num_logits = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
cu_num_logits[1:] = num_logits_per_req.cumsum(0)
num_logits = int(cu_num_logits[-1])
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
expanded_idx_mapping = idx_mapping.repeat_interleave(num_logits_per_req)
expanded_local_pos = (
torch.arange(num_logits, dtype=torch.int32, device=device)
- cu_num_logits[expanded_idx_mapping]
)
draft_logits = torch.randn(num_reqs, K, VOCAB_SIZE, device=device)
target_logits = torch.randn(num_logits, VOCAB_SIZE, device=device)
# Allocate at exactly num_logits so a lookahead past the batch is OOB.
draft_sampled = torch.randint(VOCAB_SIZE, (num_logits,), device=device)
target_argmax = target_logits.argmax(dim=-1)
for req_idx, num_matches in greedy_num_matches.items():
lo = int(cu_num_logits[req_idx])
for step in range(num_drafts[req_idx]):
token = target_argmax[lo + step]
if step >= num_matches:
token = (token + 1) % VOCAB_SIZE
draft_sampled[lo + step + 1] = token
inputs = dict(
target_logits=target_logits,
draft_logits=draft_logits if has_draft_logits else None,
draft_sampled=draft_sampled,
cu_num_logits=cu_num_logits,
pos=torch.arange(num_logits, dtype=torch.int32, device=device),
idx_mapping=idx_mapping,
expanded_idx_mapping=expanded_idx_mapping,
expanded_local_pos=expanded_local_pos,
temperature=torch.tensor(temperatures, device=device),
seed=torch.arange(num_reqs, dtype=torch.int64, device=device),
num_speculative_steps=K,
use_block_verification=use_block_verification,
)
sampled, num_sampled = rejection_sample(**inputs)
for req_idx, num_matches in greedy_num_matches.items():
lo = int(cu_num_logits[req_idx])
n = num_matches + 1
assert int(num_sampled[req_idx]) == n, f"greedy request {req_idx}"
assert torch.equal(sampled[req_idx, :n], target_argmax[lo : lo + n]), (
f"greedy request {req_idx}"
)
for req_idx in range(num_reqs):
lo, hi = int(cu_num_logits[req_idx]), int(cu_num_logits[req_idx + 1])
req_inputs = dict(inputs)
for name in (
"target_logits",
"draft_sampled",
"pos",
"expanded_idx_mapping",
"expanded_local_pos",
):
req_inputs[name] = inputs[name][lo:hi]
req_inputs["cu_num_logits"] = cu_num_logits[req_idx : req_idx + 2] - lo
req_inputs["idx_mapping"] = idx_mapping[req_idx : req_idx + 1]
req_sampled, req_num_sampled = rejection_sample(**req_inputs)
n = int(req_num_sampled[0])
assert n <= num_drafts[req_idx] + 1
assert int(num_sampled[req_idx]) == n, f"request {req_idx} mismatch"
assert torch.equal(sampled[req_idx, :n], req_sampled[0, :n]), (
f"request {req_idx} mismatch"
)
def _residual_recovery_logits(
inputs: dict,
num_sampled: torch.Tensor,
+24
View File
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
assert sampler.needs_logits_processing[3] == admitted
def test_has_topp_only_rows():
"""A row is top-p-only when it has top-p and no effective top-k; top_k
of 0 (the default) or at least the logits width disables top-k."""
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
sampler.add_request(1, SamplingParams(top_k=10))
sampler.add_request(2, SamplingParams(top_p=0.9))
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
states = sampler.sampling_states
states.apply_staged_writes()
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
idx_np = np.array(idx, dtype=np.int32)
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
assert not has_topp_only_rows([0, 1])
assert has_topp_only_rows([0, 2])
assert has_topp_only_rows([3])
# The kernel compares top_k against the logits width, so a top_k at or
# above a narrower logits width counts as no top-k.
assert has_topp_only_rows([0], vocab_size=10)
def test_logits_processing_cache_only_checks_active_requests():
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(temperature=0.0))
+7 -1
View File
@@ -368,6 +368,7 @@ def test_gpu_sampler_respects_mixed_request_watermarking(
None,
None,
False,
False,
)
assert torch.equal(sampled, torch.tensor([3, 7]))
@@ -383,7 +384,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
# reference path; the assertion is about the call order, not the kernel.
monkeypatch.setattr(
"vllm.v1.watermarking.gpu_sampler.apply_top_k_top_p",
lambda logits, k, p: apply_top_k_top_p_pytorch(
lambda logits, k, p, **kwargs: apply_top_k_top_p_pytorch(
logits, k, p, allow_cpu_sync=True
),
)
@@ -415,6 +416,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
torch.tensor([2]),
torch.tensor([0.8]),
False,
False,
)
assert watermarker.captured_logits is not None
@@ -482,6 +484,7 @@ def test_gpu_sampler_skips_watermarking_for_repeated_contexts(monkeypatch):
None,
None,
False,
False,
)
assert torch.equal(sampled, torch.tensor([3, 7]))
@@ -517,6 +520,7 @@ def test_gpu_sampler_can_disable_context_deduplication(monkeypatch):
None,
None,
False,
False,
)
assert torch.equal(sampled, torch.tensor([7, 7]))
@@ -956,6 +960,7 @@ def test_gpu_sampler_uses_fused_gumbel_for_repeated_contexts():
None,
None,
False,
False,
)
assert torch.equal(repeated, torch.tensor([True, False], device=device))
@@ -992,6 +997,7 @@ def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
None,
None,
False,
False,
)
assert actual is expected