Merge branch 'main' into patches
This commit is contained in:
@@ -9,6 +9,7 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.main import run_case
|
||||
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -84,6 +85,18 @@ def test_jitter_kv_lens():
|
||||
jitter_kv_lens(case, 0, 1000, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 2])
|
||||
def test_bench_forward_rejects_oversized_verify(capacity):
|
||||
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
|
||||
runner = SimpleNamespace(num_speculative_steps=capacity)
|
||||
case = parse_case("verify=2*4@64:3")
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
|
||||
):
|
||||
run_case(runner, case, SimpleNamespace())
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
# TODO(woosuk): Support ROCm and re-enable this test there.
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -107,6 +107,8 @@ AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
|
||||
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
|
||||
# Static per-tensor: only the aiter quant op is matched by the pass
|
||||
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
|
||||
# Blockwise
|
||||
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
|
||||
]
|
||||
@@ -197,6 +199,9 @@ class TestModel(torch.nn.Module):
|
||||
# Blockwise path
|
||||
if self.use_aiter_fusion and self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_group_quant_op()]
|
||||
elif self.group_shape.is_per_tensor():
|
||||
if self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_per_tensor_quant_op()]
|
||||
else:
|
||||
if self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_per_token_quant_op()]
|
||||
@@ -221,6 +226,17 @@ class TestModel(torch.nn.Module):
|
||||
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
|
||||
AiterRMSFp8GroupQuantPattern.FUSED_OP,
|
||||
]
|
||||
elif self.group_shape.is_per_tensor():
|
||||
# Static per-tensor aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
AiterFusedAddRMSNormStaticQuantPattern,
|
||||
AiterRMSNormStaticQuantPattern,
|
||||
)
|
||||
|
||||
return [
|
||||
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
|
||||
AiterRMSNormStaticQuantPattern.FUSED_OP,
|
||||
]
|
||||
else:
|
||||
# Per-token aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -49,6 +49,9 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import PromptTokenUsageInfo, UsageInfo
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.validation import (
|
||||
validation_exception_handler,
|
||||
)
|
||||
@@ -1034,7 +1037,7 @@ def _make_stream_chunk(
|
||||
stop_reason: int | str | None = None,
|
||||
choices: list[ChatCompletionResponseStreamChoice] | None = None,
|
||||
usage: UsageInfo | None = None,
|
||||
) -> str:
|
||||
) -> ChatCompletionStreamResponse:
|
||||
if choices is None:
|
||||
choices = [
|
||||
ChatCompletionResponseStreamChoice(
|
||||
@@ -1051,7 +1054,7 @@ def _make_stream_chunk(
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
return f"data: {chunk.model_dump_json()}"
|
||||
return chunk
|
||||
|
||||
|
||||
def _tc(*, args, id=None, name=None):
|
||||
@@ -1062,6 +1065,39 @@ def _tc(*, args, id=None, name=None):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_use"])
|
||||
async def test_stream_error_preserves_message_and_closes_block(block_type):
|
||||
async def source():
|
||||
if block_type is not None:
|
||||
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
|
||||
if block_type == "text":
|
||||
delta = DeltaMessage(content="Hello")
|
||||
elif block_type == "thinking":
|
||||
delta = DeltaMessage(reasoning="Thinking")
|
||||
else:
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[_tc(id="call_1", name="search", args='{"q":')]
|
||||
)
|
||||
yield _make_stream_chunk(delta=delta)
|
||||
yield create_error_response(ValueError("backend failed"))
|
||||
|
||||
converter = _make_stream_converter()
|
||||
events = _parse_sse_events(
|
||||
[event async for event in converter.message_stream_converter(source())]
|
||||
)
|
||||
assert events[-1] == (
|
||||
"error",
|
||||
{
|
||||
"type": "error",
|
||||
"error": {"type": "internal_error", "message": "backend failed"},
|
||||
},
|
||||
)
|
||||
assert not any(event_type == "message_stop" for event_type, _ in events)
|
||||
if block_type is not None:
|
||||
assert events[-2][0] == "content_block_stop"
|
||||
|
||||
|
||||
class TestMessageStreamConverterToolUseContentBuffering:
|
||||
"""Regression test for tool_use arguments being silently dropped.
|
||||
|
||||
@@ -1074,7 +1110,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
|
||||
async def test_tool_use_args_not_dropped_when_content_in_same_chunk(
|
||||
self,
|
||||
):
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
usage=UsageInfo(prompt_tokens=10, total_tokens=10),
|
||||
@@ -1111,11 +1147,10 @@ class TestMessageStreamConverterToolUseContentBuffering:
|
||||
completion_tokens=20,
|
||||
),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
|
||||
events = _parse_sse_events(output)
|
||||
@@ -1156,7 +1191,7 @@ class TestMessageStreamConverterToolUseContentBuffering:
|
||||
stream jumps straight from finish_reason to [DONE], skipping the
|
||||
empty-choices usage chunk."""
|
||||
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
usage=UsageInfo(prompt_tokens=10, total_tokens=10),
|
||||
@@ -1178,11 +1213,10 @@ class TestMessageStreamConverterToolUseContentBuffering:
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
# No empty-choices usage chunk — go straight to [DONE].
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
|
||||
events = _parse_sse_events(output)
|
||||
@@ -1216,7 +1250,7 @@ class TestMessageStartIncludesTypeAndRole:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_message_start_contains_message_type_and_role(self):
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(content="Hello"),
|
||||
usage=UsageInfo(
|
||||
@@ -1226,11 +1260,10 @@ class TestMessageStartIncludesTypeAndRole:
|
||||
),
|
||||
)
|
||||
yield _make_stream_chunk(finish_reason="stop")
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
|
||||
events = _parse_sse_events(output)
|
||||
@@ -1260,7 +1293,7 @@ class TestStreamingCacheUsageSemantics:
|
||||
message_start omits cache fields. The final chunk carries
|
||||
prompt_tokens_details, so message_delta carries resolved values."""
|
||||
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(role="assistant", content="hi"),
|
||||
usage=UsageInfo(prompt_tokens=100, total_tokens=100),
|
||||
@@ -1277,11 +1310,10 @@ class TestStreamingCacheUsageSemantics:
|
||||
),
|
||||
),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
events = _parse_sse_events(output)
|
||||
|
||||
@@ -1305,7 +1337,7 @@ class TestStreamingCacheUsageSemantics:
|
||||
"""When the final chunk reports cached_tokens=0, message_delta carries
|
||||
cache fields = 0 (cache miss); message_start still omits them."""
|
||||
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
usage=UsageInfo(prompt_tokens=50, total_tokens=50),
|
||||
@@ -1322,11 +1354,10 @@ class TestStreamingCacheUsageSemantics:
|
||||
),
|
||||
),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
events = _parse_sse_events(output)
|
||||
|
||||
@@ -1346,7 +1377,7 @@ class TestStreamingCacheUsageSemantics:
|
||||
"""If --enable-prompt-tokens-details is off, no chunk carries cache
|
||||
info; both message_start and message_delta omit cache fields."""
|
||||
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(role="assistant"),
|
||||
usage=UsageInfo(prompt_tokens=30, total_tokens=30),
|
||||
@@ -1356,11 +1387,10 @@ class TestStreamingCacheUsageSemantics:
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
events = _parse_sse_events(output)
|
||||
|
||||
@@ -1628,7 +1658,7 @@ class TestStopSequenceReason:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_stop_string_maps_to_stop_sequence(self):
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(content="hi"),
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=5, completion_tokens=0),
|
||||
@@ -1638,11 +1668,10 @@ class TestStopSequenceReason:
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
|
||||
events = _parse_sse_events(output)
|
||||
@@ -1654,7 +1683,7 @@ class TestStopSequenceReason:
|
||||
async def test_streaming_no_stop_string_emits_explicit_null_stop_sequence(self):
|
||||
"""exclude_unset=True drops stop_sequence unless it is set explicitly."""
|
||||
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
|
||||
yield _make_stream_chunk(delta=DeltaMessage(content="hi"))
|
||||
yield _make_stream_chunk(finish_reason="stop")
|
||||
@@ -1662,11 +1691,10 @@ class TestStopSequenceReason:
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
async for event in converter.message_stream_converter(chunk_input()):
|
||||
output.append(event)
|
||||
|
||||
events = _parse_sse_events(output)
|
||||
@@ -1724,7 +1752,7 @@ class TestToolUseStopReason:
|
||||
],
|
||||
)
|
||||
async def test_streaming(self, args, finish_reason, stop_reason, expected):
|
||||
async def sse_input():
|
||||
async def chunk_input():
|
||||
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(
|
||||
@@ -1738,11 +1766,10 @@ class TestToolUseStopReason:
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
events = _parse_sse_events(
|
||||
[e async for e in converter.message_stream_converter(sse_input())]
|
||||
[e async for e in converter.message_stream_converter(chunk_input())]
|
||||
)
|
||||
|
||||
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
The stream-translation entry point is
|
||||
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
|
||||
async iterable of OpenAI SSE chunks into Cohere's
|
||||
async iterable of OpenAI chunk objects into Cohere's
|
||||
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
|
||||
event stream.
|
||||
|
||||
@@ -15,7 +15,7 @@ state machine.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import AsyncGenerator, Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -26,10 +26,7 @@ from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
CitationSource,
|
||||
)
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
MessageStartEvent,
|
||||
)
|
||||
from vllm.entrypoints.cohere.protocol import MessageStartEvent
|
||||
from vllm.entrypoints.cohere.serving import (
|
||||
_DONE_FRAME,
|
||||
CohereServingChatV2,
|
||||
@@ -41,6 +38,7 @@ from vllm.entrypoints.cohere.serving import (
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -85,9 +83,8 @@ def _make_chunk(
|
||||
usage: dict[str, Any] | None = None,
|
||||
omit_choices: bool = False,
|
||||
citations: list[Any] | None = None,
|
||||
) -> str:
|
||||
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
|
||||
consumes."""
|
||||
) -> ChatCompletionStreamResponse:
|
||||
"""Build the raw chat chunk the production converter consumes."""
|
||||
delta: dict[str, Any] = {}
|
||||
if role is not None:
|
||||
delta["role"] = role
|
||||
@@ -119,27 +116,23 @@ def _make_chunk(
|
||||
payload["choices"] = []
|
||||
if usage is not None:
|
||||
payload["usage"] = usage
|
||||
# Use ChatCompletionStreamResponse to normalize the payload shape.
|
||||
chunk = ChatCompletionStreamResponse.model_validate(payload)
|
||||
return f"data: {chunk.model_dump_json(exclude_none=False)}\n\n"
|
||||
return ChatCompletionStreamResponse.model_validate(payload)
|
||||
|
||||
|
||||
def _make_done() -> str:
|
||||
return "data: [DONE]\n\n"
|
||||
|
||||
|
||||
async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
|
||||
async def _stream_from(
|
||||
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
|
||||
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
|
||||
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
|
||||
async def _drain(
|
||||
serving: CohereServingChatV2,
|
||||
items: Sequence[ChatCompletionStreamResponse | ErrorResponse],
|
||||
) -> list[str]:
|
||||
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
|
||||
the emitted SSE frames."""
|
||||
request = CohereChatV2Request(
|
||||
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
|
||||
)
|
||||
gen = serving._chat_completion_stream_to_v2(_stream_from(items), request)
|
||||
gen = serving._chat_completion_stream_to_v2(_stream_from(items))
|
||||
return [frame async for frame in gen]
|
||||
|
||||
|
||||
@@ -793,15 +786,14 @@ class TestChatCompletionStreamToV2:
|
||||
assert "content-end" not in types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_done_marker_in_middle_closes_open_block(self):
|
||||
# Some upstreams send [DONE] without a trailing usage-only chunk.
|
||||
async def test_stream_exhaustion_closes_open_block_without_usage(self):
|
||||
# The raw stream may finish without a trailing usage-only chunk.
|
||||
# The translator must still emit message-end before [DONE] so
|
||||
# Cohere clients don't hang.
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="Hi"),
|
||||
_make_done(),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
@@ -815,72 +807,65 @@ class TestChatCompletionStreamToV2:
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_empty_and_non_data_lines(self):
|
||||
@pytest.mark.parametrize("block_type", [None, "text", "thinking", "tool_call"])
|
||||
async def test_upstream_error_preserves_message(self, block_type):
|
||||
serving = _serving()
|
||||
items = [
|
||||
"\n",
|
||||
"event: ping\n\n",
|
||||
_make_chunk(role="assistant"),
|
||||
"data: \n\n", # empty data
|
||||
_make_chunk(content="Hi"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
items: list[ChatCompletionStreamResponse | ErrorResponse] = []
|
||||
if block_type is not None:
|
||||
items.append(_make_chunk(role="assistant"))
|
||||
if block_type == "text":
|
||||
items.append(_make_chunk(content="Hi"))
|
||||
elif block_type == "thinking":
|
||||
items.append(_make_chunk(reasoning="Thinking"))
|
||||
else:
|
||||
items.append(
|
||||
_make_chunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"function": {"name": "search", "arguments": '{"q":'},
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
items.append(serving.create_error_response(ValueError("backend failed")))
|
||||
|
||||
frames = await _drain(serving, items)
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
# Should still produce a complete lifecycle.
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
assert types[0] == "message-start"
|
||||
assert types[-1] == "message-end"
|
||||
events = [_parse_event(frame) for frame in frames[:-1]]
|
||||
error_ends = [event for event in events if event["type"] == "message-end"]
|
||||
assert len(error_ends) == 1
|
||||
assert error_ends[0]["delta"] == {
|
||||
"finish_reason": "ERROR",
|
||||
"error": "backend failed",
|
||||
}
|
||||
assert events[-1] == error_ends[0]
|
||||
if block_type is not None:
|
||||
assert events[-2]["type"] == (
|
||||
"tool-call-end" if block_type == "tool_call" else "content-end"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_in_chunk_parsing_emits_error_message_end(self):
|
||||
# An invalid JSON payload after the first chunk triggers the
|
||||
# error path: a synthetic message-end with finish_reason=ERROR
|
||||
# followed by [DONE].
|
||||
serving = _serving()
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
"data: {not valid json}\n\n",
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
# Find the error-shaped message-end.
|
||||
error_end = _parse_event(frames[-2])
|
||||
assert error_end["type"] == "message-end"
|
||||
assert error_end["delta"]["finish_reason"] == "ERROR"
|
||||
assert "error" in error_end["delta"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_message_end_is_sanitized(self):
|
||||
@pytest.mark.parametrize("started", [False, True])
|
||||
async def test_error_message_end_is_sanitized(self, started):
|
||||
# The client-visible ``error`` field on the terminal message-end
|
||||
# must be routed through ``sanitize_message`` so tracebacks,
|
||||
# host filesystem paths, and Python object memory addresses
|
||||
# can't leak from a mid-stream exception into the SSE payload.
|
||||
serving = _serving()
|
||||
|
||||
async def _raises_mid_stream() -> AsyncGenerator[str, None]:
|
||||
yield _make_chunk(role="assistant")
|
||||
async def _raises_mid_stream() -> AsyncGenerator[
|
||||
ChatCompletionStreamResponse | ErrorResponse, None
|
||||
]:
|
||||
if started:
|
||||
yield _make_chunk(role="assistant")
|
||||
raise RuntimeError(
|
||||
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
|
||||
)
|
||||
|
||||
request = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
frames = [
|
||||
f
|
||||
async for f in serving._chat_completion_stream_to_v2(
|
||||
_raises_mid_stream(), request
|
||||
)
|
||||
f async for f in serving._chat_completion_stream_to_v2(_raises_mid_stream())
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
error_end = _parse_event(frames[-2])
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.entrypoints.generate.base.serving import build_per_request_timing_metr
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import (
|
||||
OpenAIServingChat,
|
||||
@@ -43,7 +44,7 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import resolve_finish_reason
|
||||
from vllm.exceptions import QueueOverflowError, VLLMValidationError
|
||||
from vllm.exceptions import GenerationError, QueueOverflowError, VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
@@ -716,6 +717,149 @@ async def _collect_metrics_stream_chunks(
|
||||
return chunks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("raw", [False, True])
|
||||
async def test_streaming_chunks_keep_usage_and_metadata(raw):
|
||||
serving = _build_minimal_metrics_serving_chat(False)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "Test"}],
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
metadata = RequestResponseMetadata(request_id="chatcmpl-test")
|
||||
method = (
|
||||
serving.chat_completion_stream_generator_raw
|
||||
if raw
|
||||
else serving.chat_completion_stream_generator
|
||||
)
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in method(
|
||||
request,
|
||||
_single_request_output(_make_metrics_request_output()),
|
||||
"chatcmpl-test",
|
||||
"test-model",
|
||||
conversation=[],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=metadata,
|
||||
)
|
||||
]
|
||||
if raw:
|
||||
assert all(isinstance(chunk, ChatCompletionStreamResponse) for chunk in chunks)
|
||||
else:
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
first = json.loads(chunks[0].removeprefix("data: "))
|
||||
assert first["choices"][0]["logprobs"] is None
|
||||
usage = json.loads(chunks[-2].removeprefix("data: "))
|
||||
assert "system_fingerprint" not in usage
|
||||
assert "prompt_tokens_details" not in usage["usage"]
|
||||
chunks = [
|
||||
ChatCompletionStreamResponse.model_validate_json(
|
||||
chunk.removeprefix("data: ")
|
||||
)
|
||||
for chunk in chunks[:-1]
|
||||
]
|
||||
assert len(chunks) == 3
|
||||
assert chunks[0].choices[0].delta.role == "assistant"
|
||||
assert chunks[1].choices[0].delta.content == "Hello"
|
||||
assert chunks[1].choices[0].finish_reason == "stop"
|
||||
assert chunks[2].choices == []
|
||||
assert chunks[2].usage.prompt_tokens == 3
|
||||
assert chunks[2].usage.completion_tokens == 2
|
||||
assert metadata.final_usage_info.total_tokens == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_serialization_failure_emits_error_and_done():
|
||||
serving = _build_minimal_metrics_serving_chat(False)
|
||||
serving.parser_cls = HarmonyParser
|
||||
parser = MagicMock()
|
||||
parser.parse_delta.return_value = DeltaMessage(
|
||||
content="Hello", extra_object=object()
|
||||
)
|
||||
serving._make_parser = MagicMock(return_value=parser)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model", messages=[{"role": "user", "content": "Test"}], stream=True
|
||||
)
|
||||
frames = [
|
||||
frame
|
||||
async for frame in serving.chat_completion_stream_generator(
|
||||
request,
|
||||
_single_request_output(_make_metrics_request_output()),
|
||||
"chatcmpl-test",
|
||||
"test-model",
|
||||
conversation=[],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test"),
|
||||
)
|
||||
]
|
||||
assert frames[-1] == "data: [DONE]\n\n"
|
||||
error = ErrorResponse.model_validate_json(frames[-2].removeprefix("data: "))
|
||||
assert "serialize" in error.error.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("raw", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"failure", ["parser", "before_first", "after_content", "generation"]
|
||||
)
|
||||
async def test_streaming_errors_preserve_envelope(raw, failure):
|
||||
serving = _build_minimal_metrics_serving_chat(False)
|
||||
error = (
|
||||
GenerationError("backend failed")
|
||||
if failure == "generation"
|
||||
else ValueError("backend failed")
|
||||
)
|
||||
if failure == "parser":
|
||||
serving.parser_cls = HarmonyParser
|
||||
serving._make_parser = MagicMock(side_effect=error)
|
||||
|
||||
async def results():
|
||||
if failure == "after_content":
|
||||
output = _make_metrics_request_output()
|
||||
output.outputs[0].finish_reason = None
|
||||
output.finished = False
|
||||
yield output
|
||||
raise error
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model", messages=[{"role": "user", "content": "Test"}], stream=True
|
||||
)
|
||||
method = (
|
||||
serving.chat_completion_stream_generator_raw
|
||||
if raw
|
||||
else serving.chat_completion_stream_generator
|
||||
)
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in method(
|
||||
request,
|
||||
results(),
|
||||
"chatcmpl-test",
|
||||
"test-model",
|
||||
conversation=[],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test"),
|
||||
)
|
||||
]
|
||||
if raw:
|
||||
assert all(
|
||||
isinstance(chunk, (ChatCompletionStreamResponse, ErrorResponse))
|
||||
for chunk in chunks
|
||||
)
|
||||
assert isinstance(chunks[-1], ErrorResponse)
|
||||
error_response = chunks[-1]
|
||||
else:
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
assert chunks.count("data: [DONE]\n\n") == 1
|
||||
error_response = ErrorResponse.model_validate_json(
|
||||
chunks[-2].removeprefix("data: ")
|
||||
)
|
||||
assert error_response.error.message == "backend failed"
|
||||
assert error_response.error.code == (500 if failure == "generation" else 400)
|
||||
|
||||
|
||||
def test_build_per_request_timing_metrics_valid_timestamps():
|
||||
metrics = build_per_request_timing_metrics(
|
||||
_PER_REQUEST_STATS, num_generation_tokens=10
|
||||
@@ -1188,6 +1332,15 @@ async def test_streaming_reasoning_usage_lifecycle_edges(
|
||||
await iterator.aclose()
|
||||
|
||||
error_chunks = [c for c in chunks if isinstance(c, dict) and "error" in c]
|
||||
if name == "empty-finish":
|
||||
# A choice that finishes without output still gets its last chunk, unparsed.
|
||||
assert any(
|
||||
choice.get("finish_reason") == "stop"
|
||||
for item in chunks
|
||||
if isinstance(item, dict)
|
||||
for choice in item.get("choices", [])
|
||||
)
|
||||
parsers[0].parse_delta.assert_not_called()
|
||||
call_count = sum(parser.count_reasoning_tokens.call_count for parser in parsers)
|
||||
if expected_tokens is not None:
|
||||
assert outcome == "exhausted" and chunks[-1] == "[DONE]" and not error_chunks
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -17,7 +18,7 @@ from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, WeightVersionSpan
|
||||
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
@@ -310,6 +311,76 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
|
||||
assert response.metrics.speculative_decoding.num_spec_steps == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("opt_in", [False, True])
|
||||
async def test_completion_weight_versions(stream: bool, opt_in: bool):
|
||||
"""Each choice gets its spans on its final output, even with no tokens."""
|
||||
expected = [{"version": "v1", "start": 0, "end": 2}]
|
||||
spans = [WeightVersionSpan(**span) for span in expected]
|
||||
|
||||
def output(index, token_ids, finish_reason=None, weight_versions=None):
|
||||
return CompletionOutput(
|
||||
index=index,
|
||||
text="x" * len(token_ids),
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
weight_versions=weight_versions,
|
||||
)
|
||||
|
||||
# Choice 1 is aborted before its first token.
|
||||
final = [output(1, [], "abort", [])]
|
||||
if stream:
|
||||
steps = [[output(0, [100])], [output(0, [101], "length", spans), *final]]
|
||||
else:
|
||||
steps = [[output(0, [100, 101], "length", spans), *final]]
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
for i, outputs in enumerate(steps):
|
||||
yield RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=outputs,
|
||||
finished=i == len(steps) - 1,
|
||||
)
|
||||
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
max_tokens=10,
|
||||
n=2,
|
||||
stream=stream,
|
||||
return_weight_versions=opt_in,
|
||||
)
|
||||
response = await _build_serving_completion(mock_engine).create_completion(request)
|
||||
|
||||
if stream:
|
||||
finals = {}
|
||||
async for line in response:
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload == "[DONE]":
|
||||
continue
|
||||
for choice in json.loads(payload)["choices"]:
|
||||
if choice.get("finish_reason"):
|
||||
finals[choice["index"]] = choice
|
||||
else:
|
||||
assert choice.get("weight_versions") is None
|
||||
got = [finals[i].get("weight_versions") for i in (0, 1)]
|
||||
else:
|
||||
got = [choice["weight_versions"] for choice in response.model_dump()["choices"]]
|
||||
assert got == ([expected, []] if opt_in else [None, None])
|
||||
|
||||
|
||||
def _build_online_renderer_for_completion(
|
||||
model_config: MockModelConfig,
|
||||
renderer: Any | None = None,
|
||||
|
||||
@@ -10,7 +10,7 @@ import torch
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers, processors
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm import PoolingParams, TokensPrompt
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
BiEncoderIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
@@ -18,7 +18,9 @@ from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
LateInteractionIOProcessor,
|
||||
_apply_post_tokenization_to_token_type_ids,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
|
||||
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
@@ -81,6 +83,49 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
|
||||
assert nonbinding["prompt_token_ids"] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_template,request_template,expected",
|
||||
[
|
||||
("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"),
|
||||
(
|
||||
"{{ messages[0].content }} {{ messages[1].content }}",
|
||||
"{{ messages[1].content }} {{ messages[0].content }}",
|
||||
"bc a",
|
||||
),
|
||||
(None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"),
|
||||
(None, None, "abc"),
|
||||
],
|
||||
)
|
||||
def test_offline_scoring_uses_default_template_unless_overridden(
|
||||
llm_reranker_processor, default_template, request_template, expected
|
||||
):
|
||||
processor = llm_reranker_processor
|
||||
processor.chat_template = default_template
|
||||
processor.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=32)
|
||||
)
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task="classify",
|
||||
tokenization_kwargs=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
scoring_data=ScoringData(data_1=["a"], data_2=["bc"]),
|
||||
chat_template=request_template,
|
||||
pooling_params=PoolingParams(),
|
||||
)
|
||||
factory, _ = processor.get_request_factory_offline(ctx)
|
||||
(request,) = factory()
|
||||
text, prompt = processor.get_score_prompt(
|
||||
request["data_1"],
|
||||
request["data_2"],
|
||||
request["tok_params"].get_encode_kwargs(),
|
||||
chat_template=request["chat_template"],
|
||||
)
|
||||
|
||||
assert text == expected
|
||||
assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected)
|
||||
|
||||
|
||||
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
"""The cross-encoder segment boundary must survive truncate + pad.
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateTokensResponse,
|
||||
@@ -1133,6 +1134,50 @@ async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer):
|
||||
assert "<think>" not in msg["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_reasoning", [True, False])
|
||||
async def test_e2e_parsed_logprobs_follow_include_reasoning(
|
||||
parser_client, parser_tokenizer, include_reasoning
|
||||
):
|
||||
"""With a parser configured, batch derender returns logprobs only when
|
||||
reasoning is shown. Hidden reasoning drops them, as the coupled chat
|
||||
endpoint does, because decoded logprob tokens would reveal it."""
|
||||
messages = [{"role": "user", "content": "What is 2+3?"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
output_ids = _require_markers_survive(
|
||||
parser_tokenizer, "<think>Two plus three.</think>Five.", "</think>"
|
||||
)
|
||||
generate_response = _e2e_generate_response(output_ids)
|
||||
generate_response["choices"][0]["logprobs"] = {
|
||||
"content": [
|
||||
{"token_id": tid, "logprob": -0.5, "rank": 1, "top_logprobs": []}
|
||||
for tid in output_ids
|
||||
]
|
||||
}
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": generate_response,
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": PARSER_MODEL,
|
||||
"messages": messages,
|
||||
"include_reasoning": include_reasoning,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
choice = resp.json()["choices"][0]
|
||||
assert choice["message"]["content"] == "Five."
|
||||
if include_reasoning:
|
||||
assert choice["message"]["reasoning"]
|
||||
assert len(choice["logprobs"]["content"]) == len(output_ids)
|
||||
else:
|
||||
assert choice["message"]["reasoning"] is None
|
||||
assert choice["logprobs"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer):
|
||||
"""<tool_call> extracted into tool_calls field."""
|
||||
@@ -1446,19 +1491,12 @@ _HERMES_TOOL_CALL_TEXT = (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||
[
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||
("required", "Hello there", "stop", "stop"),
|
||||
],
|
||||
)
|
||||
def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
tool_choice, text, engine_finish_reason, expected_finish_reason
|
||||
def _derender_hermes_chat(
|
||||
text: str,
|
||||
tool_choice: str = "auto",
|
||||
engine_finish_reason: str = "stop",
|
||||
parser_cls: type | None = None,
|
||||
):
|
||||
"""/derender applies the same finish_reason rule as the coupled chat
|
||||
path: ``tool_calls`` only for a real call at a natural stop."""
|
||||
tokenizer = get_tokenizer(MODEL_NAME)
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
@@ -1474,6 +1512,8 @@ def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
enable_auto_tools=True,
|
||||
tool_parser="hermes",
|
||||
)
|
||||
if parser_cls is not None:
|
||||
derenderer.parser = parser_cls
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Weather in Paris?"}],
|
||||
@@ -1488,6 +1528,54 @@ def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
)
|
||||
|
||||
(choice,) = derenderer._derender_chat(response, request)
|
||||
return choice
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "engine_finish_reason", "expected_finish_reason"),
|
||||
[
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "stop", "tool_calls"),
|
||||
("auto", _HERMES_TOOL_CALL_TEXT, "length", "length"),
|
||||
("required", "Hello there", "stop", "stop"),
|
||||
],
|
||||
)
|
||||
def test_derender_chat_finish_reason_matches_chat_completion(
|
||||
tool_choice, text, engine_finish_reason, expected_finish_reason
|
||||
):
|
||||
"""/derender applies the same finish_reason rule as the coupled chat
|
||||
path: ``tool_calls`` only for a real call at a natural stop."""
|
||||
choice = _derender_hermes_chat(text, tool_choice, engine_finish_reason)
|
||||
|
||||
assert choice.finish_reason == expected_finish_reason
|
||||
assert bool(choice.message.tool_calls) == (text == _HERMES_TOOL_CALL_TEXT)
|
||||
|
||||
|
||||
def test_derender_chat_tool_call_id_matches_chat_completion():
|
||||
"""Batch /derender uses the coupled chat's default tool call ID format."""
|
||||
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT)
|
||||
|
||||
(tool_call,) = choice.message.tool_calls
|
||||
assert tool_call.id.startswith("chatcmpl-tool-")
|
||||
|
||||
|
||||
def test_derender_chat_keeps_parser_tool_call_id():
|
||||
"""Batch /derender keeps the parser's tool call ID (Kimi K2 style
|
||||
``functions.<name>:<idx>``), since chat templates reference it in
|
||||
multi-turn history."""
|
||||
|
||||
class _KimiIdParser:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def parse(self, model_output, request, **kwargs):
|
||||
call = FunctionCall(
|
||||
id="functions.get_weather:1",
|
||||
name="get_weather",
|
||||
arguments='{"city": "Paris"}',
|
||||
)
|
||||
return None, None, [call]
|
||||
|
||||
choice = _derender_hermes_chat(_HERMES_TOOL_CALL_TEXT, parser_cls=_KimiIdParser)
|
||||
|
||||
(tool_call,) = choice.message.tool_calls
|
||||
assert tool_call.id == "functions.get_weather:1"
|
||||
|
||||
@@ -142,12 +142,28 @@ def _tool_sig(response_choice: dict) -> list[tuple[str, dict]]:
|
||||
]
|
||||
|
||||
|
||||
def _tool_id_shapes(response_choice: dict) -> list[str | None]:
|
||||
return [
|
||||
_tool_call_id_shape(tc["id"])
|
||||
for tc in (response_choice["message"].get("tool_calls") or [])
|
||||
]
|
||||
|
||||
|
||||
def _engine_finish_reason(coupled_finish_reason: str) -> str:
|
||||
"""Undo the chat level `"tool_calls"` rewrite of `"stop"` so derender
|
||||
gets what the engine reported and has to reapply the rewrite itself."""
|
||||
if coupled_finish_reason == "tool_calls":
|
||||
return "stop"
|
||||
return coupled_finish_reason
|
||||
|
||||
|
||||
def _assert_parity(coupled: dict, disagg: dict) -> None:
|
||||
"""Both paths saw the same tokens, so they must agree unconditionally."""
|
||||
c, d = coupled["choices"][0], disagg["choices"][0]
|
||||
assert d["message"]["content"] == c["message"]["content"]
|
||||
assert d["message"].get("reasoning") == c["message"].get("reasoning")
|
||||
assert _tool_sig(d) == _tool_sig(c)
|
||||
assert _tool_id_shapes(d) == _tool_id_shapes(c)
|
||||
assert d["finish_reason"] == c["finish_reason"]
|
||||
assert d["token_ids"] == c["token_ids"]
|
||||
assert disagg["usage"]["prompt_tokens"] == coupled["usage"]["prompt_tokens"]
|
||||
@@ -166,7 +182,7 @@ async def _run_parity_case(
|
||||
client,
|
||||
ch["token_ids"],
|
||||
coupled["usage"]["prompt_tokens"],
|
||||
ch["finish_reason"],
|
||||
_engine_finish_reason(ch["finish_reason"]),
|
||||
chat_request,
|
||||
)
|
||||
return coupled, disagg
|
||||
@@ -316,13 +332,18 @@ async def test_parity_logprobs(client):
|
||||
|
||||
|
||||
async def _coupled_stream(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[list[dict], list[int]]:
|
||||
client: httpx.AsyncClient,
|
||||
messages: list[dict],
|
||||
require_token_ids: bool = True,
|
||||
**extra,
|
||||
) -> tuple[list[dict], list[int] | None]:
|
||||
"""Stream `/v1/chat/completions` and return the choices that carry
|
||||
`token_ids` along with the prompt token IDs.
|
||||
|
||||
The leading role only chunk has no `token_ids` and no counterpart on
|
||||
`/inference/v1/generate`, so it is dropped.
|
||||
`/inference/v1/generate`, so it is dropped. With `require_token_ids`
|
||||
off every choice is kept and `prompt_token_ids` may be `None`, for
|
||||
requests where the coupled server hides token ids (hidden reasoning).
|
||||
"""
|
||||
choices: list[dict] = []
|
||||
prompt_token_ids: list[int] | None = None
|
||||
@@ -351,9 +372,12 @@ async def _coupled_stream(
|
||||
if chunk.get("prompt_token_ids") is not None:
|
||||
prompt_token_ids = chunk["prompt_token_ids"]
|
||||
choices.extend(
|
||||
ch for ch in chunk["choices"] if ch.get("token_ids") is not None
|
||||
ch
|
||||
for ch in chunk["choices"]
|
||||
if not require_token_ids or ch.get("token_ids") is not None
|
||||
)
|
||||
assert prompt_token_ids is not None
|
||||
if require_token_ids:
|
||||
assert prompt_token_ids is not None
|
||||
return choices, prompt_token_ids
|
||||
|
||||
|
||||
@@ -390,7 +414,7 @@ async def _run_stream_parity_case(
|
||||
{"model": MODEL, "messages": messages, "return_token_ids": True, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
finish_reason=_engine_finish_reason(finish_reason),
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
@@ -473,6 +497,151 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming logprobs on the parser path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _logprob_tokens(choice: dict) -> list[tuple[str, list[int] | None]] | None:
|
||||
lp = choice.get("logprobs")
|
||||
if lp is None or lp.get("content") is None:
|
||||
return None
|
||||
return [(e["token"], e.get("bytes")) for e in lp["content"]]
|
||||
|
||||
|
||||
def _sampled_generate_logprobs(choice: dict) -> dict | None:
|
||||
"""`GenerateLogProbs` for one coupled stream chunk from its `token_ids`
|
||||
and per-token logprob values, without `top_logprobs`.
|
||||
|
||||
The coupled stream's `token_ids` are the integer ids a GPU-less worker
|
||||
would have sent, so one stream is both the derender input and the string
|
||||
reference. A second greedy run is not needed, and greedy decoding is not
|
||||
bit-reproducible across calls on every platform anyway.
|
||||
"""
|
||||
lp = choice.get("logprobs")
|
||||
if lp is None or lp.get("content") is None:
|
||||
return None
|
||||
content = lp["content"]
|
||||
assert len(content) == len(choice["token_ids"]), choice
|
||||
return {
|
||||
"content": [
|
||||
{"token_id": tid, "logprob": entry["logprob"], "top_logprobs": []}
|
||||
for tid, entry in zip(choice["token_ids"], content)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def _run_stream_logprob_parity_case(
|
||||
client: httpx.AsyncClient, messages: list[dict], **extra
|
||||
) -> tuple[dict, list[dict], list[dict]]:
|
||||
"""Stream with logprobs through the coupled server, replay the same
|
||||
chunks through the streaming derender endpoint and compare per chunk.
|
||||
Returns the assembled coupled message, the coupled choices and the
|
||||
derender choices.
|
||||
"""
|
||||
extra = {"logprobs": True, **extra}
|
||||
coupled_choices, prompt_token_ids = await _coupled_stream(client, messages, **extra)
|
||||
finish_reason = coupled_choices[-1]["finish_reason"]
|
||||
derender_choices: list[dict] = []
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in coupled_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in coupled_choices],
|
||||
{"model": MODEL, "messages": messages, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
logprobs=[_sampled_generate_logprobs(ch) for ch in coupled_choices],
|
||||
raw_choices=derender_choices,
|
||||
)
|
||||
coupled = assemble_stream(coupled_choices)
|
||||
for message in (coupled, disagg):
|
||||
for tc in message["tool_calls"]:
|
||||
tc["id"] = _tool_call_id_shape(tc["id"])
|
||||
assert disagg == coupled
|
||||
assert len(derender_choices) == len(coupled_choices)
|
||||
for ref, der in zip(coupled_choices, derender_choices):
|
||||
assert _logprob_tokens(der) == _logprob_tokens(ref), (ref, der)
|
||||
return coupled, coupled_choices, derender_choices
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_reasoning(client):
|
||||
"""Per-chunk logprobs on the parser path match the coupled stream across
|
||||
the reasoning -> content transition."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
|
||||
client, messages, include_reasoning=True, max_tokens=256
|
||||
)
|
||||
if not coupled["reasoning"]:
|
||||
pytest.skip("Model did not emit a <think> block")
|
||||
assert any(
|
||||
ch["delta"].get("reasoning") and ch.get("logprobs") for ch in derender_choices
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_tool_call(client):
|
||||
"""Chunks the parser swallows (tool call markup) still carry their
|
||||
logprobs, on both sides, so no token's logprobs are lost."""
|
||||
messages = [{"role": "user", "content": "What's the weather in Paris?"}]
|
||||
coupled, _, derender_choices = await _run_stream_logprob_parity_case(
|
||||
client,
|
||||
messages,
|
||||
tools=TOOLS,
|
||||
tool_choice=FORCE_WEATHER_TOOL,
|
||||
include_reasoning=True,
|
||||
max_tokens=512,
|
||||
)
|
||||
if not coupled["tool_calls"]:
|
||||
pytest.skip("Model did not emit a tool call")
|
||||
swallowed = [
|
||||
ch
|
||||
for ch in derender_choices
|
||||
if ch.get("logprobs")
|
||||
and not ch["delta"].get("content")
|
||||
and not ch["delta"].get("reasoning")
|
||||
and not ch["delta"].get("tool_calls")
|
||||
]
|
||||
assert swallowed, "expected at least one buffered-markup chunk with logprobs"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parity_logprobs_hidden_reasoning(client):
|
||||
"""With `include_reasoning=False` and a parser configured, neither side
|
||||
returns logprobs on any chunk; derender still emits the content."""
|
||||
messages = [{"role": "user", "content": "What is 17 times 23? Think it through."}]
|
||||
extra = {"logprobs": True, "max_tokens": 256}
|
||||
# Token ids come from a shown-reasoning run: the coupled server hides
|
||||
# them along with the logprobs when reasoning is hidden.
|
||||
shown_choices, prompt_token_ids = await _coupled_stream(
|
||||
client, messages, include_reasoning=True, **extra
|
||||
)
|
||||
hidden_choices, _ = await _coupled_stream(
|
||||
client, messages, require_token_ids=False, include_reasoning=False, **extra
|
||||
)
|
||||
assert all(ch.get("logprobs") is None for ch in hidden_choices)
|
||||
assert assemble_stream(hidden_choices)["reasoning"] is None
|
||||
|
||||
finish_reason = shown_choices[-1]["finish_reason"]
|
||||
derender_choices: list[dict] = []
|
||||
disagg = await stream_chat_derender(
|
||||
client,
|
||||
[tid for ch in shown_choices for tid in ch["token_ids"]],
|
||||
[len(ch["token_ids"]) for ch in shown_choices],
|
||||
{"model": MODEL, "messages": messages, "include_reasoning": False, **extra},
|
||||
len(prompt_token_ids),
|
||||
prompt_token_ids,
|
||||
finish_reason="stop" if finish_reason == "tool_calls" else finish_reason,
|
||||
logprobs=[_sampled_generate_logprobs(ch) for ch in shown_choices],
|
||||
raw_choices=derender_choices,
|
||||
)
|
||||
assert all(ch.get("logprobs") is None for ch in derender_choices)
|
||||
shown = assemble_stream(shown_choices)
|
||||
assert disagg["reasoning"] is None
|
||||
assert disagg["content"] == shown["content"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline text level: generate(output_mode="text") == derender(generate(...))
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1427,6 +1427,159 @@ class TestDerenderChatStreamParsed:
|
||||
assert delta.reasoning is None
|
||||
assert delta.content == "c"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprobs_attached_to_each_parsed_delta(
|
||||
self, parsed_derenderer, tokenizer
|
||||
):
|
||||
"""A chunk's logprobs ride on whatever delta the parser emits for it
|
||||
(reasoning, content or tool call), with token ids decoded, as on
|
||||
`/v1/chat/completions`."""
|
||||
chunks = [
|
||||
[_FakeParser.REASON],
|
||||
[_FakeParser.CONTENT],
|
||||
[_FakeParser.TOOL_START, _FakeParser.TOOL_ARG],
|
||||
]
|
||||
state = None
|
||||
seen = []
|
||||
for ids in chunks:
|
||||
chunk, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
ids, logprobs=_generate_logprobs(ids)
|
||||
),
|
||||
state=state,
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
choice = chunk.choices[0]
|
||||
assert choice.logprobs is not None and choice.logprobs.content is not None
|
||||
assert [e.token for e in choice.logprobs.content] == (
|
||||
convert_ids_list_to_tokens(tokenizer, ids)
|
||||
)
|
||||
seen.append(choice.delta)
|
||||
assert seen[0].reasoning == "r"
|
||||
assert seen[1].content == "c"
|
||||
assert (
|
||||
seen[2].tool_calls and seen[2].tool_calls[0].function.name == "get_weather"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_kwargs", "expected_candidates"),
|
||||
[({"logprobs": True, "top_logprobs": 2}, 2), ({}, 4)],
|
||||
)
|
||||
async def test_top_logprobs_follow_the_chat_cut(
|
||||
self, parsed_derenderer, request_kwargs, expected_candidates
|
||||
):
|
||||
"""The parsed path trims generate's candidate list to the request's
|
||||
`top_logprobs` like the plain path does, and keeps every candidate
|
||||
when the request does not ask for logprobs."""
|
||||
ids = [_FakeParser.CONTENT]
|
||||
logprobs = _generate_logprobs(ids)
|
||||
logprobs["content"][0]["top_logprobs"] = [
|
||||
{"token_id": tid, "logprob": -float(rank)}
|
||||
for rank, tid in enumerate([ids[0], 11, 12, 13])
|
||||
]
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(ids, logprobs=logprobs),
|
||||
chat_request=_chat_request(**request_kwargs),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
content = chunk.choices[0].logprobs.content
|
||||
assert len(content) == 1
|
||||
assert len(content[0].top_logprobs) == expected_candidates
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprobs_kept_when_parser_swallows_chunk(
|
||||
self, parsed_derenderer, tokenizer
|
||||
):
|
||||
"""A chunk the parser emits nothing for (buffered markup) still
|
||||
carries its logprobs on an empty delta, so no token's logprobs are
|
||||
lost. Mirrors the coupled endpoint, which emits a chunk whenever
|
||||
logprobs are present even if the delta is empty."""
|
||||
swallowed = [_FakeParser.CONTENT + 1] # unknown to _FakeParser -> None
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
swallowed, logprobs=_generate_logprobs(swallowed)
|
||||
),
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
choice = chunk.choices[0]
|
||||
assert choice.delta.content is None and not choice.delta.tool_calls
|
||||
assert [e.token for e in choice.logprobs.content] == (
|
||||
convert_ids_list_to_tokens(tokenizer, swallowed)
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_false_drops_logprobs_on_every_chunk(
|
||||
self, parsed_derenderer
|
||||
):
|
||||
"""Hidden reasoning suppresses logprobs on reasoning *and* content
|
||||
chunks (decoded logprob tokens would leak the reasoning), the same
|
||||
rule `/v1/chat/completions` applies with a parser configured."""
|
||||
state = None
|
||||
for ids in ([_FakeParser.REASON], [_FakeParser.CONTENT]):
|
||||
chunk, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
ids, logprobs=_generate_logprobs(ids)
|
||||
),
|
||||
state=state,
|
||||
chat_request=_chat_request(include_reasoning=False),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
assert chunk.choices[0].logprobs is None
|
||||
assert chunk.choices[0].delta.content == "c"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprobs_multibyte_across_chunks(self, parsed_derenderer, tokenizer):
|
||||
"""Byte-fallback correction crosses chunk boundaries on the parser
|
||||
path too, via the carried `logprob_context_token_ids`, which starts
|
||||
empty like the engine's and never includes the prompt."""
|
||||
token_ids = tokenizer.encode("👍", add_special_tokens=False)
|
||||
if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]):
|
||||
pytest.skip("Tokenizer does not byte-split this character")
|
||||
state = None
|
||||
last_chunk = None
|
||||
for tid in token_ids:
|
||||
last_chunk, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[tid], logprobs=_generate_logprobs([tid])
|
||||
),
|
||||
state=state,
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2, 3, 4, 5],
|
||||
)
|
||||
assert last_chunk is not None and state is not None
|
||||
final_entry = last_chunk.choices[0].logprobs.content[0]
|
||||
assert not final_entry.token.endswith("�"), final_entry.token
|
||||
assert state.logprob_context_token_ids == token_ids[-4:]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logprob_context_advances_without_logprobs(self, parsed_derenderer):
|
||||
"""The context tail tracks sampled ids whether or not the chunk
|
||||
carries logprobs, so a later chunk that does can still be corrected."""
|
||||
ids = [_FakeParser.REASON, _FakeParser.CONTENT, _FakeParser.CONTENT]
|
||||
_, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(ids),
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
assert state.logprob_context_token_ids == ids
|
||||
_, state = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk([], finish_reason="stop"),
|
||||
state=state,
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=[1, 2],
|
||||
)
|
||||
assert state.logprob_context_token_ids == ids
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Harmony / GPT-OSS replay — unit, no server
|
||||
|
||||
@@ -52,6 +52,8 @@ async def stream_chat_derender(
|
||||
prompt_token_ids: list[int],
|
||||
finish_reason: str = "stop",
|
||||
on_chunk: Callable[[list[dict]], None] | None = None,
|
||||
logprobs: list[dict | None] | None = None,
|
||||
raw_choices: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Feed `output_ids` through the streaming chat derender endpoint in
|
||||
the given `chunk_sizes`, threading `stream_state` across calls and
|
||||
@@ -64,7 +66,13 @@ async def stream_chat_derender(
|
||||
`tool_calls` assembled so far, letting callers assert properties of the
|
||||
intermediate deltas (e.g. monotonic argument growth) rather than only
|
||||
the final assembled result.
|
||||
|
||||
`logprobs` supplies one `GenerateLogProbs` payload (or `None`) per chunk
|
||||
and `raw_choices`, if given, collects every derendered choice so callers
|
||||
can compare per-chunk fields such as `logprobs`.
|
||||
"""
|
||||
if logprobs is not None:
|
||||
assert len(logprobs) == len(chunk_sizes)
|
||||
state = None
|
||||
choices: list[dict] = []
|
||||
pos = 0
|
||||
@@ -84,6 +92,7 @@ async def stream_chat_derender(
|
||||
"index": 0,
|
||||
"token_ids": tids,
|
||||
"finish_reason": finish_reason if is_last else None,
|
||||
"logprobs": logprobs[i] if logprobs is not None else None,
|
||||
}
|
||||
],
|
||||
},
|
||||
@@ -101,6 +110,8 @@ async def stream_chat_derender(
|
||||
assert data["chunk"]["choices"][0]["token_ids"] == tids
|
||||
state = data["stream_state"]
|
||||
choices.extend(data["chunk"]["choices"])
|
||||
if raw_choices is not None:
|
||||
raw_choices.extend(data["chunk"]["choices"])
|
||||
if on_chunk is not None:
|
||||
on_chunk(assemble_stream(choices)["tool_calls"])
|
||||
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
model_name: "deepseek-ai/DeepSeek-R1"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--data-parallel-size 8
|
||||
--enable-expert-parallel
|
||||
--attention-backend=TRITON_ATTN
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -1,12 +0,0 @@
|
||||
model_name: "deepseek-ai/DeepSeek-R1"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 8
|
||||
--enable-expert-parallel
|
||||
--attention-backend=TRITON_ATTN
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -1,12 +0,0 @@
|
||||
model_name: "deepseek-ai/DeepSeek-V3.2"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--data-parallel-size 8
|
||||
--enable-expert-parallel
|
||||
--attention-backend=TRITON_ATTN
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -1,12 +0,0 @@
|
||||
model_name: "deepseek-ai/DeepSeek-V3.2"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 8
|
||||
--enable-expert-parallel
|
||||
--attention-backend=TRITON_ATTN
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -12,4 +12,5 @@ server_args: >-
|
||||
--enable-expert-parallel
|
||||
--mamba-backend triton
|
||||
--moe-backend triton
|
||||
--attention-backend ROCM_AITER_FA
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
||||
|
||||
@@ -12,4 +12,5 @@ server_args: >-
|
||||
--enable-expert-parallel
|
||||
--mamba-backend triton
|
||||
--moe-backend triton
|
||||
--attention-backend ROCM_AITER_FA
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
DeepSeek-R1-TP_MI325.yaml
|
||||
DeepSeek-R1-DP_MI325.yaml
|
||||
DeepSeek-V3.2-TP_MI325.yaml
|
||||
DeepSeek-V3.2-DP_MI325.yaml
|
||||
Qwen3-30B-A3B-NVFP4.yaml
|
||||
Qwen3-30B-A3B-NVFP4-quark.yaml
|
||||
Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
|
||||
Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
|
||||
@@ -353,46 +353,6 @@ def test_silu_and_mul_with_clamp(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_masked_situ_and_mul(
|
||||
default_vllm_config,
|
||||
linear_beta: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Masked SITU computes valid expert rows and preserves padded zeros."""
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts, max_num_tokens, d = 4, 7, 512
|
||||
beta = 1.5
|
||||
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
|
||||
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
|
||||
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
|
||||
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
output, input, expert_num_tokens, beta, linear_beta
|
||||
)
|
||||
|
||||
gate, up = input.float().chunk(2, dim=-1)
|
||||
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
if linear_beta > 0:
|
||||
up = linear_beta * torch.tanh(up / linear_beta)
|
||||
expected = (expected * up).to(dtype)
|
||||
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
|
||||
torch.testing.assert_close(
|
||||
output[expert, :num_tokens],
|
||||
expected[expert, :num_tokens],
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.masked_situ_and_mul,
|
||||
(output, input, expert_num_tokens, beta, linear_beta),
|
||||
)
|
||||
|
||||
|
||||
MOE_ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
|
||||
pytest.param(
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -358,3 +360,113 @@ def test_causal_conv1d_varlen(
|
||||
)
|
||||
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
|
||||
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [True])
|
||||
@pytest.mark.parametrize("has_bias", [True])
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("dim", [64, 2048])
|
||||
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
|
||||
)
|
||||
def test_causal_conv1d_update_ragged_varlen(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
mk_inputs = lambda: _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
)
|
||||
inputs, state = mk_inputs()
|
||||
actual = causal_conv1d_update(conv_state=state, **inputs)
|
||||
|
||||
ref_inputs, ref_state = mk_inputs()
|
||||
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
|
||||
|
||||
for i, ref_out in enumerate(ref_outs):
|
||||
start = int(inputs["query_start_loc"][i])
|
||||
end = int(inputs["query_start_loc"][i + 1])
|
||||
entry = inputs["conv_state_indices"][i : i + 1]
|
||||
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
|
||||
assert torch.equal(state[entry], ref_state[entry])
|
||||
|
||||
|
||||
def _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
set_random_seed(0)
|
||||
|
||||
batch_size = len(q_lens_cfg)
|
||||
max_query_len = 5
|
||||
assert max(q_lens_cfg) <= max_query_len
|
||||
state_len = width - 1 + max_query_len
|
||||
total_entries = 10 * batch_size
|
||||
|
||||
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
|
||||
]
|
||||
)
|
||||
total_tokens = int(q_lens.sum())
|
||||
|
||||
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
|
||||
conv_state = torch.randn(
|
||||
total_entries, state_len, dim, device=device, dtype=itype
|
||||
).transpose(1, 2)
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
activation = None if not silu_activation else "silu"
|
||||
|
||||
conv_state_indices = (
|
||||
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
|
||||
).to(torch.int32)
|
||||
num_accepted_tokens = torch.randint(
|
||||
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
|
||||
)
|
||||
inputs = dict(
|
||||
x=x,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation=activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
return inputs, conv_state
|
||||
|
||||
|
||||
def _create_ragged_varlen_ref_outs(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation,
|
||||
conv_state_indices,
|
||||
num_accepted_tokens,
|
||||
query_start_loc,
|
||||
max_query_len,
|
||||
) -> Iterator[torch.Tensor]:
|
||||
for seq_idx in range(len(conv_state_indices)):
|
||||
start = int(query_start_loc[seq_idx])
|
||||
end = int(query_start_loc[seq_idx + 1])
|
||||
entry = conv_state_indices[seq_idx : seq_idx + 1]
|
||||
if start == end:
|
||||
yield x[start:end]
|
||||
continue
|
||||
out_ref = causal_conv1d_update(
|
||||
x[start:end],
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation=activation,
|
||||
conv_state_indices=entry,
|
||||
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
|
||||
query_start_loc=query_start_loc.new_tensor([0, end - start]),
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
yield out_ref
|
||||
|
||||
@@ -6,6 +6,10 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange, repeat
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
|
||||
_chunk_scan_fwd,
|
||||
_chunk_scan_fwd_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -578,3 +582,46 @@ def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
|
||||
rtol=rtol,
|
||||
msg=lambda x, i=i: f"seq{i} state " + x,
|
||||
)
|
||||
|
||||
|
||||
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
|
||||
"""The autotuner picks a config by speed only, so every config in the list
|
||||
must give the same finite output. On sm90, configs that pipeline a wgmma
|
||||
with `cb` in registers wrote NaN into whole rows (#61603)."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
set_random_seed(0)
|
||||
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
|
||||
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
|
||||
seqlen, nchunks = 181, 2
|
||||
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
|
||||
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
|
||||
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
|
||||
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
|
||||
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
|
||||
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
|
||||
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
|
||||
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
|
||||
|
||||
ref = None
|
||||
for config in list(_chunk_scan_fwd_kernel.configs):
|
||||
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
|
||||
for _ in range(10):
|
||||
out = torch.empty_like(x)
|
||||
try:
|
||||
_chunk_scan_fwd(
|
||||
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
|
||||
)
|
||||
except OutOfResources:
|
||||
# The autotuner skips configs that do not fit this device.
|
||||
break
|
||||
assert torch.isfinite(out).all(), f"non-finite output with {config}"
|
||||
if ref is None:
|
||||
ref = out
|
||||
torch.testing.assert_close(
|
||||
out,
|
||||
ref,
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
msg=lambda m, config=config: f"{config}: {m}",
|
||||
)
|
||||
|
||||
@@ -10,7 +10,6 @@ import torch.nn.functional as F
|
||||
import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.layers.mhc as mhc_layers
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
_torch_hc_prenorm_gemm,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_delayed_tilelang,
|
||||
)
|
||||
@@ -1002,6 +1001,19 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
|
||||
torch.testing.assert_close(actual, expected, atol=5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _torch_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
) -> None:
|
||||
assert out.shape[0] == 1
|
||||
assert sqrsum.shape[0] == 1
|
||||
x_float = x.float()
|
||||
out[0].copy_(x_float @ fn.t())
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TILELANG_MHC,
|
||||
reason="TileLang MHC support required",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Concatenate
|
||||
@@ -23,6 +24,8 @@ from vllm.utils.network_utils import get_open_port
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
LAUNCH_TIMEOUT_S = 900
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ProcessGroupInfo:
|
||||
@@ -148,7 +151,7 @@ def parallel_launch_with_config(
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> None:
|
||||
spawn(
|
||||
context = spawn(
|
||||
_worker_parallel_launch,
|
||||
args=(
|
||||
world_size,
|
||||
@@ -162,5 +165,16 @@ def parallel_launch_with_config(
|
||||
)
|
||||
+ args,
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
join=False,
|
||||
)
|
||||
# A worker stuck in communicator setup never exits, so bound the wait.
|
||||
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
|
||||
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
|
||||
if time.monotonic() >= deadline:
|
||||
for process in context.processes:
|
||||
process.kill()
|
||||
for process in context.processes:
|
||||
process.join()
|
||||
raise TimeoutError(
|
||||
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
|
||||
)
|
||||
|
||||
@@ -127,6 +127,14 @@ if has_nixl_ep():
|
||||
BACKENDS += ["nixl_ep"]
|
||||
|
||||
DEEPEP_BACKENDS = {"deepep_high_throughput", "deepep_low_latency"}
|
||||
|
||||
# These tests run on one node, where NVSHMEM reaches peers over NVLink, so
|
||||
# keep it off the NICs to avoid depending on IB setup on shared CI hosts.
|
||||
DEEPEP_SINGLE_NODE_NVSHMEM_ENV = {
|
||||
"NVSHMEM_REMOTE_TRANSPORT": "none",
|
||||
# DeepEP forces IBGDA on, so also hide the NICs from it.
|
||||
"NVSHMEM_HCA_LIST": "none",
|
||||
}
|
||||
MORI_BACKENDS = {"mori_high_throughput", "mori_low_latency"}
|
||||
|
||||
QUANT_METHODS = [
|
||||
@@ -2187,6 +2195,8 @@ def test_moe_layer(
|
||||
|
||||
rocm_deepep = current_platform.is_rocm() and backend in DEEPEP_BACKENDS
|
||||
parallel_worker = _parallel_worker_rocm_deepep if rocm_deepep else _parallel_worker
|
||||
cuda_deepep = current_platform.is_cuda() and backend in DEEPEP_BACKENDS
|
||||
env_dict = DEEPEP_SINGLE_NODE_NVSHMEM_ENV if cuda_deepep else None
|
||||
launch_failures: list[str] = []
|
||||
|
||||
try:
|
||||
@@ -2197,7 +2207,7 @@ def test_moe_layer(
|
||||
world_size,
|
||||
parallel_worker,
|
||||
vllm_config,
|
||||
None,
|
||||
env_dict,
|
||||
test_config_batch,
|
||||
verbosity,
|
||||
failure_report_path=failure_report_path,
|
||||
|
||||
@@ -582,6 +582,20 @@ def test_fused_topk_bias(
|
||||
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scoring_func", [0, 1])
|
||||
def test_fused_grouped_topk_accepts_idle_rank(scoring_func):
|
||||
"""Zero-token routing must leave CUDA ready for the next batch."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("CUDA required")
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
scores = torch.empty((0, 8), device="cuda", dtype=torch.float32)
|
||||
bias = torch.zeros(8, device="cuda")
|
||||
weights, ids = ops.grouped_topk(scores, 2, 1, 2, True, 1.0, bias, scoring_func)
|
||||
torch.testing.assert_close(weights, scores.new_empty((0, 2)))
|
||||
torch.testing.assert_close(ids, scores.new_empty((0, 2), dtype=torch.int32))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,k", MK_S)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
||||
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 2048])
|
||||
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
|
||||
def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
counts: list[int],
|
||||
group_size: int,
|
||||
@@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
x *= 16
|
||||
clean_x = x.clone()
|
||||
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
|
||||
output_dtype = (
|
||||
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
|
||||
)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
|
||||
|
||||
def set_counts(values):
|
||||
positions = []
|
||||
@@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
output.view(torch.uint8).fill_(91)
|
||||
graph.replay()
|
||||
reference_q, reference_s = quantize(x)
|
||||
assert actual_q.dtype == reference_q.dtype
|
||||
torch.testing.assert_close(
|
||||
actual_q.view(torch.uint8)[live],
|
||||
reference_q.view(torch.uint8)[live],
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
|
||||
|
||||
Supported (hidden_size, num_experts) pairs:
|
||||
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)
|
||||
|
||||
Correctness baseline: F.linear in float32. Every M in [1, 32] is covered so
|
||||
Correctness baseline: F.linear in float32. Every supported M is covered so
|
||||
all tuned geometries (wide-block, experts-per-block, token-group; boundaries
|
||||
at M=4/5, odd/even, M=15/16) are exercised on Blackwell, and the legacy
|
||||
at M=4/5, M=9/10, M=12/13, odd/even, M=15/16) are exercised on Blackwell, and the legacy
|
||||
128/1 geometry everywhere else.
|
||||
"""
|
||||
|
||||
@@ -18,8 +18,12 @@ from vllm._custom_ops import fp32_router_gemm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# (hidden_size, num_experts)
|
||||
SHAPES = [(3072, 256), (6144, 128)]
|
||||
ALL_M = list(range(1, 33))
|
||||
SHAPES = [(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)]
|
||||
CASES = [
|
||||
(m, hidden, experts)
|
||||
for hidden, experts in SHAPES
|
||||
for m in range(1, 49 if hidden == 5120 else 33)
|
||||
]
|
||||
# Absolute tolerance for fp32 kernel vs float64 reference
|
||||
ATOL_FP32 = 2e-4
|
||||
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
|
||||
@@ -40,8 +44,7 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
|
||||
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", ALL_M)
|
||||
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
|
||||
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
"""fp32 activation → fp32 output should match reference closely."""
|
||||
_requires_sm90()
|
||||
@@ -58,8 +61,7 @@ def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", ALL_M)
|
||||
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
|
||||
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
"""bf16 activation → fp32 output should match reference within bf16 error."""
|
||||
_requires_sm90()
|
||||
@@ -124,11 +126,11 @@ def test_topk_routing_consistency(num_tokens: int, hidden_dim: int, num_experts:
|
||||
)
|
||||
|
||||
|
||||
def test_zero_tokens_returns_empty():
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
def test_zero_tokens_returns_empty(hidden_dim, num_experts):
|
||||
"""M=0 is a graceful no-op returning an empty [0, E] tensor."""
|
||||
_requires_sm90()
|
||||
device = torch.device("cuda")
|
||||
hidden_dim, num_experts = SHAPES[0]
|
||||
mat_a = torch.empty(0, hidden_dim, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
out = fp32_router_gemm(mat_a, mat_b)
|
||||
@@ -136,17 +138,19 @@ def test_zero_tokens_returns_empty():
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rejects_invalid_inputs():
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
def test_rejects_invalid_inputs(hidden_dim, num_experts):
|
||||
"""The entry must fail loudly, never compute silently wrong results."""
|
||||
_requires_sm90()
|
||||
device = torch.device("cuda")
|
||||
hidden_dim, num_experts = SHAPES[0]
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
|
||||
# num_tokens > 32 (beyond the instantiated range)
|
||||
# Beyond the instantiated range for this shape.
|
||||
invalid_tokens = 49 if hidden_dim == 5120 else 33
|
||||
with pytest.raises(Exception, match="num_tokens"):
|
||||
fp32_router_gemm(
|
||||
torch.randn(33, hidden_dim, dtype=torch.float32, device=device), mat_b
|
||||
torch.randn(invalid_tokens, hidden_dim, dtype=torch.float32, device=device),
|
||||
mat_b,
|
||||
)
|
||||
|
||||
# unsupported (hidden_dim, num_experts) pair
|
||||
@@ -173,3 +177,41 @@ def test_rejects_invalid_inputs():
|
||||
fp32_router_gemm(
|
||||
torch.randn(4, hidden_dim, dtype=torch.float16, device=device), mat_b
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"tokens", [*range(1, 17), 17, 24, 32, 33, 40, 48, 56, 64, 65, 128]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_dim,num_experts", [(5120, 256), (4096, 352), (8192, 256)]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_router_retains_fp32_weights_and_logits(
|
||||
default_vllm_config, dist_init, tokens, hidden_dim, num_experts
|
||||
):
|
||||
"""Low-latency routing must retain FP32 weight precision and logits."""
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("SM100 required")
|
||||
torch.manual_seed(42)
|
||||
router = GateLinear(
|
||||
hidden_dim, num_experts, params_dtype=torch.float32, out_dtype=torch.float32
|
||||
).cuda()
|
||||
router.weight.normal_(std=0.01)
|
||||
x = torch.randn(tokens, hidden_dim, device="cuda", dtype=torch.bfloat16)
|
||||
expected = (x.double() @ router.weight.double().T).float()
|
||||
actual, bias = router(x)
|
||||
assert bias is None and actual.dtype == torch.float32
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-5)
|
||||
torch.testing.assert_close(actual.topk(8).indices, expected.topk(8).indices)
|
||||
if tokens in (1, 16, 32, 33, 48, 56):
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured, _ = router(x)
|
||||
x.normal_()
|
||||
graph.replay()
|
||||
expected = (x.double() @ router.weight.double().T).float()
|
||||
torch.testing.assert_close(captured, expected, rtol=2e-5, atol=2e-5)
|
||||
|
||||
@@ -566,7 +566,7 @@ def test_invalid_output_dtype():
|
||||
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.bfloat16)
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.float16)
|
||||
|
||||
|
||||
def test_cache_miss_compiles_dotprod():
|
||||
@@ -581,5 +581,29 @@ def test_cache_miss_compiles_dotprod():
|
||||
_assert_close(out, _ref(a, b), context="cache miss dotprod")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 2, 8, 16])
|
||||
@pytest.mark.parametrize("N", [1792, 3584])
|
||||
@torch.inference_mode()
|
||||
def test_bf16_output_and_fp32_cache_coexist(M, N):
|
||||
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
|
||||
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
|
||||
for dtype in (torch.float32, torch.bfloat16):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
|
||||
assert actual.dtype == dtype
|
||||
reference = (x.double() @ w.double().T).to(dtype)
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
|
||||
x.normal_()
|
||||
graph.replay()
|
||||
reference = (x.double() @ w.double().T).bfloat16()
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from inspect import signature
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
@@ -12,8 +12,10 @@ from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup import kernel_warmup as warmup
|
||||
import vllm.model_executor.warmup.kernel_warmup as warmup
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.model_executor.warmup.kernel_warmup import (
|
||||
_flashinfer_autotune_token_counts,
|
||||
_run_flashinfer_autotune_dummy_runs,
|
||||
@@ -45,7 +47,12 @@ def _make_runner(modules, *, max_tokens: int = 8192):
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
parallel_config=SimpleNamespace(data_parallel_rank=0),
|
||||
parallel_config=SimpleNamespace(
|
||||
enable_elastic_ep=False,
|
||||
nnodes=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
),
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
@@ -133,6 +140,258 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"has_v2,elastic_ep,node_count",
|
||||
[(False, False, 1), (True, False, 1), (True, True, 1), (True, False, 2)],
|
||||
ids=["v1", "v2", "elastic-v1", "multi-node-v1"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"hisparse,world_size,rank,skip_ops,failure",
|
||||
[
|
||||
(False, 1, 0, [], None),
|
||||
(False, 2, 0, [], None),
|
||||
(False, 2, 1, [], None),
|
||||
(True, 2, 0, ["fp4_gemm", "bmm_fp8"], None),
|
||||
(False, 2, 1, ["fp4_gemm"], "dummy"),
|
||||
(False, 2, 1, [], "replayssm"),
|
||||
(True, 2, 0, [], "kimi"),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_autotune_lifecycle(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
has_v2,
|
||||
elastic_ep,
|
||||
node_count,
|
||||
hisparse,
|
||||
world_size,
|
||||
rank,
|
||||
skip_ops,
|
||||
failure,
|
||||
):
|
||||
use_v2 = has_v2 and not elastic_ep and node_count == 1 and world_size == 1
|
||||
runner = _make_runner([])
|
||||
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
|
||||
runner.vllm_config.attention_config.hisparse_config = object() if hisparse else None
|
||||
model = runner.get_model()
|
||||
cache_path = tmp_path / "autotune.json"
|
||||
rank_cache_path = cache_path.with_name(f"autotune_dp0_rank{rank}.json")
|
||||
rank_cache_path.write_text(json.dumps({"_metadata": {}, "op": 1}))
|
||||
events = Mock()
|
||||
events.tuner.load_configs.return_value = True
|
||||
events.tuner._dirty = True
|
||||
world = SimpleNamespace(
|
||||
rank_in_group=rank,
|
||||
world_size=world_size,
|
||||
cpu_group=object(),
|
||||
broadcast_object=events.broadcast,
|
||||
barrier=events.barrier,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def autotune_context(**kwargs):
|
||||
assert torch.is_inference_mode_enabled()
|
||||
events.enter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.exit()
|
||||
|
||||
events.autotune.side_effect = autotune_context
|
||||
if failure is not None:
|
||||
getattr(events, failure).side_effect = RuntimeError("warmup failed")
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer.autotuner",
|
||||
SimpleNamespace(
|
||||
AutoTuner=SimpleNamespace(get=lambda: events.tuner),
|
||||
set_autotune_process_group=events.set_group,
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer",
|
||||
SimpleNamespace(autotune_v2=events.autotune, autotune_v2_reload=events.reload),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_world_group", lambda: world
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_pp_group",
|
||||
lambda: SimpleNamespace(world_size=1),
|
||||
)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: has_v2)
|
||||
monkeypatch.setattr(fi_utils, "autotune", events.autotune)
|
||||
monkeypatch.setattr(warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", skip_ops)
|
||||
resolve_file = Mock(return_value=cache_path)
|
||||
resolve_root = Mock(return_value=str(tmp_path))
|
||||
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_file", resolve_file)
|
||||
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_v2_root", resolve_root)
|
||||
|
||||
def gather(out, value, *, group):
|
||||
out[:] = [value] * world_size
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "all_gather_object", gather)
|
||||
monkeypatch.setattr(warmup, "_run_flashinfer_autotune_dummy_runs", events.dummy)
|
||||
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", events.replayssm)
|
||||
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", events.kimi)
|
||||
monkeypatch.setattr(
|
||||
warmup, "autotune_hisparse_flashinfer_attention", events.hisparse
|
||||
)
|
||||
|
||||
if failure is None:
|
||||
warmup.flashinfer_autotune(runner)
|
||||
else:
|
||||
with pytest.raises(RuntimeError, match="warmup failed"):
|
||||
warmup.flashinfer_autotune(runner)
|
||||
|
||||
kwargs = {"skip_ops": set(skip_ops)} if skip_ops else {}
|
||||
expected = []
|
||||
if use_v2:
|
||||
resolve_file.assert_not_called()
|
||||
resolve_root.assert_called_once_with()
|
||||
expected.append(call.autotune(mode="tune", cache_root=str(tmp_path), **kwargs))
|
||||
else:
|
||||
resolve_root.assert_not_called()
|
||||
resolve_file.assert_called_once_with(runner)
|
||||
expected.extend(
|
||||
[
|
||||
call.tuner.load_configs(str(rank_cache_path)),
|
||||
call.autotune(tune_mode=True, **kwargs),
|
||||
]
|
||||
)
|
||||
expected.extend(
|
||||
[call.set_group(world.cpu_group if world_size > 1 else None), call.enter()]
|
||||
)
|
||||
if hisparse:
|
||||
expected.append(call.hisparse(runner))
|
||||
for name, arg, kwargs in [
|
||||
("dummy", runner, {"skip_attn": hisparse}),
|
||||
("replayssm", runner, {}),
|
||||
("kimi", model, {}),
|
||||
]:
|
||||
expected.append(getattr(call, name)(arg, **kwargs))
|
||||
if failure == name:
|
||||
break
|
||||
expected.append(call.exit())
|
||||
expected.append(call.set_group(None))
|
||||
if failure is None:
|
||||
if world_size > 1:
|
||||
expected.append(call.barrier())
|
||||
if use_v2 and world_size > 1:
|
||||
expected.extend([call.reload(), call.barrier()])
|
||||
elif not use_v2:
|
||||
expected.append(call.tuner.save_configs(str(rank_cache_path)))
|
||||
assert events.mock_calls == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"node_count,elastic_ep,pp_size",
|
||||
[(1, False, 1), (2, False, 1), (1, True, 1), (1, False, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("world_size,rank", [(1, 0), (2, 0), (2, 1)])
|
||||
@pytest.mark.parametrize("model_runner_v2,max_reqs", [(False, 2), (True, 1), (True, 2)])
|
||||
def test_sparse_autotune_respects_cache_selection(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
node_count,
|
||||
elastic_ep,
|
||||
pp_size,
|
||||
world_size,
|
||||
rank,
|
||||
model_runner_v2,
|
||||
max_reqs,
|
||||
):
|
||||
import vllm.model_executor.warmup.flashinfer_sparse_mla_warmup as sparse
|
||||
|
||||
runner = _make_runner([])
|
||||
runner.max_num_reqs = max_reqs
|
||||
runner.vllm_config.use_v2_model_runner = model_runner_v2
|
||||
runner.vllm_config.kernel_config = SimpleNamespace(enable_flashinfer_autotune=True)
|
||||
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
|
||||
runner.vllm_config.parallel_config.pipeline_parallel_size = pp_size
|
||||
worker = SimpleNamespace(
|
||||
model_runner=runner,
|
||||
vllm_config=runner.vllm_config,
|
||||
execute_model=Mock(),
|
||||
sample_tokens=Mock(),
|
||||
)
|
||||
world = SimpleNamespace(
|
||||
rank_in_group=rank,
|
||||
world_size=world_size,
|
||||
broadcast_object=Mock(return_value=b"cached"),
|
||||
barrier=Mock(),
|
||||
)
|
||||
managed = Mock(return_value=nullcontext())
|
||||
legacy = Mock(return_value=nullcontext())
|
||||
reload = Mock()
|
||||
tuner = Mock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer",
|
||||
SimpleNamespace(autotune_v2=managed, autotune_v2_reload=reload),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer.autotuner",
|
||||
SimpleNamespace(AutoTuner=SimpleNamespace(get=lambda: tuner)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_world_group", lambda: world
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
|
||||
)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: True)
|
||||
monkeypatch.setattr(sparse, "has_flashinfer", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
sparse.current_platform, "is_device_capability_family", lambda _: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse, "_flashinfer_sparse_mla_decode_label", lambda *_: "DSv3.2"
|
||||
)
|
||||
monkeypatch.setattr(sparse, "resolve_flashinfer_autotune_v2_root", lambda: tmp_path)
|
||||
cache_path = tmp_path / "autotune.json"
|
||||
cache_path.write_bytes(b"cached")
|
||||
monkeypatch.setattr(
|
||||
sparse, "resolve_flashinfer_autotune_file", lambda _: cache_path
|
||||
)
|
||||
monkeypatch.setattr(sparse, "flashinfer_autotune", legacy)
|
||||
|
||||
def mixed_warmup(*args, mixed_step_context=None, **kwargs):
|
||||
context = (
|
||||
mixed_step_context if mixed_step_context is not None else nullcontext()
|
||||
)
|
||||
with context:
|
||||
assert torch.is_inference_mode_enabled()
|
||||
return True
|
||||
|
||||
mixed = Mock(side_effect=mixed_warmup)
|
||||
monkeypatch.setattr(sparse, "run_mixed_prefill_decode_warmup", mixed)
|
||||
assert sparse._run_flashinfer_sparse_mla_decode_autotune(worker, 16, frozenset())
|
||||
|
||||
if node_count == 1 and not elastic_ep and pp_size == 1 and world_size == 1:
|
||||
managed.assert_called_once_with(mode="tune", cache_root=tmp_path)
|
||||
legacy.assert_not_called()
|
||||
world.broadcast_object.assert_not_called()
|
||||
tuner.load_configs.assert_not_called()
|
||||
reload.assert_not_called()
|
||||
world.barrier.assert_not_called()
|
||||
else:
|
||||
managed.assert_not_called()
|
||||
reload.assert_not_called()
|
||||
assert legacy.call_count == (1 if rank == 0 else 0)
|
||||
world.broadcast_object.assert_called_once_with(
|
||||
b"cached" if rank == 0 else None, src=0
|
||||
)
|
||||
tuner.load_configs.assert_called_once_with(str(cache_path))
|
||||
assert mixed.call_count == (1 if model_runner_v2 and max_reqs >= 2 else 0)
|
||||
assert runner._dummy_run.call_count == (0 if mixed.called else 1)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
|
||||
"""The drafter's M can exceed the pass size; buckets must include it."""
|
||||
runner = _make_runner([])
|
||||
@@ -266,7 +525,9 @@ class _AutotuneRun:
|
||||
for rank in range(self.pp * self.tp):
|
||||
self.rank = rank
|
||||
self.tuners[rank] = _AutotuneTuner(self)
|
||||
flashinfer_autotune(_make_runner([]))
|
||||
runner = _make_runner([])
|
||||
runner.vllm_config.parallel_config.pipeline_parallel_size = self.pp
|
||||
flashinfer_autotune(runner)
|
||||
return self
|
||||
|
||||
def assert_collectives_match(self):
|
||||
@@ -283,6 +544,8 @@ def autotune_run(monkeypatch, tmp_path):
|
||||
|
||||
def make_run(*, pp=2, tp=4):
|
||||
run = _AutotuneRun(pp, tp, tmp_path)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: pp > 1)
|
||||
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
|
||||
autotuner = ModuleType("flashinfer.autotuner")
|
||||
monkeypatch.setattr(
|
||||
autotuner,
|
||||
|
||||
@@ -168,23 +168,28 @@ def _warmup_store_kernel(out, VALUE: tl.constexpr):
|
||||
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
|
||||
reason="Requires CUDA Triton async compilation",
|
||||
)
|
||||
def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
|
||||
def test_registered_warmup_overlaps_kernels_and_runtime_stays_serial(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from triton.runtime._async_compile import active_mode
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
|
||||
|
||||
def warmup_inputs():
|
||||
return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2))
|
||||
def make_store(value):
|
||||
def warmup_inputs():
|
||||
return dict(out=TritonWarmupTensor(torch.int32), value=value)
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
|
||||
)
|
||||
def store(out, value):
|
||||
return (1,), dict(VALUE=value)
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
|
||||
)
|
||||
def store(out, value):
|
||||
return (1,), dict(VALUE=value)
|
||||
|
||||
store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler.
|
||||
# Initialize Triton's binder before wrapping its compiler.
|
||||
store.get_warmup_keys()
|
||||
return store
|
||||
|
||||
stores = [make_store(value) for value in (1, 2)]
|
||||
caller = threading.get_ident()
|
||||
barrier = threading.Barrier(2, timeout=10)
|
||||
compiled_on = []
|
||||
@@ -200,14 +205,15 @@ def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
|
||||
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
|
||||
registry = JitWarmupRegistry(None)
|
||||
with registry.activate():
|
||||
store.register_warmup()
|
||||
for store in stores:
|
||||
store.register_warmup()
|
||||
registry.warmup()
|
||||
|
||||
assert len(compiled_on) == 2
|
||||
assert all(thread != caller for thread in compiled_on)
|
||||
assert active_mode.get() is None
|
||||
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
|
||||
for value in (1, 2):
|
||||
for store, value in zip(stores, (1, 2)):
|
||||
store(out, value)
|
||||
assert out.item() == value
|
||||
assert len(compiled_on) == 2
|
||||
|
||||
@@ -220,3 +220,44 @@ def test_real_triton_kernel_override_end_to_end() -> None:
|
||||
"out_ptr",
|
||||
"BLOCK",
|
||||
)
|
||||
|
||||
|
||||
def test_cpu_slot_mapping_override_forwards_kernel_arguments(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.utils import cpu_triton_utils
|
||||
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
|
||||
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
fake_torch = types.SimpleNamespace(
|
||||
ops=types.SimpleNamespace(
|
||||
_C=types.SimpleNamespace(
|
||||
compute_slot_mapping_kernel_impl=lambda *args: calls.append(args)
|
||||
)
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(cpu_triton_utils, "torch", fake_torch)
|
||||
|
||||
override = KernelOverride(
|
||||
ComputeSlotMappingKernel.kernel,
|
||||
cpu_triton_utils._compute_slot_mapping_kernel_impl,
|
||||
)
|
||||
override[(2,)](
|
||||
num_tokens=2,
|
||||
max_num_tokens=4,
|
||||
query_start_loc_ptr="query_start_loc",
|
||||
positions_ptr="positions",
|
||||
block_table_ptr="block_table",
|
||||
block_table_stride=128,
|
||||
slot_mapping_ptr="slot_mapping",
|
||||
KV_CACHE_BLOCK_SIZE=16,
|
||||
TOTAL_CP_WORLD_SIZE=1,
|
||||
TOTAL_CP_RANK=0,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=1,
|
||||
PAD_ID=-1,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
("query_start_loc", "positions", "block_table", "slot_mapping", 16)
|
||||
]
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
|
||||
|
||||
Adaptive verification splits the draft budget evenly on the host, then trims
|
||||
each verify request on device. The KDA spec path must therefore plan from the
|
||||
device offsets, never from host lengths that merely look uniform.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import (
|
||||
KimiK3ROCmKDABackend,
|
||||
KimiK3ROCmKDAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
NUM_SPEC = 3
|
||||
|
||||
|
||||
def _config(adaptive: bool):
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=NUM_SPEC,
|
||||
enable_adaptive_verification=adaptive,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = "none"
|
||||
return vllm_config
|
||||
|
||||
|
||||
def _spec() -> MambaSpec:
|
||||
return MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
)
|
||||
|
||||
|
||||
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
|
||||
return KimiK3ROCmKDAMetadataBuilder(
|
||||
kv_cache_spec=_spec(),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=_config(adaptive),
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
|
||||
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
|
||||
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
|
||||
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
|
||||
)
|
||||
common = common.replace(
|
||||
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
)
|
||||
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
|
||||
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
|
||||
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
|
||||
|
||||
|
||||
def test_adaptive_verification_plans_from_device_offsets():
|
||||
"""Equal host lengths would select the fixed-length recurrent kernel,
|
||||
which places sequence i at i * L and ignores the device cu_seqlens."""
|
||||
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
|
||||
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
|
||||
assert bound(_config(adaptive=False), _spec()) is None
|
||||
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=True))
|
||||
assert meta.num_spec_decodes == 2
|
||||
assert meta.uniform_spec_sequence_length is None
|
||||
assert torch.equal(
|
||||
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
)
|
||||
# Without adaptive verification the host lengths are exact and kept.
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=False))
|
||||
assert meta.uniform_spec_sequence_length == 3
|
||||
@@ -482,7 +482,7 @@ def _spec_for(monkeypatch, **attrs) -> MambaSpec:
|
||||
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
|
||||
for name, value in dict(
|
||||
use_prefill_checkpoint=True,
|
||||
use_fused_chunk=True,
|
||||
kda_prefill_backend="fused",
|
||||
num_spec=0,
|
||||
use_safe_gate=True,
|
||||
).items():
|
||||
@@ -506,7 +506,8 @@ def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
|
||||
"attr,value",
|
||||
[
|
||||
("use_prefill_checkpoint", False),
|
||||
("use_fused_chunk", False),
|
||||
("kda_prefill_backend", "triton"),
|
||||
("kda_prefill_backend", "gluon"),
|
||||
("use_safe_gate", False),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
use_fused_chunk=use_fused,
|
||||
backend=kw.pop("backend", "fused" if use_fused else "triton"),
|
||||
out=out,
|
||||
**kw,
|
||||
)
|
||||
@@ -112,6 +112,27 @@ def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[[1], [64], [1024], [4096], [513, 64, 1, 1200], [512] * 6],
|
||||
)
|
||||
def test_gluon_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
"""AITER's Gluon chunk_kda must match the Triton chunk path too."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_kda_gluon_prefill_available():
|
||||
pytest.skip("AITER Gluon chunk_kda is not available here")
|
||||
|
||||
inp = _inputs(seqlens, seed=len(seqlens))
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_gluon, state_gluon = _run(inp, use_fused=False, backend="gluon")
|
||||
|
||||
torch.testing.assert_close(o_gluon.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_gluon, state_ref, rtol=5e-3, atol=5e-3)
|
||||
state_rel = (state_gluon - state_ref).norm() / state_ref.norm()
|
||||
assert state_rel < 5e-3, f"final state relative L2 error {state_rel:.2e}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_out_argument_receives_the_result(use_fused: bool) -> None:
|
||||
"""``out`` is a promise about where the result lands.
|
||||
@@ -648,7 +669,7 @@ def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
|
||||
has_initial_state=warm,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
use_fused_chunk=use_fused,
|
||||
backend="fused" if use_fused else "triton",
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER fused conv1d is ROCm-only"
|
||||
)
|
||||
|
||||
WIDTH = 4 # Kimi-K3 short_conv_kernel_size
|
||||
STATE_LEN = WIDTH - 1
|
||||
HEAD_DIM = 128
|
||||
LOCAL_HEADS = 4 # per-TP-rank heads; projection = 512, packed dim = 1536
|
||||
|
||||
|
||||
def _fused_fn():
|
||||
gdn = pytest.importorskip("aiter.ops.triton.gated_delta_net")
|
||||
return gdn.causal_conv1d_split_qkv_triton_tile_fn
|
||||
|
||||
|
||||
def _make_inputs(seqlens, has_initial_state, seed=0, device="cuda"):
|
||||
torch.manual_seed(seed)
|
||||
proj = HEAD_DIM * LOCAL_HEADS
|
||||
dim = 3 * proj
|
||||
total = sum(seqlens)
|
||||
n_seqs = len(seqlens)
|
||||
# Token-major activations, as produced by the in_proj GEMM.
|
||||
mixed_qkv = torch.randn(total, dim, dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn(dim, WIDTH, dtype=torch.bfloat16, device=device) * 0.3
|
||||
# More cache lines than sequences, assigned in a shuffled order. Line 0 is
|
||||
# vLLM's null block (NULL_BLOCK_ID), which causal_conv1d_fn skips, so real
|
||||
# sequences never map to it.
|
||||
n_lines = n_seqs + 2
|
||||
conv_state = torch.randn(
|
||||
n_lines, dim, STATE_LEN, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cache_indices = (torch.randperm(n_lines - 1, device=device)[:n_seqs] + 1).to(
|
||||
torch.int32
|
||||
)
|
||||
query_start_loc = torch.tensor(
|
||||
[0] + list(torch.tensor(seqlens).cumsum(0).tolist()),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
has_initial = torch.full(
|
||||
(n_seqs,), has_initial_state, dtype=torch.bool, device=device
|
||||
)
|
||||
return mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
|
||||
|
||||
def _reference(
|
||||
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
):
|
||||
"""fp32 per-sequence depthwise causal conv + SiLU.
|
||||
|
||||
Returns (out[T, dim], new_conv_state).
|
||||
"""
|
||||
x = mixed_qkv.float().t() # (dim, T)
|
||||
w = weight.float()
|
||||
dim = x.shape[0]
|
||||
out = torch.empty_like(x)
|
||||
new_state = conv_state.float().clone()
|
||||
qsl = query_start_loc.tolist()
|
||||
for s in range(len(qsl) - 1):
|
||||
start, end = qsl[s], qsl[s + 1]
|
||||
ci = int(cache_indices[s])
|
||||
xs = x[:, start:end]
|
||||
if bool(has_initial[s]):
|
||||
xin = torch.cat([conv_state[ci].float(), xs], dim=1)
|
||||
else:
|
||||
xin = F.pad(xs, (STATE_LEN, 0))
|
||||
y = F.conv1d(xin.unsqueeze(0), w.unsqueeze(1), None, groups=dim)[0]
|
||||
out[:, start:end] = F.silu(y)
|
||||
new_state[ci] = xin[:, -STATE_LEN:]
|
||||
return out.t(), new_state
|
||||
|
||||
|
||||
def _three_launch(
|
||||
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
):
|
||||
"""The pre-fusion KDA prefill path: one causal_conv1d_fn per projection."""
|
||||
proj = mixed_qkv.shape[1] // 3
|
||||
outs = []
|
||||
for x, state, w in zip(
|
||||
mixed_qkv.split(proj, dim=-1),
|
||||
conv_state.split(proj, dim=-2),
|
||||
weight.split(proj, dim=0),
|
||||
):
|
||||
outs.append(
|
||||
causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
w,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial,
|
||||
cache_indices=cache_indices,
|
||||
query_start_loc=query_start_loc,
|
||||
).transpose(0, 1)
|
||||
)
|
||||
return outs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens", [[1024], [8, 1, 300], [64, 64, 64], [513, 64, 1, 1200]]
|
||||
)
|
||||
@pytest.mark.parametrize("has_initial_state", [False, True])
|
||||
@pytest.mark.parametrize("block_m", [8, 128])
|
||||
def test_fused_split_qkv_conv1d_matches_reference(seqlens, has_initial_state, block_m):
|
||||
fused_fn = _fused_fn()
|
||||
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial = _make_inputs(
|
||||
seqlens, has_initial_state
|
||||
)
|
||||
proj = mixed_qkv.shape[1] // 3
|
||||
|
||||
ref_out, ref_state = _reference(
|
||||
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial
|
||||
)
|
||||
|
||||
state_3 = conv_state.clone()
|
||||
q3, k3, v3 = _three_launch(
|
||||
mixed_qkv, weight, state_3, cache_indices, qsl, has_initial
|
||||
)
|
||||
|
||||
state_f = conv_state.clone()
|
||||
# Exactly the call made by KimiK3DeltaAttention._forward.
|
||||
qf, kf, vf = fused_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
state_f,
|
||||
qsl,
|
||||
proj,
|
||||
proj,
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial,
|
||||
activation="silu",
|
||||
block_m=block_m,
|
||||
metadata=None,
|
||||
preserve_input_dtype=True,
|
||||
)
|
||||
|
||||
fused = torch.cat([qf, kf, vf], dim=-1)
|
||||
assert fused.shape == mixed_qkv.shape and fused.dtype == mixed_qkv.dtype
|
||||
assert all(t.is_contiguous() for t in (qf, kf, vf))
|
||||
|
||||
tol = dict(atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(fused.float(), ref_out, **tol)
|
||||
torch.testing.assert_close(fused, torch.cat([q3, k3, v3], dim=-1), **tol)
|
||||
|
||||
# conv_state writeback: assigned lines updated, the others untouched.
|
||||
torch.testing.assert_close(state_f.float(), ref_state, **tol)
|
||||
torch.testing.assert_close(state_f, state_3, **tol)
|
||||
untouched = torch.ones(conv_state.shape[0], dtype=torch.bool, device="cuda")
|
||||
untouched[cache_indices.long()] = False
|
||||
assert untouched[NULL_BLOCK_ID]
|
||||
assert torch.equal(state_f[untouched], conv_state[untouched])
|
||||
@@ -85,6 +85,24 @@ def test_image_media_io_rgba_custom_background(tmp_path):
|
||||
assert green_numpy[0][0][2] == 0 # B
|
||||
|
||||
|
||||
@pytest.mark.parametrize("background_color", [(255, 255, 255), (12, 34, 56)])
|
||||
def test_image_media_io_rgb_png_transparency(tmp_path, background_color):
|
||||
image = Image.new("RGB", (2, 1), (255, 0, 0))
|
||||
image.putpixel((1, 0), (0, 0, 255))
|
||||
path = tmp_path / "rgb_transparency.png"
|
||||
image.save(path, transparency=(255, 0, 0))
|
||||
|
||||
with Image.open(path) as source:
|
||||
reference = Image.alpha_composite(
|
||||
Image.new("RGBA", source.size, (*background_color, 255)),
|
||||
source.convert("RGBA"),
|
||||
).convert("RGB")
|
||||
|
||||
image_io = ImageMediaIO(rgba_background_color=background_color)
|
||||
converted = image_io.load_file(path)
|
||||
np.testing.assert_array_equal(np.asarray(converted), np.asarray(reference))
|
||||
|
||||
|
||||
def test_image_media_io_no_mode_conversion(tmp_path):
|
||||
"""image_mode=None skips conversion and preserves the original mode."""
|
||||
# RGBA image: opaque black pixel on a fully transparent background
|
||||
|
||||
@@ -237,8 +237,16 @@ def _rgba_png_bytes() -> bytes:
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def test_hash_collision_media_io_config():
|
||||
data = _rgba_png_bytes()
|
||||
def _rgb_trns_png_bytes() -> bytes:
|
||||
image = Image.new("RGB", (8, 8), (255, 0, 0))
|
||||
buf = BytesIO()
|
||||
image.save(buf, format="PNG", transparency=(255, 0, 0))
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("png_bytes", [_rgba_png_bytes, _rgb_trns_png_bytes])
|
||||
def test_hash_collision_media_io_config(png_bytes):
|
||||
data = png_bytes()
|
||||
white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
|
||||
black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data)
|
||||
white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
|
||||
|
||||
@@ -5,6 +5,7 @@ from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pybase64
|
||||
import pytest
|
||||
from PIL import Image, ImageChops
|
||||
|
||||
@@ -13,6 +14,8 @@ from vllm.multimodal.image import (
|
||||
convert_image_mode,
|
||||
normalize_image,
|
||||
)
|
||||
from vllm.multimodal.media import MediaConnector
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -69,6 +72,68 @@ def test_palette_with_trns_to_rgb():
|
||||
assert (r, g, b) == (255, 0, 0)
|
||||
|
||||
|
||||
def _reopen(image: Image.Image, image_format: str, **save_kwargs) -> Image.Image:
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, image_format, **save_kwargs)
|
||||
return Image.open(buffer)
|
||||
|
||||
|
||||
def _rgb_png() -> Image.Image:
|
||||
image = Image.new("RGB", (2, 1), (255, 0, 0))
|
||||
image.putpixel((1, 0), (0, 0, 255))
|
||||
return _reopen(image, "PNG", transparency=(255, 0, 0))
|
||||
|
||||
|
||||
def _gif() -> Image.Image:
|
||||
# The transparent index 0 and the opaque index 1 are both black.
|
||||
image = Image.new("P", (2, 1))
|
||||
image.putpalette([0] * 768)
|
||||
image.putpixel((1, 0), 1)
|
||||
return _reopen(image, "GIF", transparency=0)
|
||||
|
||||
|
||||
def _rgba_png() -> Image.Image:
|
||||
image = Image.new("RGBA", (2, 1), (255, 0, 0, 0))
|
||||
image.putpixel((1, 0), (0, 0, 255, 255))
|
||||
return _reopen(image, "PNG")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"make_image",
|
||||
[
|
||||
_rgb_png,
|
||||
lambda: _rgb_png().convert("RGB"),
|
||||
_gif,
|
||||
lambda: _gif().convert("RGB"),
|
||||
_rgba_png,
|
||||
],
|
||||
ids=["rgb_png", "rgb_png_to_rgb", "gif", "gif_to_rgb", "rgba_png"],
|
||||
)
|
||||
def test_transparency_offline_matches_online(make_image):
|
||||
"""A PIL image passed offline and the same image sent online as a PNG
|
||||
end up with the same pixels, matching Pillow's own RGBA conversion.
|
||||
|
||||
After ``convert("RGB")``, Pillow keeps the transparent color in ``info``
|
||||
and saves it as a tRNS chunk, so every pixel of that color is
|
||||
transparent in both pipelines, including opaque GIF pixels that share it.
|
||||
"""
|
||||
image = make_image()
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, "PNG")
|
||||
url = f"data:image/png;base64,{pybase64.b64encode(buffer.getvalue()).decode()}"
|
||||
|
||||
parser = MultiModalDataParser()
|
||||
offline = parser.parse_mm_data({"image": image})["image"].get(0)
|
||||
online = parser.parse_mm_data({"image": MediaConnector().fetch_image(url)})
|
||||
online = online["image"].get(0)
|
||||
np.testing.assert_array_equal(np.asarray(offline), np.asarray(online))
|
||||
|
||||
expected = Image.alpha_composite(
|
||||
Image.new("RGBA", image.size, (255, 255, 255, 255)), image.convert("RGBA")
|
||||
).convert("RGB")
|
||||
np.testing.assert_array_equal(np.asarray(online), np.asarray(expected))
|
||||
|
||||
|
||||
def test_l_mode_no_trns_to_rgb():
|
||||
"""L-mode without transparency should convert directly."""
|
||||
img = Image.new("L", (4, 4), 128)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
|
||||
import pytest
|
||||
@@ -15,11 +16,14 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.minimax_m2 import (
|
||||
THINK_END,
|
||||
TOOL_CALL_END,
|
||||
TOOL_CALL_START,
|
||||
MinimaxM2Parser,
|
||||
_minimax_m2_arg_value_extender,
|
||||
minimax_m2_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -238,6 +242,123 @@ class TestStreaming:
|
||||
assert collect_tool_arguments(results) == ""
|
||||
|
||||
|
||||
WRITE_CALL_HEAD = (
|
||||
'<minimax:tool_call>\n<invoke name="write_file">\n'
|
||||
'<parameter name="path">/repo/app.py</parameter>\n'
|
||||
'<parameter name="mode">644</parameter>\n'
|
||||
'<parameter name="content">'
|
||||
)
|
||||
WRITE_CALL_TAIL = "</parameter>\n</invoke>\n</minimax:tool_call>"
|
||||
# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
|
||||
# the value, including prefixes of the closing and next-parameter tags.
|
||||
CONTENT_LINE = 'if a < b > c: print("x\\y") # </p> </param <parameter\té\n\n'
|
||||
|
||||
|
||||
class TestArgValueExtender:
|
||||
"""The open parameter value streams without re-running the converter."""
|
||||
|
||||
@pytest.fixture
|
||||
def write_tool(self):
|
||||
return ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="write_file",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"},
|
||||
"mode": {"type": "integer"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
|
||||
def test_long_call_streams_non_streaming_arguments(
|
||||
self, mock_tokenizer, mock_request, write_tool, use_schema
|
||||
):
|
||||
mock_request.tools = [write_tool] if use_schema else []
|
||||
text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
MinimaxM2Parser(mock_tokenizer),
|
||||
mock_request,
|
||||
[text[i : i + 7] for i in range(0, len(text), 7)],
|
||||
)
|
||||
result = MinimaxM2Parser(mock_tokenizer).extract_tool_calls(text, mock_request)
|
||||
|
||||
arguments = collect_tool_arguments(results)
|
||||
assert arguments == result.tool_calls[0].function.arguments
|
||||
assert json.loads(arguments)["content"] == CONTENT_LINE * 200
|
||||
assert ('"mode": 644,' if use_schema else '"mode": "644",') in arguments
|
||||
|
||||
def test_conversions_do_not_grow_with_argument_length(
|
||||
self, mock_tokenizer, mock_request, write_tool
|
||||
):
|
||||
mock_request.tools = [write_tool]
|
||||
base = minimax_m2_config()
|
||||
converter = base.arg_converter
|
||||
assert converter is not None
|
||||
|
||||
def streaming_conversions(lines: int) -> int:
|
||||
calls = 0
|
||||
|
||||
def counting_converter(raw_args: str, partial: bool) -> str:
|
||||
nonlocal calls
|
||||
calls += partial
|
||||
return converter(raw_args, partial)
|
||||
|
||||
parser = ParserEngine(
|
||||
mock_tokenizer,
|
||||
parser_engine_config=dataclasses.replace(
|
||||
base, arg_converter=counting_converter
|
||||
),
|
||||
)
|
||||
text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
|
||||
simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[text[i : i + 4] for i in range(0, len(text), 4)],
|
||||
)
|
||||
return calls
|
||||
|
||||
assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
|
||||
|
||||
def test_extender_stops_at_split_end_tag(self):
|
||||
extend = _minimax_m2_arg_value_extender('<parameter name="k">')
|
||||
assert extend is not None
|
||||
deltas = ['a"\n', "</param", "eter>"]
|
||||
assert [extend(d) for d in deltas] == ['a\\"\\n', "</param", None]
|
||||
|
||||
def test_extender_stops_at_split_next_parameter(self):
|
||||
extend = _minimax_m2_arg_value_extender("<parameter name='k'>x")
|
||||
assert extend is not None
|
||||
assert [extend(d) for d in ["< parameter", " name ="]] == ["< parameter", None]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_args",
|
||||
[
|
||||
'<parameter name="k">x</parameter>',
|
||||
'<parameter name="k',
|
||||
(
|
||||
'<parameter name="k">1</parameter><parameter name="j">2</parameter>'
|
||||
'<parameter name="k">3'
|
||||
),
|
||||
'<parameter name="a<parameter name=k>x',
|
||||
'<parameter name="k>x',
|
||||
],
|
||||
ids=[
|
||||
"closed_value",
|
||||
"open_name",
|
||||
"repeated_key_not_last",
|
||||
"earlier_unmatched_opener",
|
||||
"bare_name_opens_quote",
|
||||
],
|
||||
)
|
||||
def test_no_extender_unless_open_value_is_stable(self, raw_args):
|
||||
assert _minimax_m2_arg_value_extender(raw_args) is None
|
||||
|
||||
|
||||
class TestReasoning:
|
||||
def test_extract_reasoning_without_start_token(self, parser, mock_request):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
|
||||
@@ -380,6 +380,40 @@ class TestDelegatingPromptDetection:
|
||||
assert delta.reasoning is None
|
||||
assert delta.content == "answer"
|
||||
|
||||
def test_prior_turn_think_end_keeps_non_streaming_reasoning(
|
||||
self, mock_tokenizer, mock_request
|
||||
):
|
||||
parser = _Qwen3DelegatingParser(mock_tokenizer)
|
||||
parser.set_prompt_token_ids([_THINK_END_ID, _IM_END_ID, _IM_START_ID, _TEXT_ID])
|
||||
output_ids = [ord("a"), _THINK_END_ID, ord("b")]
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
mock_tokenizer.decode(output_ids),
|
||||
mock_request,
|
||||
model_output_token_ids=output_ids,
|
||||
)
|
||||
|
||||
assert reasoning == "a"
|
||||
assert content == "b"
|
||||
assert parser.count_reasoning_tokens(output_ids) > 0
|
||||
|
||||
def test_think_end_in_current_turn_parses_output_as_content(
|
||||
self, mock_tokenizer, mock_request
|
||||
):
|
||||
parser = _Qwen3DelegatingParser(mock_tokenizer)
|
||||
parser.set_prompt_token_ids([_IM_START_ID, _THINK_START_ID, _THINK_END_ID])
|
||||
output_ids = [ord("a"), ord("b")]
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
mock_tokenizer.decode(output_ids),
|
||||
mock_request,
|
||||
model_output_token_ids=output_ids,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "ab"
|
||||
assert parser.count_reasoning_tokens(output_ids) == 0
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_basic_streaming(self, parser):
|
||||
|
||||
@@ -0,0 +1,747 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import FunctionTool, ToolChoiceFunction
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.response_template import ResponseTemplateParser
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
# The `response_template` of the Beam checkpoints.
|
||||
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
|
||||
"defaults": {"role": "assistant"},
|
||||
"start_anchor_pattern": r"<\|start\|>assistant",
|
||||
"fields": {
|
||||
"thinking": {
|
||||
"open": "<|channel|>analysis<|message|>",
|
||||
"close": "<|end|><|start|>assistant",
|
||||
"content": "text",
|
||||
},
|
||||
"content": {
|
||||
"open": "<|channel|>final<|message|>",
|
||||
"close": ["<|return|>", "<|end|>", "<|call|>"],
|
||||
"content": "text",
|
||||
},
|
||||
"tool_calls": {
|
||||
"open_pattern": (
|
||||
r" to=(?P<name>[^\s<]+)"
|
||||
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
|
||||
),
|
||||
"close": ["<|end|><|start|>assistant", "<|call|>"],
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": (
|
||||
r'<rfl:parameter name="(?P<key>[^"]+)">'
|
||||
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
|
||||
),
|
||||
"value_parser": {"name": "text", "args": {"strip": False}},
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
VOCAB = {
|
||||
"<|start|>": 200,
|
||||
"<|channel|>": 201,
|
||||
"<|message|>": 202,
|
||||
"<|end|>": 203,
|
||||
"<|call|>": 204,
|
||||
"<|return|>": 205,
|
||||
"<|constrain|>": 206,
|
||||
}
|
||||
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
|
||||
PROMPT_TOKEN_IDS = [200, 100]
|
||||
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
|
||||
FINAL = "<|channel|>final<|message|>"
|
||||
NEXT_MESSAGE = "<|end|><|start|>assistant"
|
||||
|
||||
|
||||
def _tool(name, properties, *, strict=None):
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"parameters": {"type": "object", "properties": properties},
|
||||
"strict": strict,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}),
|
||||
]
|
||||
STRICT_TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
|
||||
]
|
||||
|
||||
|
||||
class BeamTokenizer:
|
||||
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
|
||||
self.response_template = response_template
|
||||
self.init_kwargs = {}
|
||||
self.name_or_path = ""
|
||||
self.all_special_tokens = list(VOCAB)
|
||||
self.all_special_ids = list(VOCAB.values())
|
||||
|
||||
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
|
||||
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
|
||||
return "".join(
|
||||
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
|
||||
for token_id in token_ids
|
||||
)
|
||||
|
||||
def get_vocab(self):
|
||||
return VOCAB
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
del add_special_tokens
|
||||
return list(text.encode())
|
||||
|
||||
|
||||
def request(**overrides):
|
||||
values = dict(
|
||||
tool_choice="auto",
|
||||
include_reasoning=True,
|
||||
tools=TOOLS,
|
||||
parallel_tool_calls=True,
|
||||
)
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def parameter(name, value):
|
||||
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
|
||||
|
||||
|
||||
def call(name, body, *, terminator="<|call|>"):
|
||||
return (
|
||||
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
|
||||
)
|
||||
|
||||
|
||||
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
|
||||
parser.set_prompt_token_ids(prompt_token_ids)
|
||||
return parser
|
||||
|
||||
|
||||
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
|
||||
reasoning, content, calls = make_parser(prompt_token_ids).parse(
|
||||
text, request(), enable_auto_tools=True
|
||||
)
|
||||
return reasoning, content, normalize_calls(calls)
|
||||
|
||||
|
||||
def normalize_calls(calls):
|
||||
return [(call.name, json.loads(call.arguments)) for call in calls or []]
|
||||
|
||||
|
||||
def test_beam_parsers_are_registered():
|
||||
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
|
||||
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_beam_parser():
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, BeamParser)
|
||||
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
|
||||
assert parser_cls.tool_parser_cls is BeamToolParser
|
||||
assert parser_cls._enable_auto_tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize("other", ["hermes", "hf"])
|
||||
def test_beam_parser_rejects_other_parsers(other):
|
||||
with pytest.raises(TypeError, match="cannot be combined"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name=other,
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
|
||||
def test_beam_parser_requires_checkpoint_metadata_at_startup():
|
||||
with pytest.raises(
|
||||
TypeError, match="--reasoning-parser beam requires `response_template`"
|
||||
):
|
||||
ParserManager.get_parser(
|
||||
reasoning_parser_name="beam",
|
||||
tokenizer=BeamTokenizer(response_template=None),
|
||||
)
|
||||
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
del template["fields"]["tool_calls"]
|
||||
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(template),
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_and_final_answer():
|
||||
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
|
||||
|
||||
|
||||
def test_prefilled_reasoning_continues_from_prompt():
|
||||
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
|
||||
|
||||
assert parse(text, prompt) == (" more.", "Done.", [])
|
||||
|
||||
|
||||
def test_parallel_calls_are_coerced_to_their_schemas():
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
|
||||
assert parse(text) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
|
||||
)
|
||||
|
||||
|
||||
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
|
||||
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
|
||||
|
||||
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
|
||||
def test_output_cut_before_the_next_message_start_ends_its_region(end):
|
||||
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
|
||||
its reasoning or call and lacks only the next message's start."""
|
||||
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
|
||||
|
||||
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
|
||||
assert parse(THOUGHT + weather) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"})],
|
||||
)
|
||||
|
||||
|
||||
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
|
||||
FINAL_IDS = [201, 102, 202]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_token_ids", "token_ids", "count"),
|
||||
[
|
||||
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
|
||||
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
|
||||
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
|
||||
],
|
||||
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
|
||||
)
|
||||
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
|
||||
parser = make_parser(prompt_token_ids)
|
||||
|
||||
assert parser.count_reasoning_tokens(token_ids) == count
|
||||
|
||||
|
||||
def test_reasoning_tokens_are_counted_as_the_generation_grows():
|
||||
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
|
||||
parser = make_parser()
|
||||
|
||||
assert [
|
||||
parser.count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
] == [
|
||||
make_parser().count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
]
|
||||
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
|
||||
|
||||
|
||||
def test_call_stop_token_ends_the_answer():
|
||||
assert parse(FINAL + "done<|call|>") == (None, "done", [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
|
||||
def test_streaming_matches_non_streaming(chunk_size):
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
parser = make_parser()
|
||||
reasoning, calls = [], []
|
||||
for start in range(0, len(text), chunk_size):
|
||||
end = min(start + chunk_size, len(text))
|
||||
delta = parser.parse_delta(
|
||||
text[start:end], [], request(), finished=end == len(text)
|
||||
)
|
||||
if delta is not None:
|
||||
reasoning.append(delta.reasoning or "")
|
||||
calls.extend(delta.tool_calls)
|
||||
|
||||
assert "".join(reasoning) == "plan"
|
||||
assert [
|
||||
(call.function.name, json.loads(call.function.arguments)) for call in calls
|
||||
] == parse(text)[2]
|
||||
|
||||
|
||||
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
|
||||
WEATHER = [("get_weather", {"city": "Paris"})]
|
||||
|
||||
|
||||
def chat_request(tools=STRICT_TOOLS, **body):
|
||||
return ChatCompletionRequest(
|
||||
model="beam",
|
||||
messages=[{"role": "user", "content": "Weather?"}],
|
||||
tools=tools and [tool.model_dump() for tool in tools],
|
||||
**body,
|
||||
)
|
||||
|
||||
|
||||
def named(name):
|
||||
return {"type": "function", "function": {"name": name}}
|
||||
|
||||
|
||||
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
|
||||
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
|
||||
return req
|
||||
|
||||
|
||||
def accepts(req, text):
|
||||
grammar = Grammar.from_structural_tag(
|
||||
req.extract_structured_outputs().structural_tag
|
||||
)
|
||||
return _is_grammar_accept_string(grammar, text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "calls"),
|
||||
[
|
||||
("required", THOUGHT + WEATHER_CALL, WEATHER),
|
||||
("required", WEATHER_CALL, None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ call("set_count", parameter("count", "7")),
|
||||
[*WEATHER, ("set_count", {"count": 7})],
|
||||
),
|
||||
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
|
||||
None,
|
||||
),
|
||||
("required", THOUGHT + FINAL + "Paris<|return|>", None),
|
||||
(named("set_count"), THOUGHT + WEATHER_CALL, None),
|
||||
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
|
||||
(
|
||||
named("get_weather"),
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ WEATHER_CALL,
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
|
||||
"""From the message start, the grammar opens with the reasoning, and
|
||||
accepts only calls the parser reads back."""
|
||||
req = adjust(chat_request(tool_choice=tool_choice))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is (calls is not None)
|
||||
if calls is not None:
|
||||
assert parse(text)[2] == calls
|
||||
|
||||
|
||||
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"),
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_parallel_tool_calls_false_allows_a_single_call():
|
||||
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
|
||||
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strict", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
|
||||
)
|
||||
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
|
||||
"""A model that skips a parameter's opener must not write whitespace until
|
||||
`max_tokens`."""
|
||||
write = _tool(
|
||||
"write",
|
||||
{"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
strict=strict,
|
||||
)
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = parameter("path", "a.py") + separator + parameter("content", "x")
|
||||
|
||||
assert accepts(req, THOUGHT + call("write", body)) is accepted
|
||||
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
|
||||
|
||||
|
||||
def test_forced_calls_in_prefilled_reasoning_start_after_it():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
)
|
||||
|
||||
assert not req._grammar_from_parser
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_calls_cannot_follow_a_prefilled_answer():
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
|
||||
def test_prefilled_answer_can_only_be_finished():
|
||||
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
assert accepts(req, "Sunny.<|return|>")
|
||||
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_call_with_unencodable_tool_name_is_rejected():
|
||||
tools = [_tool("get weather", {"city": {"type": "string"}})]
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="tool name"):
|
||||
adjust(chat_request(tools, tool_choice="required"))
|
||||
|
||||
|
||||
def test_string_formats_and_patterns_are_held_in_raw_text():
|
||||
"""Like XGrammar, a strict string keeps its format or pattern, unless a
|
||||
match could end the value early."""
|
||||
contact = _tool(
|
||||
"set_contact",
|
||||
{
|
||||
"email": {"type": "string", "format": "email"},
|
||||
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
|
||||
"n": {"type": "integer"},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
contact.function.parameters["required"] = ["email", "code", "n"]
|
||||
req = adjust(chat_request([contact], tool_choice="required"))
|
||||
|
||||
def contact_call(email, code, *rest):
|
||||
body = parameter("email", email) + parameter("code", code) + "".join(rest)
|
||||
return THOUGHT + call("set_contact", body)
|
||||
|
||||
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
|
||||
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert accepts(req, text)
|
||||
assert normalize_calls(calls) == [
|
||||
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
|
||||
]
|
||||
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
|
||||
# Line breaks could end the value.
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request([dotted], tool_choice="required"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + WEATHER_CALL, True),
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
(THOUGHT + call("set_count", parameter("count", "x")), False),
|
||||
],
|
||||
)
|
||||
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
|
||||
assert accepts(adjust(chat_request()), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(FINAL + "Sunny.<|return|>", False),
|
||||
(WEATHER_CALL, False),
|
||||
(THOUGHT + WEATHER_CALL, False),
|
||||
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
],
|
||||
)
|
||||
def test_tool_choice_none_allows_only_an_answer(text, accepted):
|
||||
req = adjust(chat_request(TOOLS, tool_choice="none"))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
|
||||
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
|
||||
req = adjust(chat_request(tools, tool_choice=tool_choice))
|
||||
count_call = THOUGHT + call("set_count", parameter("count", "3"))
|
||||
|
||||
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT + "Sunny.")
|
||||
assert accepts(req, count_call) is (tool_choice == "auto")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
|
||||
@pytest.mark.parametrize("strict", [None, True])
|
||||
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||
def test_automatic_choice_lets_calls_name_undeclared_tools(
|
||||
strict, tool_choice, response_format
|
||||
):
|
||||
"""Unless a forced choice limits calls to the declared tools. The parser
|
||||
passes such calls on, so the harness can tell the model the tool does not
|
||||
exist."""
|
||||
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
|
||||
req = adjust(
|
||||
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
|
||||
)
|
||||
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
|
||||
parser = make_parser(tools=[bash])
|
||||
|
||||
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
|
||||
assert accepts(req, undeclared) is (tool_choice == "auto")
|
||||
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "body", "accepted"),
|
||||
[
|
||||
("set_count", parameter("count", "1"), True),
|
||||
("set_count", parameter("count", "x"), False),
|
||||
("get_weather", parameter("town", "Paris"), False),
|
||||
# The grammar cannot enforce `minProperties`.
|
||||
("find", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("search_files", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
|
||||
],
|
||||
)
|
||||
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
|
||||
name, body, accepted
|
||||
):
|
||||
find = _tool("find", {"q": {"type": "string"}})
|
||||
find.function.parameters["minProperties"] = 1
|
||||
req = adjust(chat_request([*TOOLS, find]))
|
||||
|
||||
assert accepts(req, THOUGHT + call(name, body)) is accepted
|
||||
|
||||
|
||||
def test_value_ends_before_an_opener_on_the_next_line():
|
||||
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
|
||||
text = THOUGHT + call("write", body)
|
||||
|
||||
assert accepts(req, text)
|
||||
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [
|
||||
("write", {"content": "print(1)\n", "path": "a.py"})
|
||||
]
|
||||
|
||||
|
||||
def test_always_constrained_output_can_be_disabled(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
|
||||
|
||||
assert adjust(chat_request(TOOLS)).structured_outputs is None
|
||||
|
||||
|
||||
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
|
||||
assert adjust(chat_request([dotted])).structured_outputs is None
|
||||
|
||||
|
||||
def test_generic_parser_does_not_derive_the_template_grammar():
|
||||
req = chat_request(TOOLS, tool_choice="required")
|
||||
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
parser.adjust_request(req)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", [None, TOOLS])
|
||||
def test_structured_output_is_framed_as_the_answer(tools):
|
||||
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
|
||||
req = adjust(
|
||||
chat_request(
|
||||
tools,
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "answer", "schema": schema},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert req.response_format is None
|
||||
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
|
||||
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
|
||||
|
||||
|
||||
def test_responses_named_tool_choice_follows_the_template_grammar():
|
||||
req = adjust(
|
||||
ResponsesRequest(
|
||||
input="Weather?",
|
||||
tools=[
|
||||
FunctionTool(
|
||||
type="function",
|
||||
name="get_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
],
|
||||
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
|
||||
text={"format": {"type": "json_object"}},
|
||||
)
|
||||
)
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert req.text.format.type == "json_object"
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, FINAL + "{}<|return|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["in_progress", "completed"])
|
||||
@pytest.mark.parametrize("inside_reasoning", [False, True])
|
||||
def test_responses_preserves_text_config_with_template_grammar(
|
||||
status, inside_reasoning
|
||||
):
|
||||
text = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "answer",
|
||||
"strict": True,
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "integer"}},
|
||||
"required": ["a"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"verbosity": "low",
|
||||
}
|
||||
req = ResponsesRequest(input="Answer", text=text)
|
||||
expected_text = req.text.model_dump(by_alias=True)
|
||||
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
|
||||
adjust(req, prompt_token_ids=prompt)
|
||||
params = req.to_sampling_params(default_max_tokens=128)
|
||||
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
|
||||
prefix = "" if inside_reasoning else THOUGHT
|
||||
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
|
||||
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
|
||||
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
|
||||
assert response.model_dump(by_alias=True)["text"] == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reverse_alternatives", [False, True])
|
||||
def test_reasoning_budget_initializes_from_checkpoint_pattern(
|
||||
monkeypatch, reverse_alternatives
|
||||
):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
alternatives = [
|
||||
r"^<\|channel\|>analysis<\|message\|>",
|
||||
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
|
||||
]
|
||||
if reverse_alternatives:
|
||||
alternatives.reverse()
|
||||
template["fields"]["thinking"] = {
|
||||
"open_pattern": "(?:" + "|".join(alternatives) + ")",
|
||||
"close": "<|end|>",
|
||||
"content": "text",
|
||||
}
|
||||
tokenizer = BeamTokenizer(template)
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
|
||||
)
|
||||
config = ReasoningConfig(reasoning_parser="beam")
|
||||
config.initialize_token_ids(SimpleNamespace())
|
||||
|
||||
assert config.enabled
|
||||
# The marker must work both after the prompt's assistant header and in
|
||||
# subsequent complete assistant messages, regardless of regex branch order.
|
||||
assert config.reasoning_start_token_ids == tokenizer.encode(
|
||||
"<|channel|>analysis<|message|>"
|
||||
)
|
||||
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
|
||||
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
thinking = template["fields"]["thinking"]
|
||||
del thinking["open"]
|
||||
thinking["open_pattern"] = pattern
|
||||
|
||||
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
|
||||
@@ -9,6 +9,9 @@ import pytest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.qwen3 import qwen3_config
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
@@ -417,3 +420,179 @@ def test_parse_auto_tools_no_calls_returns_none(tokenizer):
|
||||
assert reasoning is None
|
||||
assert content == PLAIN_TEXT
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Prompt that already closes the reasoning block
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CONTINUE_MESSAGES = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "Sure,"},
|
||||
]
|
||||
REASONED_OUTPUT = "<think>\nlet me think\n</think>\n\n" + PLAIN_TEXT
|
||||
TOOL_TURN_MESSAGES = [
|
||||
{"role": "user", "content": "Weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "<think>\nneed a tool\n</think>\n\n",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
|
||||
|
||||
def reasoning_parser(tokenizer, name="qwen3"):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
reasoning_parser_name=name, tokenizer=tokenizer
|
||||
)
|
||||
return parser_cls(tokenizer, None)
|
||||
|
||||
|
||||
def encode(tokenizer, text):
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
|
||||
def prompt_case(tokenizer, case):
|
||||
"""(rendered prompt ids, request, model output) for each prompt shape."""
|
||||
if case == "continue_final_message":
|
||||
# The template renders an empty, closed think block before "Sure,".
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
CONTINUE_MESSAGES,
|
||||
tokenize=False,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
request = make_request(
|
||||
messages=CONTINUE_MESSAGES,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), request, " here is the answer"
|
||||
if case == "template_closed_thinking":
|
||||
# The template switches thinking off, but the parser is not told
|
||||
# through chat_template_kwargs (#53284).
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), PLAIN_TEXT
|
||||
if case == "earlier_turn_closed_reasoning":
|
||||
# Only a previous assistant turn closes reasoning; this one is open.
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
TOOL_TURN_MESSAGES,
|
||||
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
|
||||
assert case == "reasoning_open"
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"])
|
||||
def test_parse_prompt_closed_reasoning_is_content(tokenizer, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
parser = reasoning_parser(tokenizer)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == output
|
||||
assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parser_name", "case"),
|
||||
[
|
||||
("qwen3", "reasoning_open"),
|
||||
("qwen3", "earlier_turn_closed_reasoning"),
|
||||
# BaseThinkingReasoningParser scans the prompt without turn
|
||||
# boundaries, so it must not decide non-streaming classification.
|
||||
("deepseek_r1", "earlier_turn_closed_reasoning"),
|
||||
],
|
||||
)
|
||||
def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
parser = reasoning_parser(tokenizer, parser_name)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning.strip() == "let me think"
|
||||
assert content.strip() == PLAIN_TEXT
|
||||
assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0
|
||||
|
||||
|
||||
def test_parse_prompt_ignored_without_turn_boundaries(tokenizer):
|
||||
"""Without turn boundaries, an earlier turn's </think> looks like this
|
||||
turn's, so the engine must not start the output in content."""
|
||||
prompt_ids, request, output = prompt_case(
|
||||
tokenizer, "earlier_turn_closed_reasoning"
|
||||
)
|
||||
parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True))
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning.strip() == "let me think"
|
||||
assert content.strip() == PLAIN_TEXT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"continue_final_message",
|
||||
"template_closed_thinking",
|
||||
"reasoning_open",
|
||||
"earlier_turn_closed_reasoning",
|
||||
],
|
||||
)
|
||||
def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
output_ids = encode(tokenizer, output)
|
||||
|
||||
parser = reasoning_parser(tokenizer)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=output_ids
|
||||
)
|
||||
non_streaming = (reasoning or "", content or "")
|
||||
non_streaming_count = parser.count_reasoning_tokens(output_ids)
|
||||
|
||||
parser = reasoning_parser(tokenizer)
|
||||
streamed_reasoning = streamed_content = ""
|
||||
for i, token_id in enumerate(output_ids):
|
||||
delta = parser.parse_delta(
|
||||
tokenizer.decode([token_id]),
|
||||
[token_id],
|
||||
request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=i == len(output_ids) - 1,
|
||||
)
|
||||
if delta is not None:
|
||||
streamed_reasoning += delta.reasoning or ""
|
||||
streamed_content += delta.content or ""
|
||||
|
||||
assert (streamed_reasoning, streamed_content) == non_streaming
|
||||
assert parser.count_reasoning_tokens(output_ids) == non_streaming_count
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.parser.chat_parsing.response_templates import load_response_template
|
||||
from vllm.parser.response_template_grammar import (
|
||||
answer_rest_structural_tag,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
|
||||
TEMPLATE: dict[str, Any] = {
|
||||
"start_anchor": "<assistant>",
|
||||
"fields": {
|
||||
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
|
||||
"content": {"content": "text"},
|
||||
"tool_calls": {
|
||||
"open_pattern": r'<call name="(?P<name>[\w.-]+)">',
|
||||
"close": "</call>",
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)</arg>'
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
ANSWER_TEMPLATE = deepcopy(TEMPLATE)
|
||||
ANSWER_TEMPLATE["fields"]["content"] = {
|
||||
"open": "<answer>",
|
||||
"close": "</answer>",
|
||||
"content": "text",
|
||||
}
|
||||
|
||||
|
||||
def tool(parameters, name="f", strict=True) -> dict[str, Any]:
|
||||
return {"name": name, "parameters": parameters, "strict": strict}
|
||||
|
||||
|
||||
WEATHER = tool(
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "days": {"type": "integer"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
name="get.weather",
|
||||
)
|
||||
PING = {"name": "ping"}
|
||||
CALL = '<call name="get.weather"><arg name="city">Paris</arg></call>'
|
||||
|
||||
|
||||
def build(
|
||||
template=TEMPLATE,
|
||||
functions=(WEATHER, PING),
|
||||
*,
|
||||
parallel=True,
|
||||
thinking=True,
|
||||
answer=False,
|
||||
answer_format=None,
|
||||
sorted_arguments=False,
|
||||
):
|
||||
fields = load_response_template(template).fields
|
||||
return tool_call_structural_tag(
|
||||
fields["tool_calls"],
|
||||
list(functions),
|
||||
parallel_tool_calls=parallel,
|
||||
thinking=fields["thinking"] if thinking else None,
|
||||
answer=fields["content"] if answer else None,
|
||||
answer_format=answer_format,
|
||||
sorted_arguments=sorted_arguments,
|
||||
)
|
||||
|
||||
|
||||
def accepts(tag, text):
|
||||
return _is_grammar_accept_string(Grammar.from_structural_tag(json.dumps(tag)), text)
|
||||
|
||||
|
||||
def call(*arguments):
|
||||
body = "".join(f'<arg name="{key}">{value}</arg>' for key, value in arguments)
|
||||
return f'<call name="f">{body}</call>'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
("<think>plan</think>" + CALL, True),
|
||||
(CALL, True),
|
||||
(CALL.removesuffix("</call>"), True),
|
||||
(
|
||||
'<call name="get.weather"><arg name="city">Paris</arg>\n'
|
||||
'<arg name="days">3</arg></call>',
|
||||
True,
|
||||
),
|
||||
(
|
||||
'<call name="get.weather"><arg name="city">Paris</arg>'
|
||||
'<arg name="days">soon</arg></call>',
|
||||
False,
|
||||
),
|
||||
('<call name="get.weather"><arg name="days">3</arg></call>', False),
|
||||
(CALL + '<call name="ping"><arg name="any">x</arg></call>', True),
|
||||
('<call name="other"></call>', False),
|
||||
("<think>plan</think>The answer.", False),
|
||||
],
|
||||
)
|
||||
def test_forced_calls_follow_the_template_format(text, accepted):
|
||||
assert accepts(build(), text) is accepted
|
||||
|
||||
|
||||
def test_single_call_when_parallel_calls_are_disabled():
|
||||
tag = build(parallel=False)
|
||||
|
||||
assert accepts(tag, CALL)
|
||||
assert not accepts(tag, CALL + CALL)
|
||||
|
||||
|
||||
def test_thinking_is_rejected_when_not_requested():
|
||||
assert not accepts(build(thinking=False), "<think>plan</think>" + CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
("<think>plan</think><answer>Sunny.", True),
|
||||
("<answer>Sunny.</answer>", True),
|
||||
("<think>plan</think>" + CALL, True),
|
||||
("<answer>Sunny.</answer>" + CALL, False),
|
||||
('<call name="get.weather"><arg name="days">soon</arg></call>', False),
|
||||
],
|
||||
)
|
||||
def test_answer_field_lets_the_model_reply_instead(text, accepted):
|
||||
assert accepts(build(ANSWER_TEMPLATE, answer=True), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("required", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("text", "optional_only"),
|
||||
[
|
||||
("<think>plan</think>" + CALL, False),
|
||||
("<think>\nplan</think><answer>Sunny.", False),
|
||||
("<think> \n</think><answer>Sunny.", False),
|
||||
(CALL, True),
|
||||
("<answer>Sunny.", True),
|
||||
("<think></think><answer>Sunny.", True),
|
||||
("<think><</think><answer>Sunny.", True),
|
||||
],
|
||||
)
|
||||
def test_required_reasoning_opens_the_message_with_nonempty_reasoning(
|
||||
required, text, optional_only
|
||||
):
|
||||
fields = load_response_template(ANSWER_TEMPLATE).fields
|
||||
tag = tool_call_structural_tag(
|
||||
fields["tool_calls"],
|
||||
[WEATHER],
|
||||
parallel_tool_calls=True,
|
||||
thinking=fields["thinking"],
|
||||
answer=fields["content"],
|
||||
message_start=["<assistant>"],
|
||||
requires_reasoning=required,
|
||||
)
|
||||
|
||||
assert accepts(tag, text) is (not required or not optional_only)
|
||||
|
||||
|
||||
def test_answer_rest_finishes_the_open_answer():
|
||||
answer = load_response_template(ANSWER_TEMPLATE).fields["content"]
|
||||
tag = answer_rest_structural_tag(answer)
|
||||
|
||||
assert accepts(tag, "Sunny.</answer>")
|
||||
assert accepts(tag, "Sunny.")
|
||||
assert not accepts(tag, "Sunny.</answer>" + CALL)
|
||||
|
||||
|
||||
def test_answer_without_literal_opener_is_not_derived():
|
||||
assert build(answer=True) is None
|
||||
|
||||
|
||||
def test_answer_holds_the_answer_format():
|
||||
schema = {"type": "json_schema", "json_schema": {"type": "object"}}
|
||||
with_tools = build(ANSWER_TEMPLATE, answer=True, answer_format=schema)
|
||||
answer_only = build(ANSWER_TEMPLATE, (), answer=True, answer_format=schema)
|
||||
|
||||
for tag in (with_tools, answer_only):
|
||||
assert accepts(tag, '<think>plan</think><answer>{"a": 1}</answer>')
|
||||
assert not accepts(tag, "<answer>Sunny.</answer>")
|
||||
assert accepts(with_tools, CALL)
|
||||
assert not accepts(answer_only, CALL)
|
||||
|
||||
|
||||
def test_json_arguments_follow_the_schema():
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"].update(content="json", content_args={})
|
||||
tag = build(template, [WEATHER])
|
||||
|
||||
assert accepts(tag, '<call name="get.weather">{"city": "Paris"}</call>')
|
||||
assert not accepts(tag, '<call name="get.weather">{"days": 3}</call>')
|
||||
|
||||
|
||||
END_X = {"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)X'}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"content": "json", "content_args": {"unquoted_keys": True}},
|
||||
{
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "fixed", "arguments": "{content}"},
|
||||
}
|
||||
},
|
||||
{"content": "kv-lines", "content_args": {}},
|
||||
# Framing in which argument text can spell a closer, where the parser
|
||||
# would end the call.
|
||||
{"close": ["}X", "</call>"], "content_args": END_X},
|
||||
{
|
||||
"close": ["}X", "</call>"],
|
||||
"content_args": {**END_X, "value_parser": {"name": "json"}},
|
||||
},
|
||||
{"close": ["7X", "</call>"], "content_args": END_X},
|
||||
{"close": [">X", "</call>"]},
|
||||
{"close": ["></call>", "</call>"]},
|
||||
{"close": ["><a", "</call>"]},
|
||||
{"close": ["}</call>", "</call>"], "content": "json", "content_args": {}},
|
||||
],
|
||||
)
|
||||
def test_underivable_tool_formats_are_not_derived(change):
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"].update(change)
|
||||
|
||||
assert build(template) is None
|
||||
|
||||
|
||||
def test_non_literal_thinking_delimiters_are_not_derived():
|
||||
template = deepcopy(TEMPLATE)
|
||||
del template["fields"]["thinking"]["open"]
|
||||
template["fields"]["thinking"]["open_pattern"] = r"<think(?: mode=\w+)?>"
|
||||
|
||||
assert build(template) is None
|
||||
assert build(template, thinking=False) is not None
|
||||
|
||||
|
||||
LEVEL = {"type": "object", "properties": {"level": {"$ref": "#/$defs/level"}}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "value", "accepted"),
|
||||
[
|
||||
({"type": "integer", "nullable": True}, "null", True),
|
||||
({"type": "integer", "nullable": True}, "soon", False),
|
||||
({"type": ["integer", "null"]}, "null", True),
|
||||
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "true", True),
|
||||
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "soon", False),
|
||||
({"type": ["string", "integer"]}, "soon", True),
|
||||
({"enum": ["celsius", "fahrenheit"]}, "celsius", True),
|
||||
({"enum": ["celsius", "fahrenheit"]}, "kelvin", False),
|
||||
({"type": "string", "const": "on"}, "off", False),
|
||||
(LEVEL, '{"level": 2}', True),
|
||||
(LEVEL, '{"level": "high"}', False),
|
||||
],
|
||||
)
|
||||
def test_argument_values_follow_how_the_parser_reads_them(schema, value, accepted):
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"v": schema},
|
||||
"required": ["v"],
|
||||
"$defs": {"level": {"type": "integer"}},
|
||||
}
|
||||
|
||||
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "strict", "derivable"),
|
||||
[
|
||||
# The parser reads values without a type as text.
|
||||
({"enum": [1, 2]}, True, False),
|
||||
({"$ref": "#/$defs/level"}, True, False),
|
||||
({"$ref": "#/$defs/level"}, False, True),
|
||||
# Raw text cannot hold its closer.
|
||||
({"enum": ["a</arg>b"]}, True, False),
|
||||
({"type": "object", "properties": {"x": {"$ref": "#/x"}}}, True, False),
|
||||
(False, True, False),
|
||||
# A match could write the start of the value's end.
|
||||
({"type": "string", "pattern": "^[a-z<]+$"}, True, False),
|
||||
({"type": "string", "pattern": "^[a-z<]+$"}, False, True),
|
||||
({"type": "string", "pattern": "^a.b$"}, True, False),
|
||||
# XGrammar holds length bounds only in JSON.
|
||||
({"type": "string", "minLength": 3}, True, False),
|
||||
({"type": "string", "format": "ipv6"}, True, False),
|
||||
],
|
||||
)
|
||||
def test_schemas_text_values_cannot_hold_are_not_derived(schema, strict, derivable):
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"v": schema},
|
||||
"$defs": {"level": {"type": "integer"}},
|
||||
}
|
||||
|
||||
assert (build(functions=[tool(parameters, strict=strict)]) is not None) is derivable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "value", "accepted"),
|
||||
[
|
||||
({"type": "string", "format": "email"}, "a.b@x.co", True),
|
||||
({"type": "string", "format": "email"}, "ab", False),
|
||||
({"type": "string", "format": "date-time"}, "2026-10-06T12:00:00Z", True),
|
||||
({"type": "string", "format": "date-time"}, "2026-13-06T12:00:00Z", False),
|
||||
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABC", True),
|
||||
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABCD", False),
|
||||
# Like XGrammar, the format overrides the pattern and the length bounds.
|
||||
(
|
||||
{"type": "string", "format": "uuid", "pattern": "^x$", "maxLength": 1},
|
||||
"123e4567-e89b-12d3-a456-426614174000",
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_text_values_hold_string_formats_and_patterns(schema, value, accepted):
|
||||
parameters = {"type": "object", "properties": {"v": schema}, "required": ["v"]}
|
||||
|
||||
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
|
||||
|
||||
|
||||
def test_values_cannot_hold_a_closer():
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"]["close"] = ["</call>", "STOP"]
|
||||
text = {"type": "object", "properties": {"v": {"type": "string"}}}
|
||||
strings = {"type": "array", "items": {"type": "string"}}
|
||||
json_text = {"type": "object", "properties": {"v": strings}}
|
||||
|
||||
assert accepts(build(template, [tool(text)]), call(("v", "ab")))
|
||||
assert not accepts(build(template, [tool(text)]), call(("v", "aSTOPb")))
|
||||
assert not accepts(
|
||||
build(template, [tool(None, strict=False)]), call(("v", "aSTOPb"))
|
||||
)
|
||||
assert accepts(build(template, [tool(json_text)]), call(("v", '["ab"]')))
|
||||
assert not accepts(build(template, [tool(json_text)]), call(("v", '["aSTOPb"]')))
|
||||
|
||||
|
||||
def test_extra_arguments_need_additional_properties():
|
||||
parameters = {"type": "object", "properties": {"city": {"type": "string"}}}
|
||||
extra = call(("city", "Paris"), ("zip", "75001"))
|
||||
|
||||
assert not accepts(build(functions=[tool(parameters)]), extra)
|
||||
# Next to declared keys, an extra one could repeat them unchecked.
|
||||
allowed = {**parameters, "additionalProperties": True}
|
||||
assert build(functions=[tool(allowed)]) is None
|
||||
free_form = {"type": "object", "additionalProperties": True}
|
||||
assert accepts(build(functions=[tool(free_form)]), extra)
|
||||
|
||||
|
||||
def test_only_strict_tools_follow_their_schema():
|
||||
off_schema = call(("days", "soon"))
|
||||
parameters = WEATHER["parameters"]
|
||||
|
||||
assert accepts(build(functions=[tool(parameters, strict=False)]), off_schema)
|
||||
assert not accepts(build(functions=[tool(parameters)]), off_schema)
|
||||
assert accepts(build(functions=[tool(None)]), call())
|
||||
assert not accepts(build(functions=[tool(None)]), call(("a", "x")))
|
||||
|
||||
|
||||
def test_annotations_keep_strict_arguments_to_their_schema():
|
||||
parameters = {
|
||||
**WEATHER["parameters"],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "weather",
|
||||
}
|
||||
tag = build(functions=[tool(parameters)])
|
||||
|
||||
assert accepts(tag, call(("city", "Paris"), ("days", "3")))
|
||||
assert not accepts(tag, call(("city", "Paris"), ("days", "soon")))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arguments", "accepted", "accepted_sorted"),
|
||||
[
|
||||
((("path", "a"), ("content", "b")), True, True),
|
||||
((("content", "b"), ("path", "a")), False, True),
|
||||
((("path", "a"),), True, True),
|
||||
((("content", "b"),), False, False),
|
||||
((("path", "a"), ("path", "c")), False, False),
|
||||
((("path", "a"), ("mode", "c")), False, False),
|
||||
],
|
||||
)
|
||||
def test_strict_arguments_follow_declared_or_sorted_order(
|
||||
arguments, accepted, accepted_sorted
|
||||
):
|
||||
parameters = {
|
||||
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
"required": ["path"],
|
||||
}
|
||||
tags = [
|
||||
build(functions=[tool(parameters)], sorted_arguments=sorted_arguments)
|
||||
for sorted_arguments in (False, True)
|
||||
]
|
||||
|
||||
assert [accepts(tag, call(*arguments)) for tag in tags] == [
|
||||
accepted,
|
||||
accepted_sorted,
|
||||
]
|
||||
|
||||
|
||||
LOOKAHEAD_TEMPLATE = deepcopy(TEMPLATE)
|
||||
LOOKAHEAD_TEMPLATE["fields"]["tool_calls"]["content_args"]["tag_pattern"] = (
|
||||
r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "accepted_loose", "accepted_strict"),
|
||||
[
|
||||
('<arg name="city">Paris\n</arg>\n<arg name="days">3\n</arg>', True, True),
|
||||
('<arg name="city">Paris\n<arg name="days">3\n</arg>', True, True),
|
||||
('<arg name="days">3\n<arg name="city">Paris\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="city">Lyon\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="hours">3\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="', False, False),
|
||||
('<arg name="city">Paris\n<arg name="days\n</arg>', False, False),
|
||||
('<arg name="city">a <arg name="b"> c\n</arg>', True, True),
|
||||
],
|
||||
)
|
||||
def test_values_end_where_a_lookahead_end_does(body, accepted_loose, accepted_strict):
|
||||
parameters = WEATHER["parameters"]
|
||||
tags = [
|
||||
build(LOOKAHEAD_TEMPLATE, [tool(parameters, strict=strict)])
|
||||
for strict in (False, True)
|
||||
]
|
||||
text = f'<call name="f">{body}</call>'
|
||||
|
||||
assert [accepts(tag, text) for tag in tags] == [accepted_loose, accepted_strict]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lookahead", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("keys", "accepted"),
|
||||
[
|
||||
("b", True),
|
||||
("ab", True),
|
||||
("bc", True),
|
||||
("bd", True),
|
||||
("be", True),
|
||||
("abde", True),
|
||||
("bcde", True),
|
||||
("", False),
|
||||
("a", False),
|
||||
("acd", False),
|
||||
("bdc", False),
|
||||
("bcc", False),
|
||||
],
|
||||
)
|
||||
def test_last_parameter_follows_every_required_one(keys, accepted, lookahead):
|
||||
parameters = {
|
||||
"properties": {key: {"type": "string"} for key in "abcde"},
|
||||
"required": ["b"],
|
||||
}
|
||||
tag = build(LOOKAHEAD_TEMPLATE, [tool(parameters)])
|
||||
ends = ["\n" if lookahead else "\n</arg>"] * (len(keys) - 1) + ["\n</arg>"]
|
||||
body = "".join(f'<arg name="{k}">x{e}' for k, e in zip(keys, ends))
|
||||
|
||||
assert accepts(tag, f'<call name="f">{body}</call>') is accepted
|
||||
|
||||
|
||||
def test_required_argument_without_schema_is_still_required():
|
||||
parameters = {"type": "object", "properties": {}, "required": ["a"]}
|
||||
tag = build(functions=[tool(parameters)])
|
||||
|
||||
assert accepts(tag, call(("a", "x")))
|
||||
assert not accepts(tag, call())
|
||||
assert build(functions=[tool({**parameters, "required": ["a", 1]})]) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("function", "match"),
|
||||
[
|
||||
(tool({}, name="get weather"), "tool name"),
|
||||
(tool({"properties": {'a"b': {"type": "string"}}}), "parameter name"),
|
||||
],
|
||||
)
|
||||
def test_unencodable_names_are_rejected(function, match):
|
||||
with pytest.raises(ValueError, match=match):
|
||||
build(functions=[function])
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
@@ -376,6 +377,32 @@ class TestRenderPrompt:
|
||||
assert results[0]["prompt_token_ids"] == tokens
|
||||
assert results[0]["prompt"] == "[1, 2, 3, 4]"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("truncation_side", "expected"),
|
||||
[("left", [106, 107, 108, 109]), ("right", [100, 101, 102, 103])],
|
||||
)
|
||||
def test_detokenization_matches_truncated_tokens(self, truncation_side, expected):
|
||||
"""Echoed prompt text must match the truncated tokens sent to the engine."""
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
params = TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side=truncation_side,
|
||||
needs_detokenization=True,
|
||||
)
|
||||
|
||||
def render():
|
||||
return renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, list(range(100, 110)))
|
||||
)
|
||||
|
||||
results = renderer.tokenize_prompts(render(), params)
|
||||
async_results = asyncio.run(renderer.tokenize_prompts_async(render(), params))
|
||||
|
||||
for result in (*results, *async_results):
|
||||
assert result["prompt_token_ids"] == expected
|
||||
assert result["prompt"] == str(expected)
|
||||
|
||||
def test_explicit_side_tokenizer_unbounded(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
|
||||
@@ -48,6 +48,14 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
|
||||
assert "VLLM_API_KEY" not in envs.compile_factors()
|
||||
|
||||
|
||||
def test_ray_bundle_indices_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
|
||||
# Engines sharing a placement group get different bundle indices but
|
||||
# compile the same graphs, so they must share a cache key.
|
||||
monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", "4,5,6,7")
|
||||
|
||||
assert "VLLM_RAY_BUNDLE_INDICES" not in envs.compile_factors()
|
||||
|
||||
|
||||
def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False)
|
||||
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False)
|
||||
|
||||
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
cu_num_tokens,
|
||||
cu_num_tokens[1:] - cu_num_tokens[:-1],
|
||||
KERNEL_BLOCK_SIZE=kernel_block_size,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
|
||||
bt.stride(0),
|
||||
bt.stride(1),
|
||||
cu,
|
||||
cu[1:] - cu[:-1],
|
||||
KERNEL_BLOCK_SIZE=kbs,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ Known Issues:
|
||||
|
||||
import sys
|
||||
from types import MethodType, SimpleNamespace
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -47,6 +48,9 @@ from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
MLAPrefillSelectorConfig,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
|
||||
is_cute_dsl_mla_prefill_supported,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -418,6 +422,11 @@ for backend in BACKENDS_TO_TEST:
|
||||
if query_len_support != QueryLenSupport.SINGLE_ONLY:
|
||||
SPEC_DECODE_BACKENDS.append(backend)
|
||||
|
||||
# Backends whose decode pipeline replays device-trimmed verify batches.
|
||||
ADAPTIVE_VARLEN_BACKENDS = [
|
||||
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
|
||||
]
|
||||
|
||||
BACKEND_BLOCK_SIZES = {}
|
||||
for backend in BACKENDS_TO_TEST:
|
||||
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
|
||||
@@ -470,6 +479,10 @@ BATCH_SPECS = {
|
||||
"spec_decode_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
|
||||
),
|
||||
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
|
||||
"spec_decode_ragged_small": BatchSpec(
|
||||
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
|
||||
),
|
||||
"spec_decode_medium": BatchSpec(
|
||||
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
|
||||
),
|
||||
@@ -1474,6 +1487,7 @@ def run_attention_backend(
|
||||
|
||||
# Force the prefill backend selection (None means auto-select).
|
||||
vllm_config.attention_config.mla_prefill_backend = prefill_backend
|
||||
mla_attention_module.backend_supports_prefill_query_quantization.cache_clear()
|
||||
|
||||
# Set the current vllm config so that get_current_vllm_config() works
|
||||
# in the backend implementations
|
||||
@@ -1581,6 +1595,42 @@ def run_attention_backend(
|
||||
return output
|
||||
|
||||
|
||||
def _with_even_cpu_query_split(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> CommonAttentionMetadata:
|
||||
"""Host query lengths as adaptive verification leaves them.
|
||||
|
||||
The device keeps the trimmed per-request lengths; the host sees the same
|
||||
total spread evenly, the first requests taking the remainder
|
||||
(AdaptiveVerificationManager.compact_batch).
|
||||
"""
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
|
||||
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
|
||||
cpu_lens[: total % num_reqs] += 1
|
||||
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
|
||||
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
|
||||
return common_attn_metadata.replace(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=int(
|
||||
(
|
||||
common_attn_metadata.query_start_loc[1:]
|
||||
- common_attn_metadata.query_start_loc[:-1]
|
||||
)
|
||||
.max()
|
||||
.item()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _fp8_cache_roundtrip(tensor: torch.Tensor, scale: float) -> torch.Tensor:
|
||||
"""Reproduce cache storage rounding for the SDPA reference."""
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
limits = torch.finfo(fp8_dtype)
|
||||
quantized = (tensor.float() / scale).clamp(limits.min, limits.max).to(fp8_dtype)
|
||||
return (quantized.float() * scale).to(tensor.dtype)
|
||||
|
||||
|
||||
def _run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -1595,6 +1645,9 @@ def _run_backend_correctness(
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
adaptive_verification: bool = False,
|
||||
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"] = "trtllm-gen",
|
||||
use_prefill_query_quantization: bool = False,
|
||||
):
|
||||
"""Test that all backends produce similar outputs to a reference implementation
|
||||
using torch.nn.functional.scaled_dot_product_attention.
|
||||
@@ -1626,6 +1679,10 @@ def _run_backend_correctness(
|
||||
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
|
||||
# CUTLASS_MLA does not support non-1 Q/K scales
|
||||
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||
if adaptive_verification:
|
||||
backends_to_test = [
|
||||
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
|
||||
]
|
||||
if not backends_to_test:
|
||||
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
|
||||
|
||||
@@ -1670,18 +1727,25 @@ def _run_backend_correctness(
|
||||
hf_config_override["num_attention_heads"]
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.attention_config.trtllm_ragged_backend = trtllm_ragged_backend
|
||||
vllm_config.attention_config.use_prefill_query_quantization = (
|
||||
use_prefill_query_quantization
|
||||
)
|
||||
|
||||
# For spec decode tests, add a speculative_config to set the reorder_batch_threshold
|
||||
if is_spec_decode_test:
|
||||
from vllm.config import SpeculativeConfig
|
||||
|
||||
# Get the query length from the batch spec (they should all be uniform)
|
||||
query_len = batch_spec.query_lens[0]
|
||||
# Get the query length from the batch spec (they should all be uniform,
|
||||
# except under adaptive verification, where the longest is k+1)
|
||||
query_len = max(batch_spec.query_lens)
|
||||
# Set num_speculative_tokens to query_len - 1
|
||||
# (since threshold is 1 + num_spec_tokens)
|
||||
# Use ngram method which doesn't require a draft model
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=query_len - 1
|
||||
method="ngram",
|
||||
num_speculative_tokens=query_len - 1,
|
||||
enable_adaptive_verification=adaptive_verification,
|
||||
)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
@@ -1756,6 +1820,19 @@ def _run_backend_correctness(
|
||||
# K_PE (rope component): [s_len, 1, qk_rope_head_dim]
|
||||
k_pe_full = torch.randn(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
|
||||
|
||||
kv_c_cached, k_pe_cached = kv_c_full, k_pe_full
|
||||
kv_c_prefill, k_pe_prefill = kv_c_full, k_pe_full
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
kv_c_cached = _fp8_cache_roundtrip(kv_c_full, k_scale)
|
||||
k_pe_cached = _fp8_cache_roundtrip(k_pe_full, k_scale)
|
||||
# Prefill reads cached context and unquantized new tokens.
|
||||
kv_c_prefill = torch.cat(
|
||||
[kv_c_cached[:context_len], kv_c_full[context_len:]]
|
||||
)
|
||||
k_pe_prefill = torch.cat(
|
||||
[k_pe_cached[:context_len], k_pe_full[context_len:]]
|
||||
)
|
||||
|
||||
# Determine if this sequence uses the decode pipeline or prefill
|
||||
# pipeline for each backend
|
||||
# NOTE: For spec decode tests with uniform query_len > 1, backends that
|
||||
@@ -1805,10 +1882,10 @@ def _run_backend_correctness(
|
||||
q_mqa = torch.cat([ql_nope, q_pe], dim=-1)
|
||||
# K: [s_len, kv_lora_rank + qk_rope_head_dim]
|
||||
# (broadcasted to all heads)
|
||||
k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1)
|
||||
k_mqa = torch.cat([kv_c_cached, k_pe_cached.squeeze(1)], dim=-1)
|
||||
k_mqa = k_mqa.unsqueeze(1).expand(-1, num_q_heads, -1)
|
||||
# V: [s_len, kv_lora_rank] (broadcasted to all heads)
|
||||
v_mqa = kv_c_full.unsqueeze(1).expand(-1, num_q_heads, -1)
|
||||
v_mqa = kv_c_cached.unsqueeze(1).expand(-1, num_q_heads, -1)
|
||||
|
||||
# Create custom attention mask for decode path:
|
||||
# - Query tokens can attend to all context tokens
|
||||
@@ -1837,13 +1914,18 @@ def _run_backend_correctness(
|
||||
#######################################################
|
||||
# Prefill path: MHA-style attention with full sequence
|
||||
# Apply kv_b_proj to the full kv_c tensor
|
||||
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_full, kv_b_proj_weight)
|
||||
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_prefill, kv_b_proj_weight)
|
||||
k_nope_full, v_full = kv_nope_full.split([qk_nope_head_dim, v_head_dim], dim=-1)
|
||||
|
||||
# Build attention inputs for full sequence
|
||||
q_mha = torch.cat([q_nope, q_pe], dim=-1) # [q_len, num_heads, total_dim]
|
||||
k_pe_full_expanded = k_pe_full.expand(-1, num_q_heads, -1)
|
||||
k_pe_full_expanded = k_pe_prefill.expand(-1, num_q_heads, -1)
|
||||
k_full = torch.cat([k_nope_full, k_pe_full_expanded], dim=-1)
|
||||
if use_prefill_query_quantization:
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
q_mha = q_mha.to(fp8_dtype).to(dtype)
|
||||
k_full = k_full.to(fp8_dtype).to(dtype)
|
||||
v_full = v_full.to(fp8_dtype).to(dtype)
|
||||
|
||||
# Create custom attention mask:
|
||||
# - Query tokens can attend to all context tokens
|
||||
@@ -1951,6 +2033,13 @@ def _run_backend_correctness(
|
||||
)
|
||||
kv_cache_per_block_size[block_size] = kv_cache
|
||||
|
||||
if adaptive_verification:
|
||||
# Only after the cache is filled: prepopulation reads the context
|
||||
# lengths off the host query lengths.
|
||||
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
|
||||
common_attn_metadata
|
||||
)
|
||||
|
||||
# 4. Run vLLM backends and compare
|
||||
rtol = 1e-2
|
||||
atol = {
|
||||
@@ -2111,6 +2200,8 @@ def test_backend_correctness(
|
||||
_prefill_backend_dimension_params(),
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("trtllm_ragged_backend", ["trtllm-gen", "cute-dsl"])
|
||||
@pytest.mark.parametrize("use_prefill_query_quantization", [False, True])
|
||||
def test_chunked_context_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -2119,8 +2210,26 @@ def test_chunked_context_backend_correctness(
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_cache_dtype: str,
|
||||
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"],
|
||||
use_prefill_query_quantization: bool,
|
||||
):
|
||||
"""Split, packed, and context-free requests match the SDPA reference."""
|
||||
if prefill_backend != MLAPrefillBackendEnum.TRTLLM_RAGGED and (
|
||||
trtllm_ragged_backend == "cute-dsl" or use_prefill_query_quantization
|
||||
):
|
||||
pytest.skip("TRTLLM ragged-specific test configuration")
|
||||
if use_prefill_query_quantization and kv_cache_dtype == "auto":
|
||||
pytest.skip("FP8 prefill queries require an FP8 KV cache")
|
||||
if trtllm_ragged_backend == "cute-dsl":
|
||||
query_dtype = (
|
||||
current_platform.fp8_dtype()
|
||||
if use_prefill_query_quantization
|
||||
else torch.bfloat16
|
||||
)
|
||||
if not is_cute_dsl_mla_prefill_supported(
|
||||
qk_nope_head_dim + 64, v_head_dim, query_dtype
|
||||
):
|
||||
pytest.skip("Unsupported CuTe DSL prefill dimensions or query dtype")
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
@@ -2135,4 +2244,32 @@ def test_chunked_context_backend_correctness(
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
chunked_prefill_workspace_size=1024,
|
||||
trtllm_ragged_backend=trtllm_ragged_backend,
|
||||
use_prefill_query_quantization=use_prefill_query_quantization,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
|
||||
)
|
||||
def test_adaptive_varlen_decode_correctness(
|
||||
default_vllm_config, dist_init, workspace_init
|
||||
):
|
||||
"""Device query lengths 1..k+1 against an even host split, as adaptive
|
||||
verification hands them to the target model's attention. TP 8 gives 16
|
||||
heads, the padded Kimi-K3 count."""
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
"spec_decode_ragged_small",
|
||||
"deepseek-ai/DeepSeek-R1",
|
||||
8,
|
||||
"fp8",
|
||||
1.0,
|
||||
1.0,
|
||||
None,
|
||||
128,
|
||||
128,
|
||||
adaptive_verification=True,
|
||||
)
|
||||
|
||||
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
|
||||
stride_0,
|
||||
stride_1,
|
||||
paged_kv_indptr,
|
||||
num_real_tokens,
|
||||
*,
|
||||
KERNEL_BLOCK_SIZE,
|
||||
BLOCK_SIZE,
|
||||
CLAMP_TAIL=False,
|
||||
):
|
||||
# Both block-table strides are passed explicitly: the kernel no longer
|
||||
# assumes the column stride is 1. This stand-in indexes the tensor
|
||||
@@ -145,6 +147,7 @@ def _builder(
|
||||
)
|
||||
),
|
||||
_mtp_decode_qlen=mtp_decode_qlen,
|
||||
_varlen_decode_qlen=0,
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = 576**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=4,
|
||||
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
|
||||
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
|
||||
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.scale = head_dim**-0.5
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=1,
|
||||
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
|
||||
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
|
||||
)
|
||||
decode = SimpleNamespace(
|
||||
varlen_q_rows=None,
|
||||
max_qo_len=qlen,
|
||||
paged_kv_indptr=torch.tensor(
|
||||
[0, local_kv.shape[0]], dtype=torch.int32, device=device
|
||||
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
|
||||
|
||||
def _adaptive_config(
|
||||
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
enable_adaptive_verification=adaptive,
|
||||
parallel_drafting=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
|
||||
|
||||
def test_varlen_cudagraph_bound_follows_adaptive_verification():
|
||||
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_adaptive_config(), SimpleNamespace()) == 8
|
||||
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
|
||||
# DCP routes plan per-row windows from host lengths.
|
||||
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
|
||||
# Kimi-K3 DSpark draft layers share the target's group; causality is
|
||||
# decided per batch, so the merged non-causal spec keeps the bound.
|
||||
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
|
||||
assert bound(_adaptive_config(), shared_draft_group) == 8
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for KVCacheConfigBuilder resolution."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import Platform
|
||||
from vllm.v1.core.kv_cache_config_builder import (
|
||||
KVCacheConfigBuilder,
|
||||
get_kv_cache_config_builder,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
DefaultKVCacheConfigBuilder,
|
||||
get_kv_cache_groups_from_builder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
|
||||
def _make_vllm_config(supports_grouping: bool = False) -> MagicMock:
|
||||
"""Create a minimal mock VllmConfig for builder resolution tests.
|
||||
|
||||
``supports_grouping`` selects whether the resolved model class declares
|
||||
``SupportsKVCacheGrouping`` (exposing a ``get_kv_cache_groups`` classmethod).
|
||||
"""
|
||||
cfg = MagicMock()
|
||||
cfg.model_config.supports_kv_cache_grouping = supports_grouping
|
||||
cfg.attention_config.hisparse_config = None
|
||||
if supports_grouping:
|
||||
cfg.model_config.registry.resolve_model_cls.return_value = (
|
||||
_GroupedModel,
|
||||
"arch",
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
CUSTOM_PATH = "tests.v1.core.test_kv_cache_config_builder.CustomBuilder"
|
||||
DEFAULT_PATH = "vllm.v1.core.kv_cache_planning.DefaultKVCacheConfigBuilder"
|
||||
|
||||
|
||||
class CustomBuilder(DefaultKVCacheConfigBuilder):
|
||||
"""A test builder subclass."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class _GroupedModel:
|
||||
"""A test model class with a ``get_kv_cache_groups`` classmethod."""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
|
||||
return []
|
||||
|
||||
|
||||
class _PartialGroupedModel:
|
||||
"""A test model class whose grouping omits layers."""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
|
||||
|
||||
return [KVCacheGroupSpec(["a"], kv_cache_spec["a"])]
|
||||
|
||||
|
||||
class TestPlatformHookResolution:
|
||||
"""The platform hook selects only the placement builder."""
|
||||
|
||||
def test_default_hook_returns_default_builder(self):
|
||||
cfg = _make_vllm_config()
|
||||
assert Platform.get_kv_cache_config_builder_cls(cfg) == DEFAULT_PATH
|
||||
|
||||
|
||||
class TestBuilderResolution:
|
||||
@patch("vllm.platforms.current_platform")
|
||||
@pytest.mark.parametrize(
|
||||
("builder_cls_path", "expected_cls"),
|
||||
[
|
||||
(None, DefaultKVCacheConfigBuilder),
|
||||
(CUSTOM_PATH, CustomBuilder),
|
||||
],
|
||||
)
|
||||
def test_resolves_hook_selected_builder(
|
||||
self, mock_platform, builder_cls_path, expected_cls
|
||||
):
|
||||
mock_platform.get_kv_cache_config_builder_cls.return_value = (
|
||||
builder_cls_path or DEFAULT_PATH
|
||||
)
|
||||
cfg = _make_vllm_config()
|
||||
assert type(get_kv_cache_config_builder(cfg)) is expected_cls
|
||||
|
||||
@patch("vllm.platforms.current_platform")
|
||||
def test_resolves_fresh_instance_per_call(self, mock_platform):
|
||||
mock_platform.get_kv_cache_config_builder_cls.return_value = CUSTOM_PATH
|
||||
cfg = _make_vllm_config()
|
||||
assert get_kv_cache_config_builder(cfg) is not get_kv_cache_config_builder(cfg)
|
||||
|
||||
def test_public_interface_declares_two_placement_hooks(self):
|
||||
# get_kv_cache_configs lives in kv_cache_planning and is Core-owned;
|
||||
# the builder only declares the placement hooks.
|
||||
assert KVCacheConfigBuilder.__abstractmethods__ == {
|
||||
"get_kv_cache_groups",
|
||||
"get_kv_cache_config_from_groups",
|
||||
}
|
||||
assert issubclass(DefaultKVCacheConfigBuilder, KVCacheConfigBuilder)
|
||||
assert (
|
||||
"pool_bytes_per_block"
|
||||
not in DefaultKVCacheConfigBuilder.__abstractmethods__
|
||||
)
|
||||
assert (
|
||||
"max_memory_usage_bytes"
|
||||
not in DefaultKVCacheConfigBuilder.__abstractmethods__
|
||||
)
|
||||
|
||||
|
||||
class TestModelGroupingPriority:
|
||||
"""A model class with a get_kv_cache_groups classmethod owns grouping."""
|
||||
|
||||
def test_model_classmethod_owns_grouping(self):
|
||||
cfg = _make_vllm_config(supports_grouping=True)
|
||||
assert DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {}) == []
|
||||
|
||||
def test_model_grouping_rejects_hisparse(self):
|
||||
cfg = _make_vllm_config(supports_grouping=True)
|
||||
cfg.attention_config.hisparse_config = object()
|
||||
with pytest.raises(ValueError, match="HiSparse"):
|
||||
DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {})
|
||||
|
||||
|
||||
class TestCoreChecksGrouping:
|
||||
"""Core checks the grouping whether a model or a platform produced it."""
|
||||
|
||||
_SPEC = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
|
||||
)
|
||||
|
||||
def test_model_grouping_is_checked(self):
|
||||
cfg = _make_vllm_config(supports_grouping=True)
|
||||
cfg.model_config.registry.resolve_model_cls.return_value = (
|
||||
_PartialGroupedModel,
|
||||
"arch",
|
||||
)
|
||||
with pytest.raises(AssertionError, match="miss layers"):
|
||||
get_kv_cache_groups_from_builder(
|
||||
DefaultKVCacheConfigBuilder(), cfg, {"a": self._SPEC, "b": self._SPEC}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("groups", "match"),
|
||||
[
|
||||
([KVCacheGroupSpec(["a"], _SPEC)], "miss layers"),
|
||||
(
|
||||
[KVCacheGroupSpec(["a", "b"], _SPEC), KVCacheGroupSpec(["b"], _SPEC)],
|
||||
"overlaps",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_platform_grouping_is_checked(self, groups, match):
|
||||
class FixedGroupsBuilder(DefaultKVCacheConfigBuilder):
|
||||
def get_kv_cache_groups(self, vllm_config, kv_cache_spec):
|
||||
return groups
|
||||
|
||||
with pytest.raises(AssertionError, match=match):
|
||||
get_kv_cache_groups_from_builder(
|
||||
FixedGroupsBuilder(),
|
||||
_make_vllm_config(),
|
||||
{"a": self._SPEC, "b": self._SPEC},
|
||||
)
|
||||
|
||||
|
||||
class TestPlatformCustomPriority:
|
||||
"""A vendor platform can override the hook to customize priority."""
|
||||
|
||||
def test_platform_builder_wins_over_model_declaration(self):
|
||||
class PlatformFirstPlatform(Platform):
|
||||
@classmethod
|
||||
def get_kv_cache_config_builder_cls(cls, vllm_config):
|
||||
return CUSTOM_PATH
|
||||
|
||||
cfg = _make_vllm_config()
|
||||
assert PlatformFirstPlatform.get_kv_cache_config_builder_cls(cfg) == CUSTOM_PATH
|
||||
with patch("vllm.platforms.current_platform", PlatformFirstPlatform):
|
||||
assert isinstance(get_kv_cache_config_builder(cfg), CustomBuilder)
|
||||
|
||||
|
||||
class TestProjectionPreservesGroupMetadata:
|
||||
"""PP projection must carry host_resident/role/enable_kv_transfer."""
|
||||
|
||||
def test_partial_projection_preserves_metadata(self):
|
||||
from vllm.v1.core.kv_cache_planning import _project_kv_cache_groups_to_worker
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupRole,
|
||||
)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
|
||||
)
|
||||
group = MagicMock()
|
||||
group.layer_names = ["layer1", "layer2", "layer3"]
|
||||
group.kv_cache_spec = spec
|
||||
group.is_eagle_group = False
|
||||
group.host_resident = True
|
||||
group.enable_kv_transfer = False
|
||||
group.role = KVCacheGroupRole.HISPARSE_SOURCE
|
||||
|
||||
projected = _project_kv_cache_groups_to_worker(
|
||||
[group], {"layer1": spec, "layer2": spec}
|
||||
)
|
||||
assert len(projected) == 1
|
||||
p = projected[0]
|
||||
assert p.host_resident is True
|
||||
assert p.enable_kv_transfer is False
|
||||
assert p.role is KVCacheGroupRole.HISPARSE_SOURCE
|
||||
assert p.layer_names == ["layer1", "layer2"]
|
||||
|
||||
|
||||
class TestBytesPerBlockProbe:
|
||||
"""The single-block probe must not be polluted by num_gpu_blocks_override."""
|
||||
|
||||
def test_probe_suppresses_override(self):
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
DefaultKVCacheConfigBuilder,
|
||||
_pool_bytes_per_block,
|
||||
_pool_bytes_per_block_from_builder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.float16
|
||||
)
|
||||
vc = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=100),
|
||||
cache_config=CacheConfig(block_size=16),
|
||||
)
|
||||
resolve_kv_cache_layout(vc, [["BLHNC"]], [spec])
|
||||
groups = [
|
||||
KVCacheGroupSpec(["a"], spec),
|
||||
KVCacheGroupSpec(["b"], spec),
|
||||
]
|
||||
|
||||
pure_c = _pool_bytes_per_block(vc, groups)
|
||||
vc.cache_config.num_gpu_blocks_override = 500
|
||||
probed_c = _pool_bytes_per_block_from_builder(
|
||||
DefaultKVCacheConfigBuilder(), vc, groups
|
||||
)
|
||||
assert probed_c == pure_c
|
||||
assert vc.cache_config.num_gpu_blocks_override == 500
|
||||
|
||||
@pytest.mark.parametrize("declares_pool_bytes", [False, True])
|
||||
def test_several_device_allocations_need_declared_pool_bytes(
|
||||
self, declares_pool_bytes
|
||||
):
|
||||
"""A placement with one allocation per group cannot be sized from one
|
||||
backing; the builder must declare its per-block bytes."""
|
||||
from vllm.v1.core.kv_cache_planning import _pool_bytes_per_block_from_builder
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
|
||||
class PerGroupAllocationBuilder(DefaultKVCacheConfigBuilder):
|
||||
def get_kv_cache_config_from_groups(self, vllm_config, groups, num_blocks):
|
||||
tensors = [
|
||||
KVCacheTensor(
|
||||
size=page * num_blocks,
|
||||
layers=group.layer_names,
|
||||
layer_stride=page * num_blocks,
|
||||
block_stride=page,
|
||||
)
|
||||
for group, page in zip(groups, (64, 32))
|
||||
]
|
||||
return KVCacheConfig(num_blocks, tensors, groups)
|
||||
|
||||
def _get_pool_bytes_per_block(self, vllm_config, groups):
|
||||
return 64 + 32 if declares_pool_bytes else None
|
||||
|
||||
vc = _make_vllm_config()
|
||||
vc.cache_config.num_gpu_blocks_override = None
|
||||
groups = [KVCacheGroupSpec([name], MagicMock()) for name in "ab"]
|
||||
builder = PerGroupAllocationBuilder()
|
||||
if declares_pool_bytes:
|
||||
assert _pool_bytes_per_block_from_builder(builder, vc, groups) == 96
|
||||
else:
|
||||
with pytest.raises(ValueError, match="_get_pool_bytes_per_block"):
|
||||
_pool_bytes_per_block_from_builder(builder, vc, groups)
|
||||
@@ -18,6 +18,7 @@ from vllm.inputs import PromptType
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import EngineCoreRequestType
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.core_client import DPLBAsyncMPClient
|
||||
from vllm.v1.metrics.loggers import StatLoggerBase
|
||||
@@ -336,12 +337,12 @@ async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_late_request_does_not_block_drain():
|
||||
"""A request arriving after pause must not leave the coordinator believing
|
||||
"""A request arriving after pause must not make the coordinator believe
|
||||
the engines are running.
|
||||
|
||||
Paused engines discard START_DP_WAVE, so nothing can report wave
|
||||
completion afterwards; if forwarding the wake also marks the engines as
|
||||
running, drain has no way back and can only end in a timeout.
|
||||
Paused engines start no wave for it, so nothing could report wave
|
||||
completion afterwards; a running flag set on its behalf would leave drain
|
||||
no way back but a timeout.
|
||||
|
||||
MoE only: wave coordination is enabled iff the model is MoE, so a dense
|
||||
model never reaches the coordinator state this exercises.
|
||||
@@ -365,11 +366,9 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
|
||||
# A design that never reports the engines as running would satisfy
|
||||
# every drain assertion below while destroying the signal, so pin it
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
# Rank 0 reports the wave only after its first step, which can outlast
|
||||
# the 2s on a cold device, so wait for the report rather than sample it.
|
||||
# down first. Rank 0 reports the wave only after its first step, which
|
||||
# can outlast the 2s on a cold device, so wait for the report rather
|
||||
# than sample it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert await _poll_flag(engine, True, timeout=30), (
|
||||
@@ -382,24 +381,12 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
await engine.pause_generation(mode="abort")
|
||||
await engine.wait_for_requests_to_drain(drain_timeout=30)
|
||||
|
||||
# Awaiting add_request guarantees the new-request notification has been
|
||||
# sent to the coordinator - the message that used to latch the flag.
|
||||
collector = await engine.add_request(
|
||||
request_id="late",
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
# The front-end marks the engines running off the back of that
|
||||
# notification. This is what makes the test non-vacuous: it is the
|
||||
# path that used to leave the coordinator stuck.
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the late request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# It must settle back by itself. Unfixed it never does, because the
|
||||
# paused engines discard the wake and so never report wave completion.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
assert not await _poll_flag(engine, True, timeout=5)
|
||||
|
||||
# The late request was held rather than dropped: it completes on resume.
|
||||
await engine.resume_generation()
|
||||
@@ -409,6 +396,46 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
break
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_request_reaching_rank_after_resume_wakes_peers():
|
||||
"""A request can reach its rank only after resume (e.g. from another API
|
||||
server), with no other work anywhere. That rank must wake its peers itself,
|
||||
or it steps alone. Modelled here by holding the request's ADD on the client
|
||||
side across pause and resume."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
_get_dp_pause_engine_args(expert_parallel=True)
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
client = engine.engine_core
|
||||
params = SamplingParams(max_tokens=5)
|
||||
|
||||
await _consume(engine.generate(DP_PAUSE_PROMPT, params, "warmup"))
|
||||
await engine.pause_generation(mode="abort")
|
||||
assert await _poll_flag(engine, False, timeout=30)
|
||||
|
||||
held: list[tuple] = []
|
||||
send_input = client._send_input
|
||||
|
||||
def hold_add(request_type, request, engine_id=None):
|
||||
if request_type == EngineCoreRequestType.ADD:
|
||||
held.append((request_type, request, engine_id))
|
||||
return asyncio.sleep(0)
|
||||
return send_input(request_type, request, engine_id)
|
||||
|
||||
client._send_input = hold_add
|
||||
task = asyncio.create_task(
|
||||
_consume(engine.generate(DP_PAUSE_PROMPT, params, "late"))
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
assert held
|
||||
|
||||
await engine.resume_generation()
|
||||
client._send_input = send_input
|
||||
await send_input(*held[0])
|
||||
await asyncio.wait_for(task, timeout=60)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
"""The same latch, reached through sleep rather than pause.
|
||||
@@ -441,14 +468,7 @@ async def test_dp_sleep_late_request_does_not_block_drain():
|
||||
prompt=DP_PAUSE_PROMPT,
|
||||
params=SamplingParams(max_tokens=5),
|
||||
)
|
||||
|
||||
assert await _poll_flag(engine, True, timeout=5), (
|
||||
"the request did not notify the coordinator"
|
||||
)
|
||||
|
||||
# Sleeping engines cannot report wave completion, so a coordinator that
|
||||
# marked them running when it forwarded the wake never hears otherwise.
|
||||
assert await _poll_flag(engine, False, timeout=60)
|
||||
assert not await _poll_flag(engine, True, timeout=5)
|
||||
|
||||
await engine.wake_up()
|
||||
assert not await engine.is_sleeping()
|
||||
@@ -628,7 +648,7 @@ async def test_dp_pause_keep_race_staggered_engines():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_barrier_request_deadlock():
|
||||
"""Test that start_dp_wave is ignored while paused.
|
||||
"""Test that a request sent while paused does not wake any engine.
|
||||
|
||||
Sequence:
|
||||
1. Pause all engines (PAUSED_ALL).
|
||||
@@ -697,7 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
|
||||
# Yield so generate() preprocessing completes and
|
||||
# add_request_async is called (which, in buggy code,
|
||||
# would send FIRST_REQ and wake engine 1).
|
||||
# would wake engine 1).
|
||||
for _ in range(200):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
@@ -725,7 +745,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
for t in mid_barrier_tasks:
|
||||
t.cancel()
|
||||
pytest.fail(
|
||||
"Staggered barrier deadlocked — FIRST_REQ sent while "
|
||||
"Staggered barrier deadlocked — a request sent while "
|
||||
"paused caused collective-op mismatch between engines"
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.conftest import HfRunner, VllmRunner
|
||||
from tests.utils import (
|
||||
large_gpu_mark,
|
||||
@@ -168,9 +169,16 @@ def test_with_eagle3_spec_decoding(
|
||||
reason=("XPU matmul/attention kernels are not batch-invariant"),
|
||||
)
|
||||
def test_with_ngram_gpu_spec_decoding(
|
||||
vllm_runner: type[VllmRunner], hf_runner: type[HfRunner]
|
||||
vllm_runner: type[VllmRunner],
|
||||
hf_runner: type[HfRunner],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Check ngram accuracy and acceptance across scheduling configurations."""
|
||||
if current_platform.is_rocm():
|
||||
# hipBLASLt picks GEMM solutions by batch shape, so sync and async runs
|
||||
# can flip near-tie greedy tokens and shift ngram acceptance.
|
||||
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
ngram_gpu_config = {
|
||||
"method": "ngram_gpu",
|
||||
"num_speculative_tokens": 3,
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.cpu_model
|
||||
@@ -112,3 +113,24 @@ def test_prefix_cache_hit_matches_cold_cache(full_prefill_refs):
|
||||
name_0="cold_cache",
|
||||
name_1="warm_cache",
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_reaches_last_full_block():
|
||||
"""A repeated prompt must hit every full block of the hybrid block size.
|
||||
|
||||
The workers raise the block size to fit the GDN state pages, and the
|
||||
engine core reruns its config checks after its handshake; the raised size
|
||||
must survive them, or the prefill's state checkpoint misses the block
|
||||
boundaries and nothing is reused. One chunk holds the whole prompt.
|
||||
"""
|
||||
llm = _make_llm(enable_prefix_caching=True, max_num_batched_tokens=2048)
|
||||
block_size = llm.llm_engine.vllm_config.cache_config.block_size
|
||||
# Ends mid-block: only the last full block's state can be reused.
|
||||
num_tokens = block_size + block_size // 2
|
||||
assert num_tokens <= 2048
|
||||
prompt = TokensPrompt(prompt_token_ids=[1000 + i for i in range(num_tokens)])
|
||||
sp = SamplingParams(max_tokens=1)
|
||||
hits = [llm.generate(prompt, sp)[0].num_cached_tokens for _ in range(2)]
|
||||
del llm
|
||||
|
||||
assert hits == [0, block_size]
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine import EngineCoreRequest, EngineCoreRequestType
|
||||
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
@@ -689,6 +689,54 @@ def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
|
||||
assert not core.pending_pause
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_wave,engines_running,pause_state,announces",
|
||||
[
|
||||
pytest.param(3, False, PauseState.UNPAUSED, True, id="current-wave"),
|
||||
pytest.param(2, False, PauseState.UNPAUSED, True, id="stale-wave"),
|
||||
pytest.param(4, False, PauseState.UNPAUSED, True, id="newer-wave"),
|
||||
pytest.param(3, True, PauseState.UNPAUSED, False, id="running"),
|
||||
pytest.param(3, False, PauseState.PAUSED_ALL, False, id="paused"),
|
||||
],
|
||||
)
|
||||
def test_idle_dp_rank_announces_wave_for_new_work(
|
||||
request_wave, engines_running, pause_state, announces
|
||||
):
|
||||
"""An idle rank handed work announces the wave itself, since only engines
|
||||
start waves; a paused rank waits for resume."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.has_coordinator = True
|
||||
core.current_wave = 3
|
||||
core.engines_running = engines_running
|
||||
core.scheduler = MagicMock(pause_state=pause_state)
|
||||
core.output_queue = MagicMock()
|
||||
|
||||
with patch.object(EngineCore, "add_request"):
|
||||
core.add_request(MagicMock(), request_wave)
|
||||
|
||||
if announces:
|
||||
assert core.engines_running
|
||||
_, outputs = core.output_queue.put_nowait.call_args.args[0]
|
||||
assert outputs.start_wave == max(request_wave, 3)
|
||||
else:
|
||||
core.output_queue.put_nowait.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("paused", [False, True])
|
||||
def test_paused_dp_rank_ignores_start_wave(paused):
|
||||
"""A START_DP_WAVE still in flight when the pause completes must not wake
|
||||
the paused rank."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.ignore_start_dp_wave = paused
|
||||
core.engine_index = 0
|
||||
core.current_wave = 3
|
||||
core.engines_running = False
|
||||
|
||||
core._handle_client_request(EngineCoreRequestType.START_DP_WAVE, (3, 1))
|
||||
|
||||
assert core.engines_running != paused
|
||||
|
||||
|
||||
def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
|
||||
core = object.__new__(EngineCoreProc)
|
||||
|
||||
@@ -314,7 +314,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
client.current_wave = 0
|
||||
client.resources = SimpleNamespace(stats_update_task=None)
|
||||
client.stats_update_address = "inproc://scale-down-stats"
|
||||
client.first_req_sock_addr = "inproc://scale-down-first-request"
|
||||
client.scale_sock_addr = "inproc://scale-down-notify"
|
||||
pause_started = asyncio.Event()
|
||||
|
||||
async def pause_scheduler(*args, **kwargs):
|
||||
@@ -326,11 +326,11 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
with (
|
||||
zmq.asyncio.Context() as ctx,
|
||||
ctx.socket(zmq.XPUB) as coordinator,
|
||||
ctx.socket(zmq.PAIR) as first_req_socket,
|
||||
ctx.socket(zmq.PAIR) as scale_socket,
|
||||
):
|
||||
client.ctx = ctx
|
||||
coordinator.bind(client.stats_update_address)
|
||||
first_req_socket.bind(client.first_req_sock_addr)
|
||||
scale_socket.bind(client.scale_sock_addr)
|
||||
client._ensure_stats_update_task()
|
||||
scale_task = None
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Widen the dummy weight init for the MoRIIO dummy-weight P/D test.
|
||||
|
||||
The dummy loader fills float weights in +-1e-3, which keeps every activation so
|
||||
small that outputs barely depend on the input, so a wrong KV prefix would go
|
||||
unnoticed. The P/D test names ``WideDummyWeights`` as each server's
|
||||
``--worker-extension-cls``. Workers import it before they load the model, which
|
||||
re-registers the ``dummy`` load format in every worker process with a loader
|
||||
that fills 16/32-bit float weights in +-``MORIIO_DUMMY_WEIGHT_RANGE``; 8-bit
|
||||
weights and scales keep the default init. The format keeps the name ``dummy``
|
||||
so every config path that special-cases dummy weights still applies.
|
||||
|
||||
The test process itself never imports this module.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.model_loader import register_model_loader
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
|
||||
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
initialize_single_dummy_weight,
|
||||
)
|
||||
|
||||
_RANGE = float(os.environ["MORIIO_DUMMY_WEIGHT_RANGE"])
|
||||
|
||||
|
||||
@register_model_loader("dummy")
|
||||
class WideDummyModelLoader(DummyModelLoader):
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
# DummyModelLoader.load_weights with a wider init for 16/32-bit floats.
|
||||
for layer in model.modules():
|
||||
info = get_layerwise_info(layer)
|
||||
if info.can_load():
|
||||
self._process_online_quant_layer(layer, info)
|
||||
continue
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name in layer._non_persistent_buffers_set:
|
||||
continue
|
||||
if tensor.is_floating_point() and torch.finfo(tensor.dtype).bits >= 16:
|
||||
initialize_single_dummy_weight(
|
||||
tensor, low=-_RANGE, high=_RANGE, name=name
|
||||
)
|
||||
else:
|
||||
initialize_single_dummy_weight(tensor, name=name)
|
||||
|
||||
|
||||
class WideDummyWeights:
|
||||
"""Worker extension that only exists to import this module in workers."""
|
||||
@@ -0,0 +1,736 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Single-node MoRIIO WRITE P/D checks on dummy weights.
|
||||
|
||||
Each case serves a real model architecture with ``--load-format dummy``, widened
|
||||
by ``moriio_wide_dummy`` (see ``DUMMY_WEIGHT_RANGE``): the
|
||||
config keeps the real hidden size, head geometry, MLA ranks and recurrent-state
|
||||
shapes, so the connector registers and transfers exactly the production cache
|
||||
layout, but the layer stack is cut to a few layers that still contain every
|
||||
cache-group kind and the MoE is cut to a handful of experts. Dummy weights are
|
||||
seeded, so the prefill, decode and reference servers hold identical weights.
|
||||
Cases marked ``HMA_off`` run every server with
|
||||
``--disable-hybrid-kv-cache-manager``, so all layers, sliding-window ones
|
||||
included, share a single cache group.
|
||||
|
||||
Prefill runs on one GPU, decode on another, and MoRI moves the KV over xGMI.
|
||||
Either way decode computes the last prompt token itself. For hybrid models
|
||||
prefill computes only the first ``n - 1`` tokens and transfers the KV and the
|
||||
recurrent state after them. For attention-only models prefill computes and
|
||||
transfers all ``n``, and decode's scheduler, seeing the whole prompt cached,
|
||||
recomputes the last token. A direct serve splits the same prompt the same way
|
||||
when the first ``n - 1`` tokens end on a chunk boundary that both servers share,
|
||||
so the prompts are chosen to make that happen: one is a single chunk plus one
|
||||
token, the others are a whole number of hybrid cache blocks plus one token
|
||||
(prefix caching ends chunks on block boundaries) or, for attention-only models,
|
||||
a whole number of chunks plus one token. With the split matched, a correct
|
||||
transfer reproduces the direct serve's first-token log-probabilities exactly,
|
||||
so the comparison needs no noise allowance. The
|
||||
servers compile with Inductor's deterministic mode, since otherwise each one
|
||||
benchmarks its own reduction launch configs and separately started servers
|
||||
can drift apart by a few bf16 logit steps. Guards:
|
||||
|
||||
* decode must report an external prefix-cache hit for every transferred token
|
||||
of every prompt, so the match is not a local recompute;
|
||||
* changing the first half of a prompt must move the direct serve's log-probs by
|
||||
at least ``MIN_CONTEXT_EFFECT``, so a wrong prefix would be visible;
|
||||
* ``test_corrupted_prefill_state_is_detected`` makes prefill compute its KV with
|
||||
a different norm epsilon (same shapes, different values) and requires every
|
||||
prompt to fail the comparison.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[4]
|
||||
_PROXY = (
|
||||
_ROOT / "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
|
||||
)
|
||||
|
||||
# Max allowed difference between centered first-token log-probabilities of the
|
||||
# P/D and direct outputs, over the direct serve's top-k tokens. The split is
|
||||
# matched, so correct transfers produce identical logits; this only absorbs the
|
||||
# fp32 rounding of the log-softmax (~1e-7), far below one bf16 logit step.
|
||||
TOLERANCE = float(os.getenv("MORIIO_DUMMY_E2E_TOL", "1e-5"))
|
||||
# Several bf16 steps of the logits at DUMMY_WEIGHT_RANGE.
|
||||
MIN_CONTEXT_EFFECT = 0.05
|
||||
# Half-width of the uniform dummy init for 16/32-bit float weights, applied in
|
||||
# every server worker by moriio_wide_dummy.py. The loader default (1e-3) shrinks
|
||||
# every activation so far that a context change moves the first-token log-probs
|
||||
# by ~1e-4; at 0.1 it moves them by >0.1 and a corrupted prefill changes the
|
||||
# top-k set entirely.
|
||||
DUMMY_WEIGHT_RANGE = 0.1
|
||||
_WIDE_DUMMY_DIR = Path(__file__).resolve().parent
|
||||
TOP_K = 20
|
||||
MAX_TOKENS = 4
|
||||
MAX_MODEL_LEN = 4096
|
||||
|
||||
# Prefill token budget of every server. Prompts longer than it are chunked, so
|
||||
# the final-chunk save path runs; the short prompt fills exactly one chunk.
|
||||
PREFILL_CHUNK = 299
|
||||
SHORT_PROMPT_LEN = PREFILL_CHUNK + 1
|
||||
# Hybrid blocks per aligned prompt, before the trailing token.
|
||||
ALIGNED_BLOCKS = (1, 2)
|
||||
# Prefill chunks per long attention-only prompt, before the trailing token.
|
||||
ALIGNED_CHUNKS = (2, 4)
|
||||
|
||||
|
||||
def _mori_available() -> bool:
|
||||
try:
|
||||
import mori.io # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not _mori_available(),
|
||||
reason="MoRIIO P/D needs ROCm and MoRI",
|
||||
),
|
||||
pytest.mark.skipif(
|
||||
current_platform.is_rocm() and current_platform.device_count() < 2,
|
||||
reason="needs one GPU for prefill and one for decode",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model architectures
|
||||
|
||||
|
||||
def _kimi_linear_overrides(cfg) -> dict[str, Any]:
|
||||
# 3 KDA layers then 1 MLA layer; layer 0 stays dense (first_k_dense_replace).
|
||||
linear = dict(cfg.linear_attn_config)
|
||||
linear["kda_layers"] = [1, 2, 3]
|
||||
linear["full_attn_layers"] = [4]
|
||||
groups = max(1, cfg.num_expert_group)
|
||||
return {
|
||||
"num_hidden_layers": 4,
|
||||
"linear_attn_config": linear,
|
||||
"num_experts": 2 * groups,
|
||||
"num_experts_per_token": 2,
|
||||
"topk_group": min(cfg.topk_group, groups),
|
||||
}
|
||||
|
||||
|
||||
def _qwen3_5_moe_overrides(cfg) -> dict[str, Any]:
|
||||
# 3 Gated DeltaNet layers then 1 full-attention layer.
|
||||
return {
|
||||
"text_config": {
|
||||
"num_hidden_layers": 4,
|
||||
"layer_types": ["linear_attention"] * 3 + ["full_attention"],
|
||||
"num_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _qwen3_dense_overrides(cfg) -> dict[str, Any]:
|
||||
# Attention-only: decode gets the KV for the whole prompt.
|
||||
return {"num_hidden_layers": 4}
|
||||
|
||||
|
||||
def _deepseek_v3_overrides(cfg) -> dict[str, Any]:
|
||||
# MLA attention-only: layer 0 dense, layer 1 MoE, no MTP layer.
|
||||
return {
|
||||
"num_hidden_layers": 2,
|
||||
"first_k_dense_replace": 1,
|
||||
"n_routed_experts": 8,
|
||||
"num_experts_per_tok": 2,
|
||||
"n_group": 1,
|
||||
"topk_group": 1,
|
||||
"num_nextn_predict_layers": 0,
|
||||
}
|
||||
|
||||
|
||||
def _gemma3_overrides(cfg) -> dict[str, Any]:
|
||||
# Five sliding-window layers then one global layer, as in the real
|
||||
# 5:1 pattern. Without the hybrid KV cache manager vLLM gives the sliding
|
||||
# layers full-attention cache specs, so all six share one group.
|
||||
return {
|
||||
"num_hidden_layers": 6,
|
||||
"layer_types": ["sliding_attention"] * 5 + ["full_attention"],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchCase:
|
||||
model: str
|
||||
overrides: Callable[[Any], dict[str, Any]]
|
||||
# Norm-epsilon override that perturbs every computed KV/state value
|
||||
# without changing a single shape. Used by the corruption control.
|
||||
corrupt_overrides: dict[str, Any]
|
||||
env: dict[str, str] = field(default_factory=dict)
|
||||
args: tuple[str, ...] = ()
|
||||
# Has recurrent-state (mamba-style) layers, so prefill stops at n - 1.
|
||||
hybrid: bool = True
|
||||
|
||||
|
||||
ARCHS: dict[str, ArchCase] = {
|
||||
"kimi_linear": ArchCase(
|
||||
model="moonshotai/Kimi-Linear-48B-A3B-Instruct",
|
||||
overrides=_kimi_linear_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "1"},
|
||||
args=("--trust-remote-code", "--mamba-ssm-cache-dtype", "float32"),
|
||||
),
|
||||
"qwen3_5_moe": ArchCase(
|
||||
model="Qwen/Qwen3.5-35B-A3B",
|
||||
overrides=_qwen3_5_moe_overrides,
|
||||
corrupt_overrides={"text_config": {"rms_norm_eps": 1.0}},
|
||||
# AITER's GDN conv-update kernel rejects the DS conv-state layout that
|
||||
# MoRIIO's recurrent-state transfer requires.
|
||||
env={"VLLM_ROCM_USE_AITER": "0"},
|
||||
args=("--language-model-only",),
|
||||
),
|
||||
"qwen3_dense": ArchCase(
|
||||
model="Qwen/Qwen3-32B",
|
||||
overrides=_qwen3_dense_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "0"},
|
||||
hybrid=False,
|
||||
),
|
||||
"deepseek_v3": ArchCase(
|
||||
model="deepseek-ai/DeepSeek-V3",
|
||||
overrides=_deepseek_v3_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "1"},
|
||||
hybrid=False,
|
||||
),
|
||||
# Text-only Gemma 3. The multimodal Gemma3ForConditionalGeneration takes
|
||||
# the inputs_embeds path on KV consumers (enable_mm_embeds is forced on
|
||||
# there) and computes differently from a direct serve without any transfer.
|
||||
"gemma3": ArchCase(
|
||||
model="google/gemma-3-1b-it",
|
||||
overrides=_gemma3_overrides,
|
||||
corrupt_overrides={"rms_norm_eps": 1.0},
|
||||
env={"VLLM_ROCM_USE_AITER": "0"},
|
||||
hybrid=False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _arch_overrides(arch: ArchCase) -> dict[str, Any]:
|
||||
from vllm.transformers_utils.config import get_config
|
||||
|
||||
return arch.overrides(get_config(arch.model, trust_remote_code=True))
|
||||
|
||||
|
||||
def _merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]:
|
||||
merged = dict(base)
|
||||
for key, value in extra.items():
|
||||
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key] = _merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serving
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Case:
|
||||
arch: str
|
||||
attention_backend: str
|
||||
prefill_graph: str
|
||||
decode_graph: str
|
||||
# Run every server of the case without the hybrid KV cache manager.
|
||||
hma_off: bool = False
|
||||
|
||||
@property
|
||||
def extra_args(self) -> tuple[str, ...]:
|
||||
return ("--disable-hybrid-kv-cache-manager",) if self.hma_off else ()
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return (
|
||||
f"{self.arch}-{self.attention_backend}-"
|
||||
f"P_{self.prefill_graph}-D_{self.decode_graph}"
|
||||
+ ("-HMA_off" if self.hma_off else "")
|
||||
)
|
||||
|
||||
|
||||
_DEFAULT = "FULL_AND_PIECEWISE"
|
||||
CASES = [
|
||||
# Kimi-Linear: KDA + MLA groups.
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT),
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", "NONE", "NONE"),
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", "PIECEWISE", "PIECEWISE"),
|
||||
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, "FULL_DECODE_ONLY"),
|
||||
Case("kimi_linear", "TRITON_MLA", _DEFAULT, _DEFAULT),
|
||||
# Qwen3.5: Gated DeltaNet + full-attention groups.
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, _DEFAULT),
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", "NONE", "NONE"),
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", "PIECEWISE", "PIECEWISE"),
|
||||
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, "FULL_DECODE_ONLY"),
|
||||
# ROCM_ATTN keeps a (2, num_blocks, ...) KV layout and opts out of KV
|
||||
# connectors, so the second ROCm attention backend is the unified one.
|
||||
Case("qwen3_5_moe", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
|
||||
# Attention-only: prefill transfers the whole prompt.
|
||||
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT),
|
||||
Case("qwen3_dense", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
|
||||
# Without the hybrid KV cache manager: one cache group per model.
|
||||
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
|
||||
Case("deepseek_v3", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT, hma_off=True),
|
||||
Case("gemma3", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
|
||||
]
|
||||
|
||||
|
||||
def _graph_args(mode: str) -> list[str]:
|
||||
if mode == "NONE":
|
||||
return ["--enforce-eager"]
|
||||
# Inductor otherwise picks reduction launch configs by timing them, so each
|
||||
# separately compiled server may sum in a different order and drift by a
|
||||
# few bf16 logit steps; deterministic mode fixes the choice.
|
||||
config = {
|
||||
"cudagraph_mode": mode,
|
||||
"inductor_compile_config": {"deterministic": True},
|
||||
}
|
||||
return ["--compilation-config", json.dumps(config)]
|
||||
|
||||
|
||||
def _visible_devices() -> tuple[str, list[str]]:
|
||||
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
|
||||
selector = next((k for k in selectors if k in os.environ), selectors[0])
|
||||
visible = os.getenv(selector)
|
||||
devices = (
|
||||
visible.split(",")
|
||||
if visible
|
||||
else [str(i) for i in range(current_platform.device_count())]
|
||||
)
|
||||
return selector, devices
|
||||
|
||||
|
||||
def _device_env(selector: str, device: str) -> dict[str, str]:
|
||||
# vLLM mirrors HIP_VISIBLE_DEVICES into CUDA_VISIBLE_DEVICES at import, so
|
||||
# the test process carries both; pin every selector or the server refuses
|
||||
# the mismatch. ROCR narrows the set first, so HIP/CUDA then index into it.
|
||||
env = {selector: device}
|
||||
for key in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
|
||||
if key != selector:
|
||||
env[key] = "0" if selector == "ROCR_VISIBLE_DEVICES" else device
|
||||
return env
|
||||
|
||||
|
||||
# KV consumers force enable_mm_embeds on, which moves multimodal-capable models
|
||||
# (Qwen3.5) from input_ids to inputs_embeds, but the AOT compile-cache key does
|
||||
# not cover it, so a consumer can load a producer's graph and crash in
|
||||
# profile_run. Keep each role's compile cache apart until
|
||||
# vllm-project/vllm#54642 lands (vllm-project/vllm#50891).
|
||||
_CACHE_ROOT = Path(tempfile.mkdtemp(prefix="moriio_dummy_e2e_cache_"))
|
||||
atexit.register(shutil.rmtree, _CACHE_ROOT, ignore_errors=True)
|
||||
|
||||
|
||||
def _server(
|
||||
arch: ArchCase,
|
||||
overrides: dict[str, Any],
|
||||
backend: str,
|
||||
graph: str,
|
||||
device: str,
|
||||
selector: str,
|
||||
*,
|
||||
port: int | None = None,
|
||||
extra_args: tuple[str, ...] = (),
|
||||
kv_transfer_config: dict[str, Any] | None = None,
|
||||
) -> RemoteOpenAIServer:
|
||||
port = port or get_open_port()
|
||||
args = [
|
||||
*arch.args,
|
||||
*extra_args,
|
||||
"--port",
|
||||
str(port),
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--worker-extension-cls",
|
||||
"moriio_wide_dummy.WideDummyWeights",
|
||||
"--hf-overrides",
|
||||
json.dumps(overrides),
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--attention-backend",
|
||||
backend,
|
||||
"--max-model-len",
|
||||
str(MAX_MODEL_LEN),
|
||||
"--max-num-batched-tokens",
|
||||
str(PREFILL_CHUNK),
|
||||
"--max-num-seqs",
|
||||
"16",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(2 * 1024**3),
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--enable-prefix-caching",
|
||||
*_graph_args(graph),
|
||||
]
|
||||
role = "direct"
|
||||
if kv_transfer_config is not None:
|
||||
args += ["--kv-transfer-config", json.dumps(kv_transfer_config)]
|
||||
role = kv_transfer_config["kv_role"]
|
||||
env = {
|
||||
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
|
||||
**_device_env(selector, device),
|
||||
"VLLM_HOST_IP": "127.0.0.1",
|
||||
"VLLM_PORT": str(get_open_port()),
|
||||
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT": "DS",
|
||||
"VLLM_CACHE_ROOT": str(_CACHE_ROOT / role),
|
||||
# Only this directory, so the source tree cannot shadow installed vllm.
|
||||
"PYTHONPATH": os.pathsep.join(
|
||||
filter(None, [str(_WIDE_DUMMY_DIR), os.environ.get("PYTHONPATH")])
|
||||
),
|
||||
"MORIIO_DUMMY_WEIGHT_RANGE": str(DUMMY_WEIGHT_RANGE),
|
||||
**arch.env,
|
||||
}
|
||||
return RemoteOpenAIServer(
|
||||
arch.model, args, env_dict=env, auto_port=False, max_wait_seconds=900
|
||||
)
|
||||
|
||||
|
||||
def _moriio_config(role: str, http_port: int, ping_port: int) -> dict[str, Any]:
|
||||
return {
|
||||
"kv_connector": "MoRIIOConnector",
|
||||
"kv_role": role,
|
||||
"kv_connector_extra_config": {
|
||||
"backend": "xgmi",
|
||||
"read_mode": False,
|
||||
"host_ip": "127.0.0.1",
|
||||
"proxy_ip": "127.0.0.1",
|
||||
"proxy_ping_port": str(ping_port),
|
||||
"http_port": str(http_port),
|
||||
"handshake_port": str(get_open_port()),
|
||||
"notify_port": str(get_open_port()),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _proxy(ping_port: int) -> Iterator[str]:
|
||||
# Free ports for both sockets, so cases from parallel shards on one host
|
||||
# do not collide.
|
||||
port = get_open_port()
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(_PROXY),
|
||||
"--port",
|
||||
str(port),
|
||||
"--service-discovery-port",
|
||||
str(ping_port),
|
||||
],
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
assert process.poll() is None, "MoRIIO proxy exited during startup"
|
||||
try:
|
||||
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
|
||||
break
|
||||
except requests.ConnectionError:
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise TimeoutError("MoRIIO proxy did not start")
|
||||
yield f"http://127.0.0.1:{port}"
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _pd_pair(
|
||||
arch: ArchCase,
|
||||
prefill_overrides: dict[str, Any],
|
||||
decode_overrides: dict[str, Any],
|
||||
case: Case,
|
||||
) -> Iterator[tuple[str, RemoteOpenAIServer]]:
|
||||
selector, devices = _visible_devices()
|
||||
assert len(devices) >= 2, "needs one GPU for prefill and one for decode"
|
||||
prefill_port, decode_port, ping_port = (get_open_port() for _ in range(3))
|
||||
with _proxy(ping_port) as proxy_url, contextlib.ExitStack() as stack:
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
with ThreadPoolExecutor(2) as pool:
|
||||
futures = [
|
||||
pool.submit(
|
||||
_server,
|
||||
arch,
|
||||
prefill_overrides,
|
||||
case.attention_backend,
|
||||
case.prefill_graph,
|
||||
devices[0],
|
||||
selector,
|
||||
port=prefill_port,
|
||||
extra_args=case.extra_args,
|
||||
kv_transfer_config=_moriio_config(
|
||||
"kv_producer", prefill_port, ping_port
|
||||
),
|
||||
),
|
||||
pool.submit(
|
||||
_server,
|
||||
arch,
|
||||
decode_overrides,
|
||||
case.attention_backend,
|
||||
case.decode_graph,
|
||||
devices[1],
|
||||
selector,
|
||||
port=decode_port,
|
||||
extra_args=case.extra_args,
|
||||
kv_transfer_config=_moriio_config(
|
||||
"kv_consumer", decode_port, ping_port
|
||||
),
|
||||
),
|
||||
]
|
||||
for future in futures:
|
||||
servers.append(future.result())
|
||||
_wait_for_registration(proxy_url, arch.model)
|
||||
yield proxy_url, servers[1]
|
||||
|
||||
|
||||
def _wait_for_registration(proxy_url: str, model: str) -> None:
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
# The proxy hands decode max_tokens - 1 (prefill emits the first
|
||||
# token), so the probe needs at least 2 for decode to accept it.
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{proxy_url}/v1/completions",
|
||||
json={"model": model, "prompt": [1000, 1001], "max_tokens": 2},
|
||||
timeout=300,
|
||||
)
|
||||
# The proxy streams the decode body, so a decode-side error can
|
||||
# surface as a truncated response rather than a status code.
|
||||
if response.status_code == 200 and response.json()["choices"]:
|
||||
return
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise TimeoutError("prefill/decode never registered with the MoRIIO proxy")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Measurement
|
||||
|
||||
|
||||
def _metric(server: RemoteOpenAIServer, name: str) -> list[Any]:
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
response = requests.get(server.url_for("metrics"), timeout=10)
|
||||
response.raise_for_status()
|
||||
return [
|
||||
sample
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
for sample in family.samples
|
||||
if sample.name == name
|
||||
]
|
||||
|
||||
|
||||
def _block_size(server: RemoteOpenAIServer) -> int:
|
||||
# Hybrid models pad the attention block to the recurrent-state page, so the
|
||||
# effective size is only known once the server has started.
|
||||
sizes = {
|
||||
int(sample.labels["block_size"])
|
||||
for sample in _metric(server, "vllm:cache_config_info")
|
||||
}
|
||||
assert len(sizes) == 1, f"expected one cache block size, got {sizes}"
|
||||
return sizes.pop()
|
||||
|
||||
|
||||
def _prompt_lengths(arch: ArchCase, server: RemoteOpenAIServer) -> list[int]:
|
||||
# Hybrid chunks end on cache-block boundaries; attention-only chunks end on
|
||||
# the prefill budget, so the last token is computed on its own either way.
|
||||
if arch.hybrid:
|
||||
unit, counts = _block_size(server), ALIGNED_BLOCKS
|
||||
else:
|
||||
unit, counts = PREFILL_CHUNK, ALIGNED_CHUNKS
|
||||
aligned = [
|
||||
k * unit + 1 for k in counts if k * unit + 1 + MAX_TOKENS <= MAX_MODEL_LEN
|
||||
]
|
||||
assert aligned, f"alignment unit {unit} leaves no aligned prompt"
|
||||
return [SHORT_PROMPT_LEN, *aligned]
|
||||
|
||||
|
||||
def _prompts(lengths: list[int]) -> list[list[int]]:
|
||||
rng = random.Random(0)
|
||||
return [[rng.randrange(1000, 30000) for _ in range(n)] for n in lengths]
|
||||
|
||||
|
||||
def _first_step(url: str, model: str, prompt: list[int]) -> dict[str, float]:
|
||||
response = requests.post(
|
||||
f"{url}/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"logprobs": TOP_K,
|
||||
"return_tokens_as_token_ids": True,
|
||||
},
|
||||
timeout=600,
|
||||
)
|
||||
response.raise_for_status()
|
||||
top = response.json()["choices"][0]["logprobs"]["top_logprobs"]
|
||||
assert top, "server returned no logprobs"
|
||||
return top[0]
|
||||
|
||||
|
||||
def _distance(ref: dict[str, float], got: dict[str, float]) -> float:
|
||||
"""Max difference of centered log-probs over the shared top-k tokens.
|
||||
|
||||
Centering removes the log-sum-exp offset, leaving the logit differences the
|
||||
KV actually determines.
|
||||
"""
|
||||
common = sorted(ref.keys() & got.keys())
|
||||
if len(common) < len(ref) // 2:
|
||||
return float("inf")
|
||||
ref_mean = sum(ref[t] for t in common) / len(common)
|
||||
got_mean = sum(got[t] for t in common) / len(common)
|
||||
return max(abs((ref[t] - ref_mean) - (got[t] - got_mean)) for t in common)
|
||||
|
||||
|
||||
def _external_hits(server: RemoteOpenAIServer) -> float:
|
||||
samples = _metric(server, "vllm:external_prefix_cache_hits_total")
|
||||
return sum(sample.value for sample in samples)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reference:
|
||||
prompts: list[list[int]]
|
||||
logprobs: list[dict[str, float]]
|
||||
hybrid: bool
|
||||
|
||||
@property
|
||||
def expected_external_hits(self) -> int:
|
||||
# Hybrid prefill stops one token short; attention-only prefill hands
|
||||
# decode the whole prompt.
|
||||
return sum(len(p) - int(self.hybrid) for p in self.prompts)
|
||||
|
||||
|
||||
def _reference(arch: ArchCase, overrides: dict[str, Any], case: Case) -> Reference:
|
||||
"""Direct serve; also checks that the comparison can see a wrong prefix."""
|
||||
selector, devices = _visible_devices()
|
||||
# Prefill graphs run the chunks and the decode-only last step runs the same
|
||||
# captured graph in every decode mode used here, so the prefill mode
|
||||
# reproduces both halves of the P/D computation.
|
||||
with _server(
|
||||
arch,
|
||||
overrides,
|
||||
case.attention_backend,
|
||||
case.prefill_graph,
|
||||
devices[0],
|
||||
selector,
|
||||
extra_args=case.extra_args,
|
||||
) as server:
|
||||
url = server.url_root
|
||||
lengths = _prompt_lengths(arch, server)
|
||||
prompts = _prompts(lengths)
|
||||
refs = [_first_step(url, arch.model, p) for p in prompts]
|
||||
# Same tail, different early context: the comparison has to see it.
|
||||
for prompt, ref in zip(prompts, refs):
|
||||
altered = [t + 1 for t in prompt[: len(prompt) // 2]]
|
||||
altered += prompt[len(prompt) // 2 :]
|
||||
moved = _distance(ref, _first_step(url, arch.model, altered))
|
||||
print(f"[sensitivity] len={len(prompt)} context-change distance={moved}")
|
||||
assert moved >= MIN_CONTEXT_EFFECT, (
|
||||
f"first-token log-probs barely depend on the context (moved "
|
||||
f"{moved:.4g} for len {len(prompt)}); a wrong prefix would go "
|
||||
"unnoticed"
|
||||
)
|
||||
# Determinism of the reference itself (served from its prefix cache).
|
||||
again = _first_step(url, arch.model, prompts[-1])
|
||||
print(f"[noise] repeat distance={_distance(refs[-1], again)}")
|
||||
return Reference(prompts, refs, arch.hybrid)
|
||||
|
||||
|
||||
def _pd_distances(
|
||||
arch: ArchCase,
|
||||
prefill_overrides: dict[str, Any],
|
||||
decode_overrides: dict[str, Any],
|
||||
case: Case,
|
||||
ref: Reference,
|
||||
) -> tuple[list[float], float]:
|
||||
with _pd_pair(arch, prefill_overrides, decode_overrides, case) as (
|
||||
proxy_url,
|
||||
decode,
|
||||
):
|
||||
hits_before = _external_hits(decode)
|
||||
distances = [
|
||||
_distance(expected, _first_step(proxy_url, arch.model, prompt))
|
||||
for prompt, expected in zip(ref.prompts, ref.logprobs)
|
||||
]
|
||||
hits = _external_hits(decode) - hits_before
|
||||
lengths = [len(p) for p in ref.prompts]
|
||||
print(f"[pd] {case.id} lens={lengths} distances={distances} external_hits={hits}")
|
||||
return distances, hits
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id)
|
||||
def test_write_matches_direct_serve(case: Case):
|
||||
arch = ARCHS[case.arch]
|
||||
overrides = _arch_overrides(arch)
|
||||
ref = _reference(arch, overrides, case)
|
||||
distances, hits = _pd_distances(arch, overrides, overrides, case, ref)
|
||||
|
||||
# Every transferred token of every prompt must come from prefill.
|
||||
assert hits == ref.expected_external_hits, (
|
||||
f"decode reported {hits} external prefix-cache hits, expected "
|
||||
f"{ref.expected_external_hits}; it did not use all the transferred KV"
|
||||
)
|
||||
lengths = [len(p) for p in ref.prompts]
|
||||
bad = {n: d for n, d in zip(lengths, distances) if not d <= TOLERANCE}
|
||||
assert not bad, f"P/D first-token log-probs differ from direct serve: {bad}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arch_name", sorted(ARCHS))
|
||||
def test_corrupted_prefill_state_is_detected(arch_name: str):
|
||||
"""Negative control: KV computed with different values must fail."""
|
||||
arch = ARCHS[arch_name]
|
||||
default = next(c for c in CASES if c.arch == arch_name)
|
||||
case = Case(
|
||||
arch_name, default.attention_backend, _DEFAULT, _DEFAULT, default.hma_off
|
||||
)
|
||||
overrides = _arch_overrides(arch)
|
||||
ref = _reference(arch, overrides, case)
|
||||
corrupted = _merge(overrides, arch.corrupt_overrides)
|
||||
distances, hits = _pd_distances(arch, corrupted, overrides, case, ref)
|
||||
|
||||
assert hits == ref.expected_external_hits, (
|
||||
"decode did not use the transferred KV; the control proves nothing"
|
||||
)
|
||||
assert all(d > TOLERANCE for d in distances), (
|
||||
f"P/D outputs with wrongly computed prefill KV still matched the "
|
||||
f"reference (distances {distances}); the comparison cannot detect "
|
||||
"transfer corruption"
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("endpoint", ["/v1/completions", "/v1/chat/completions"])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_response_media_type_matches_payload(monkeypatch, endpoint, stream):
|
||||
"""Clients must select SSE parsing for streaming completion responses."""
|
||||
path = (
|
||||
Path(__file__).parents[4]
|
||||
/ "examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py"
|
||||
)
|
||||
proxy = runpy.run_path(str(path))["_handle_completions"].__globals__
|
||||
app = proxy["app"]
|
||||
app.state.ready = asyncio.Event()
|
||||
app.state.ready.set()
|
||||
app.state.prefill_iterator = itertools.cycle([({}, 0)])
|
||||
app.state.decode_iterator = itertools.cycle([0])
|
||||
app.state.decode_clients = [{}]
|
||||
payload = b'{"choices":[{"text":"hello"}]}'
|
||||
body = b"data: " + payload + b"\n\ndata: [DONE]\n\n" if stream else payload
|
||||
|
||||
async def decode_response(*args, **kwargs):
|
||||
yield body
|
||||
|
||||
monkeypatch.setitem(proxy, "send_request_to_service", AsyncMock())
|
||||
monkeypatch.setitem(proxy, "stream_service_response", decode_response)
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app), base_url="http://proxy"
|
||||
) as client:
|
||||
response = await client.post(endpoint, json={"stream": stream})
|
||||
|
||||
expected_type = "text/event-stream" if stream else "application/json"
|
||||
assert response.headers["content-type"].split(";")[0] == expected_type
|
||||
assert response.content == body
|
||||
@@ -146,6 +146,7 @@ def _write_consumer_scheduler_for_finished_request(tp_size: int = 2):
|
||||
scheduler.is_producer = False
|
||||
scheduler.mode = MoRIIOMode.WRITE
|
||||
scheduler.tp_size = tp_size
|
||||
scheduler._has_mamba = False
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.unmap_request_id = MagicMock()
|
||||
return scheduler
|
||||
@@ -158,6 +159,10 @@ def _write_producer_scheduler(block_size: int = 1) -> Any:
|
||||
scheduler.mode = MoRIIOMode.WRITE
|
||||
scheduler.is_producer = True
|
||||
scheduler.block_size = block_size
|
||||
scheduler._has_mamba = False
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=object())]
|
||||
)
|
||||
scheduler.blocks_per_sw = [0]
|
||||
scheduler._group_block_sizes = [block_size]
|
||||
scheduler.transfer_id_to_request_id = {}
|
||||
@@ -944,6 +949,7 @@ def test_clamp_to_prompt_blocks_clamps_each_group_by_its_own_block_size():
|
||||
# their own prompt-block count; a sliding-window group is left untouched.
|
||||
scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler)
|
||||
scheduler.blocks_per_sw = [0, 0, 3]
|
||||
scheduler._has_mamba = False
|
||||
scheduler._group_block_sizes = [16, 32, 16]
|
||||
|
||||
req = SimpleNamespace(num_prompt_tokens=40)
|
||||
|
||||
@@ -12,12 +12,13 @@ wiring.
|
||||
import importlib
|
||||
from dataclasses import asdict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs
|
||||
|
||||
@@ -52,6 +53,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
|
||||
attributes read by the method under test are set."""
|
||||
|
||||
def __init__(self, **attrs):
|
||||
self.mode = MoRIIOMode.READ
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._attn_group_ids: list[int] = [0]
|
||||
self._num_ssm_scratch_blocks = 0
|
||||
@@ -287,7 +289,8 @@ def test_scheduler_rejects_mamba_group_without_two_states():
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
|
||||
@pytest.mark.parametrize("read_mode", [True, False])
|
||||
def test_scheduler_rejects_non_dspark_speculative_hybrid(read_mode):
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
@@ -298,7 +301,9 @@ def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(speculative_config=_spec_config("ngram")),
|
||||
_gate_vllm_config(
|
||||
read_mode=read_mode, speculative_config=_spec_config("ngram")
|
||||
),
|
||||
"engine",
|
||||
config,
|
||||
)
|
||||
@@ -336,14 +341,16 @@ def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True):
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize("read_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)]
|
||||
)
|
||||
def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
dcp_size, dcp_sharded, expected_tail
|
||||
dcp_size, dcp_sharded, expected_tail, read_mode
|
||||
):
|
||||
config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded)
|
||||
vllm_config = _gate_vllm_config(
|
||||
read_mode=read_mode,
|
||||
speculative_config=_spec_config("dspark"),
|
||||
block_size=4,
|
||||
num_lookahead_tokens=7,
|
||||
@@ -384,19 +391,234 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_hybrid_write():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
|
||||
],
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_clamps_attention_and_excludes_scratch_slots():
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
assert sched._clamp_to_prompt_blocks(
|
||||
SimpleNamespace(num_prompt_tokens=9), [[10, 11, 12, 13], [1, 2, 3, 4]]
|
||||
) == [[11], [1, 2]]
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False), "engine", config
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_resolves_current_state_instead_of_cached_block_ids():
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
state = KVConnectorBlockState(
|
||||
req_ids={"req"},
|
||||
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
|
||||
boundary_state_offloads={},
|
||||
)
|
||||
assert sched._get_write_blocks(
|
||||
SimpleNamespace(request_id="req", num_prompt_tokens=9),
|
||||
[[10, 11, 12], [1]],
|
||||
SimpleNamespace(kv_connector_block_state=state),
|
||||
) == [[51], [31, 32]]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize("external_tokens", [0, 9])
|
||||
def test_hybrid_write_notifies_grouped_slots_or_releases_a_complete_local_hit(
|
||||
external_tokens,
|
||||
):
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False), "engine", config
|
||||
)
|
||||
sched.send_notify_block = Mock()
|
||||
sched._send_transfer_release = Mock()
|
||||
req = _mk_request(
|
||||
list(range(10)),
|
||||
params={
|
||||
"do_remote_prefill": True,
|
||||
"transfer_id": "tx",
|
||||
"remote_host": "127.0.0.2",
|
||||
"remote_notify_port": 7000,
|
||||
},
|
||||
)
|
||||
req.request_id = "req"
|
||||
sched.update_state_after_alloc(
|
||||
req, _FakeBlocks(([10, 11, 12], [1, 2, 3])), external_tokens
|
||||
)
|
||||
if external_tokens:
|
||||
sched.send_notify_block.assert_called_once_with(
|
||||
req_id="req",
|
||||
transfer_id="tx",
|
||||
block_notify_list=[[10], [1, 2, 3]],
|
||||
host="127.0.0.2",
|
||||
port=7000,
|
||||
)
|
||||
sched._send_transfer_release.assert_not_called()
|
||||
else:
|
||||
sched.send_notify_block.assert_not_called()
|
||||
sched._send_transfer_release.assert_called_once_with("tx", "127.0.0.2", 7000)
|
||||
assert not req.kv_transfer_params["do_remote_prefill"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_keeps_separate_draft_groups_and_their_block_spans():
|
||||
config = _dspark_wrapped_attention_config()
|
||||
# A disabled cache is absent from the wire. Target and draft may have
|
||||
# different token spans, and neither can borrow the other's block IDs.
|
||||
draft = SimpleNamespace(
|
||||
enable_kv_transfer=True,
|
||||
kv_cache_spec=FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
)
|
||||
disabled = SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object())
|
||||
config.kv_cache_groups.insert(1, disabled)
|
||||
config.kv_cache_groups.append(draft)
|
||||
config.transfer_groups = tuple(
|
||||
g for g in config.kv_cache_groups if g.enable_kv_transfer
|
||||
)
|
||||
config.select_transfer_block_ids = lambda blocks: (blocks[0], blocks[2], blocks[3])
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(
|
||||
read_mode=False, dcp_size=2, speculative_config=_spec_config("dspark")
|
||||
),
|
||||
"engine",
|
||||
config,
|
||||
)
|
||||
req = _mk_request(
|
||||
list(range(9)),
|
||||
params={
|
||||
"do_remote_decode": True,
|
||||
"transfer_id": "tx",
|
||||
},
|
||||
)
|
||||
req.request_id = "req"
|
||||
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
blocks = ([10, 11, 12, 13], [99], [20, 21, 22], [30, 31, 32, 33])
|
||||
expected = [[11], [20, 21], [30, 31, 32]]
|
||||
assert sched._clamp_to_prompt_blocks(req, blocks) == expected
|
||||
# Exercise the connector entry point too: completion must not flatten
|
||||
# a multi-group WRITE payload through READ's single-attention helper.
|
||||
delay_free, params = _FakeConnector(sched).request_finished_all_groups(req, blocks)
|
||||
assert delay_free
|
||||
assert params["remote_block_ids"] == expected
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize(
|
||||
"blocks,delay_free",
|
||||
[(([0, 51, 52, 53], [31, 32, 33]), True), (([], []), False)],
|
||||
)
|
||||
def test_hybrid_write_request_finished_defers_free_only_with_blocks_to_send(
|
||||
blocks, delay_free
|
||||
):
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
sched.is_producer = True
|
||||
req = _mk_request(
|
||||
list(range(9)), params={"do_remote_decode": True, "transfer_id": "tx"}
|
||||
)
|
||||
req.request_id = "req"
|
||||
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
assert sched.request_finished(req, blocks)[0] is delay_free
|
||||
# The WRITE ACK, not the READ watchdog, releases the deferred blocks.
|
||||
if delay_free:
|
||||
assert sched._deferred_send_deadlines["req"][1] == "tx"
|
||||
else:
|
||||
assert "req" not in sched._deferred_send_deadlines
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
def test_hybrid_write_build_meta_saves_current_state_on_the_final_chunk():
|
||||
"""Chunked prefill must not ship block ids cached at admission: align-mode
|
||||
state slots move between chunks without appending a block."""
|
||||
config = _dspark_wrapped_attention_config()
|
||||
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
|
||||
sched = moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
|
||||
)
|
||||
sched.is_producer = True
|
||||
set_role(moriio_common.ROLE.PRODUCER)
|
||||
params = {
|
||||
"do_remote_decode": True,
|
||||
"transfer_id": "tx",
|
||||
"remote_engine_id": "decode",
|
||||
"remote_block_ids": [],
|
||||
"remote_host": "127.0.0.1",
|
||||
"remote_handshake_port": 6000,
|
||||
"remote_notify_port": 6001,
|
||||
}
|
||||
req = _mk_request(list(range(9)), params=params)
|
||||
req.request_id = "req"
|
||||
req.num_computed_tokens = 0
|
||||
sched._reqs_need_save["req"] = (req, [[10, 11, 12], [1]])
|
||||
sched._req_kv_params["req"] = dict(params)
|
||||
|
||||
def _output(cached_req_ids, new_block_ids, num_scheduled):
|
||||
return SimpleNamespace(
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=cached_req_ids, new_block_ids=new_block_ids
|
||||
),
|
||||
num_scheduled_tokens={"req": num_scheduled},
|
||||
new_block_ids_to_zero=None,
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
req_ids={"req"},
|
||||
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
|
||||
boundary_state_offloads={},
|
||||
),
|
||||
)
|
||||
|
||||
# First chunk: not final, so nothing is saved yet.
|
||||
meta = sched.build_connector_meta(_output([], [], 4))
|
||||
assert not meta.reqs_to_save
|
||||
assert "req" in sched._reqs_need_pending_save
|
||||
|
||||
# Final chunk: stale cached ids plus new ids are ignored in favour of
|
||||
# the scheduler's current table, clamped to the prompt.
|
||||
req.num_computed_tokens = 4
|
||||
meta = sched.build_connector_meta(_output(["req"], [([77], [78])], 5))
|
||||
assert meta.reqs_to_save["req"].local_block_ids == [[51], [31, 32]]
|
||||
assert "req" not in sched._reqs_need_pending_save
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,deadlines,expected",
|
||||
[
|
||||
(MoRIIOMode.WRITE, {"req": (0.0, "tx")}, True),
|
||||
(MoRIIOMode.WRITE, {}, False),
|
||||
(MoRIIOMode.READ, {"req": (0.0, "tx")}, False),
|
||||
],
|
||||
)
|
||||
def test_has_pending_push_work_tracks_write_deferred_frees(mode, deadlines, expected):
|
||||
connector = _FakeConnector(_FakeScheduler(_deferred_send_deadlines=deadlines))
|
||||
connector.mode = mode
|
||||
assert connector.has_pending_push_work() is expected
|
||||
|
||||
|
||||
def test_has_pending_push_work_is_false_on_worker_role():
|
||||
connector = _FakeConnector(None)
|
||||
connector.mode = MoRIIOMode.WRITE
|
||||
assert connector.has_pending_push_work() is False
|
||||
|
||||
|
||||
def test_loaded_kv_cache_groups_are_every_transfer_group():
|
||||
# The connector restores the request's own blocks, not a hash-addressed
|
||||
# prefix, so a transfer-enabled Mamba group counts as loaded too.
|
||||
connector = _FakeConnector(None)
|
||||
connector._kv_cache_config = SimpleNamespace(
|
||||
transfer_group_ids=(0, 2), prefix_cacheable_group_ids=(0,)
|
||||
)
|
||||
assert connector.get_loaded_kv_cache_group_ids(SimpleNamespace()) == (0, 2)
|
||||
|
||||
|
||||
def test_split_block_groups_rejects_non_hybrid_use():
|
||||
@@ -728,11 +950,15 @@ def test_session_build_rejects_per_layer_region_count_mismatch():
|
||||
worker._get_built_session("prefill")
|
||||
|
||||
|
||||
def test_register_kv_caches_rejects_hybrid_write_before_registration():
|
||||
worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"})
|
||||
def test_register_kv_caches_rejects_invalid_hybrid_write_state_layout():
|
||||
worker = _FakeWorker(
|
||||
mode=MoRIIOMode.WRITE,
|
||||
_transfer_layer_names={"kda.0"},
|
||||
layer_to_spec={"kda.0": _mamba_spec(1)},
|
||||
)
|
||||
worker._is_mamba_layer = lambda _layer_name: True
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"):
|
||||
worker.register_kv_caches({"kda.0": object()})
|
||||
|
||||
|
||||
@@ -933,6 +1159,40 @@ def test_read_matches_only_requests_with_pending_remote_prefill(params, expected
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"has_mamba,expected", [(False, (32, True)), (True, (31, True))]
|
||||
)
|
||||
def test_write_consumer_matches_full_prompt_unless_hybrid(has_mamba, expected):
|
||||
"""Without a mamba group (attention-only, or HMA disabled) prefill pushed
|
||||
KV for all N tokens; with one it pushed h(N-1)."""
|
||||
scheduler = _FakeScheduler(
|
||||
is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=has_mamba
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
num_prompt_tokens=32, kv_transfer_params={"do_remote_prefill": True}
|
||||
)
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"is_producer,has_mamba,expected_prompt_len",
|
||||
[(True, True, 4), (True, False, 5), (False, True, 5)],
|
||||
)
|
||||
def test_on_new_request_truncates_only_hybrid_producers(
|
||||
is_producer, has_mamba, expected_prompt_len
|
||||
):
|
||||
"""Attention-only prefill (including models run with HMA disabled)
|
||||
computes all N prompt tokens; only a hybrid producer drops the last."""
|
||||
scheduler = _FakeScheduler(is_producer=is_producer, _has_mamba=has_mamba)
|
||||
request = _mk_request([10, 11, 12, 13, 14], params={"do_remote_decode": True})
|
||||
|
||||
scheduler.on_new_request(request)
|
||||
|
||||
assert request.num_prompt_tokens == expected_prompt_len
|
||||
assert len(request.prompt_token_ids) == expected_prompt_len
|
||||
|
||||
|
||||
def test_read_rematch_after_alloc_keeps_queued_transfer():
|
||||
"""Clearing the flag blocks another external hit, not the queued READ."""
|
||||
sched = _make_read_scheduler()
|
||||
@@ -981,12 +1241,37 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
|
||||
assert is_async is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [None, {}, {"do_remote_prefill": False}])
|
||||
def test_write_does_not_wait_again_after_receive_or_for_local_requests(params):
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
|
||||
req = _mk_request(list(range(10)))
|
||||
req.kv_transfer_params = params
|
||||
assert sched.get_num_new_matched_tokens(req, 0) == (0, False)
|
||||
|
||||
|
||||
def test_write_never_declares_an_empty_async_load():
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False)
|
||||
req = _mk_request(list(range(10)), params={"do_remote_prefill": True})
|
||||
assert sched.get_num_new_matched_tokens(req, 10) == (0, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_len,computed,expected", [(16, 0, 15), (17, 16, 0), (1, 0, 0)]
|
||||
)
|
||||
def test_hybrid_write_advertises_only_the_state_computed_by_prefill(
|
||||
prompt_len, computed, expected
|
||||
):
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
|
||||
req = _mk_request(list(range(prompt_len)), params={"do_remote_prefill": True})
|
||||
assert sched.get_num_new_matched_tokens(req, computed) == (expected, expected > 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "num_computed_tokens", "expected", "is_async"),
|
||||
[
|
||||
(MoRIIOMode.READ, 0, 9, False),
|
||||
(MoRIIOMode.READ, 10, 0, False),
|
||||
(MoRIIOMode.WRITE, 2, 8, True),
|
||||
(MoRIIOMode.WRITE, 2, 7, True),
|
||||
],
|
||||
)
|
||||
def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
|
||||
|
||||
@@ -7,6 +7,7 @@ import threading
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -106,7 +107,7 @@ def _write_task(layer_name: str, transfer_id: str = "xfer") -> Any:
|
||||
request_id="req",
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[1, 3],
|
||||
local_block_ids=[[1, 3]],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name=layer_name,
|
||||
event=None,
|
||||
@@ -345,6 +346,8 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeWorker:
|
||||
_has_mamba: bool
|
||||
layer_to_group: dict[str, int]
|
||||
kv_caches: dict[str, torch.Tensor]
|
||||
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
|
||||
|
||||
@@ -364,11 +367,13 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
return ([call_id], [call_id + 10], [call_id + 20])
|
||||
|
||||
fake_worker = FakeWorker()
|
||||
fake_worker._has_mamba = False
|
||||
fake_worker.layer_to_group = dict.fromkeys(kv_caches, 0)
|
||||
fake_worker.kv_caches = kv_caches
|
||||
fake_worker.layer_name_to_local_kv_cache_metadata = {name: [] for name in kv_caches}
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
remote_meta = _remote_meta()
|
||||
|
||||
dense0_plan = writer._prepare_transfer_plan(
|
||||
@@ -395,7 +400,7 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
|
||||
|
||||
def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
@@ -411,6 +416,80 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
|
||||
assert writer._sealed_writes["xfer"] == 2
|
||||
|
||||
|
||||
def test_hybrid_write_attention_uses_its_own_group_not_the_first_group():
|
||||
worker = SimpleNamespace(
|
||||
_has_mamba=True,
|
||||
kv_caches={"draft": torch.empty((8, 1, 4, 3))},
|
||||
layer_to_group={"draft": 2},
|
||||
_region_session_indices=lambda _: [5],
|
||||
_compute_block_transfer_offsets=Mock(return_value=([1], [2], [3])),
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("draft")
|
||||
task.local_block_ids = [[10], [20], [30, 31]]
|
||||
remote = RemoteAllocInfo(block_ids=[[40], [50], [60, 61, 62]])
|
||||
meta = _remote_meta()
|
||||
plan = writer._prepare_transfer_plan(task, remote, meta)
|
||||
worker._compute_block_transfer_offsets.assert_called_once_with(
|
||||
"draft", [30, 31], [60, 61], meta, remote_engine_id=task.dst_engine_id
|
||||
)
|
||||
assert plan.sess_idx == 5
|
||||
|
||||
|
||||
def test_hybrid_write_posts_both_regions_before_layer_completion():
|
||||
from unittest.mock import Mock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
|
||||
wrapper = _wrapper_for_messages()
|
||||
info = RemoteAllocInfo(block_ids=[[], [7], [9]])
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = info
|
||||
wrapper.write_remote_data = Mock(side_effect=["conv-status", "ssm-status"])
|
||||
worker = SimpleNamespace(
|
||||
moriio_wrapper=wrapper,
|
||||
tp_rank=0,
|
||||
get_engine_name_with_dp=lambda name, rank: name,
|
||||
_get_built_session=lambda name: (["conv-session", "ssm-session"], None),
|
||||
_is_mamba_layer=lambda name: True,
|
||||
_mamba_payload_index_by_layer={"kda": 2},
|
||||
_region_session_indices=lambda name: [0, 1],
|
||||
_compute_mamba_transfer_offsets=Mock(
|
||||
return_value=([10, 20, 30, 40], [50, 60, 70, 80], [2, 4, 8, 16], 3)
|
||||
),
|
||||
)
|
||||
worker._mamba_blocks_for_layer = lambda name, blocks: (
|
||||
MoRIIOConnectorWorker._mamba_blocks_for_layer(worker, name, blocks)
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("kda")
|
||||
task.local_block_ids = [[], [1], [3]]
|
||||
task.remote_tp_size = 2
|
||||
task.event = Mock()
|
||||
writer._execute_write_task(task)
|
||||
|
||||
task.event.synchronize.assert_called_once()
|
||||
worker._compute_mamba_transfer_offsets.assert_called_once_with(
|
||||
"kda", [3], [9], remote_tp_size=2
|
||||
)
|
||||
assert wrapper.write_remote_data.call_args_list[0].args == (
|
||||
[2, 4, 8],
|
||||
[10, 20, 30],
|
||||
[50, 60, 70],
|
||||
"conv-session",
|
||||
)
|
||||
assert wrapper.write_remote_data.call_args_list[1].args == (
|
||||
[16],
|
||||
[40],
|
||||
[80],
|
||||
"ssm-session",
|
||||
)
|
||||
assert info.transfer_statuses == ["conv-status", "ssm-status"]
|
||||
assert info.writes_done == 1
|
||||
assert not info.completion_notified
|
||||
|
||||
|
||||
def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
@@ -445,7 +524,7 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
)
|
||||
|
||||
wrapper = FakeWrapper()
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
|
||||
request_info = RemoteAllocInfo(block_ids=[[4, 5]], writes_expected=2)
|
||||
request_info.transfer_statuses.extend(["status-a", "status-b"])
|
||||
request_info.completion_request_id = "req"
|
||||
# Final notify port is resolved in _execute_write_task; finalize reads it
|
||||
@@ -487,6 +566,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
wrapper._transfer_timeout = 1
|
||||
wrapper.moriio_engine = object()
|
||||
global_status = FakeStatus()
|
||||
scoped_status = FakeStatus()
|
||||
wrapper.transfer_status = [global_status]
|
||||
@@ -500,7 +580,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
|
||||
|
||||
def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 2
|
||||
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
|
||||
@@ -518,7 +598,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
|
||||
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 1
|
||||
writer._scheduled_layers["xfer"] = {"dense0"}
|
||||
@@ -610,7 +690,7 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
|
||||
|
||||
if expected == "remote_blocks":
|
||||
request_info = wrapper.done_remote_allocate_req_dict["xfer"]
|
||||
assert request_info.block_ids == [4, 5]
|
||||
assert request_info.block_ids == [[4, 5]]
|
||||
assert request_info.decode_dp_rank == 3
|
||||
elif expected == "write_done":
|
||||
assert wrapper.done_write_cache_req_ids == ["xfer"]
|
||||
|
||||
@@ -11,6 +11,8 @@ import importlib
|
||||
import importlib.util
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -127,3 +129,62 @@ def test_wait_blocks_until_terminal():
|
||||
assert time.monotonic() - start >= 0.15
|
||||
finally:
|
||||
timer.cancel()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("timeout,expected_ms", [(0.05, 50), (0.0001, 1)])
|
||||
def test_native_wait_receives_all_statuses_and_a_positive_deadline(
|
||||
timeout, expected_ms
|
||||
):
|
||||
statuses = [_pending(), _ok()]
|
||||
engine = SimpleNamespace(
|
||||
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
|
||||
)
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=timeout)
|
||||
wrapper.waiting_for_transfer_complete(statuses)
|
||||
engine.wait_all.assert_called_once_with(statuses, timeout_ms=expected_ms)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"result,statuses,message",
|
||||
[
|
||||
("ERR_RDMA_OP", [_pending(), _bad()], "SQ full"),
|
||||
("IN_PROGRESS", [_pending(), _ok()], "timed out"),
|
||||
("ERR_INVALID_ARGS", [_pending()], "ERR_INVALID_ARGS"),
|
||||
],
|
||||
)
|
||||
def test_native_wait_reports_failure_or_timeout(result, statuses, message):
|
||||
engine = SimpleNamespace(
|
||||
wait_all=Mock(return_value=getattr(moriio_engine.StatusCode, result))
|
||||
)
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
with pytest.raises(TransferError, match=message):
|
||||
wrapper.waiting_for_transfer_complete(statuses)
|
||||
|
||||
|
||||
def test_wait_falls_back_without_native_status_enum(monkeypatch):
|
||||
monkeypatch.setattr(moriio_engine, "StatusCode", None)
|
||||
engine = SimpleNamespace(wait_all=Mock())
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
with pytest.raises(TransferError, match="SQ full"):
|
||||
wrapper.waiting_for_transfer_complete([_bad(), _ok()])
|
||||
engine.wait_all.assert_not_called()
|
||||
|
||||
|
||||
def test_nonblocking_poll_does_not_call_native_wait():
|
||||
engine = SimpleNamespace(wait_all=Mock())
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
assert wrapper.poll_transfer_batch([_pending()]) is TransferBatchState.PENDING
|
||||
wrapper.waiting_for_transfer_complete([])
|
||||
engine.wait_all.assert_not_called()
|
||||
|
||||
|
||||
def test_native_wait_drains_only_the_unscoped_status_snapshot():
|
||||
statuses = [_pending(), _ok()]
|
||||
engine = SimpleNamespace(
|
||||
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
|
||||
)
|
||||
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
|
||||
wrapper.transfer_status.extend(statuses)
|
||||
wrapper.waiting_for_transfer_complete()
|
||||
engine.wait_all.assert_called_once_with(statuses, timeout_ms=50)
|
||||
assert wrapper.transfer_status == []
|
||||
|
||||
@@ -14,6 +14,8 @@ import pytest
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ROLE,
|
||||
LayerTransferPlan,
|
||||
MoRIIOError,
|
||||
MoRIIOMode,
|
||||
RemoteAllocInfo,
|
||||
ReqMeta,
|
||||
@@ -96,7 +98,7 @@ def _write_task(
|
||||
request_id=request_id,
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[1, 3],
|
||||
local_block_ids=[[1, 3]],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name="layer0",
|
||||
event=None,
|
||||
@@ -249,6 +251,7 @@ def test_finalize_single_pod_port_uses_global_rank():
|
||||
|
||||
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = False
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
@@ -266,13 +269,14 @@ def test_write_blocks_for_req_forwards_routing_to_schedule_write():
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
|
||||
assert captured["remote_dp_size_local"] == 8
|
||||
assert captured["remote_ip"] == "10.0.0.1"
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
assert captured["local_block_ids"] == [[1]]
|
||||
assert captured["remote_block_ids"] == [[2]]
|
||||
assert not hasattr(worker, "multi_pod_hosts")
|
||||
|
||||
|
||||
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = False
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
@@ -290,8 +294,8 @@ def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
|
||||
assert captured["remote_dp_size_local"] == 4
|
||||
assert captured["local_block_ids"] == [1]
|
||||
assert captured["remote_block_ids"] == [2]
|
||||
assert captured["local_block_ids"] == [[1]]
|
||||
assert captured["remote_block_ids"] == [[2]]
|
||||
|
||||
|
||||
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
|
||||
@@ -373,3 +377,73 @@ def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
|
||||
assert results.failed_recving == {"req-1"}
|
||||
assert wrapper.pop_finished_write_req_ids() == set()
|
||||
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
|
||||
|
||||
|
||||
def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer():
|
||||
from unittest.mock import Mock
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = True
|
||||
worker.world_size = 2
|
||||
worker.schedule_write_blocks = Mock()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
|
||||
worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None)
|
||||
|
||||
worker.schedule_write_blocks.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"notify,expected",
|
||||
[
|
||||
# Plain WRITE peers still send one flat attention list.
|
||||
([7, 8], [[7, 8]]),
|
||||
# Hybrid WRITE sends one list per transfer group, kept as-is even
|
||||
# when the first group is empty (full attention hit, state only).
|
||||
([[7], [9]], [[7], [9]]),
|
||||
([[], [9]], [[], [9]]),
|
||||
],
|
||||
)
|
||||
def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected):
|
||||
wrapper = _real_wrapper()
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
|
||||
wrapper._handle_remote_blocks_message(
|
||||
{"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2}
|
||||
)
|
||||
info = wrapper.done_remote_allocate_req_dict["tA"]
|
||||
assert info.block_ids == expected
|
||||
assert info.decode_dp_rank == 2
|
||||
|
||||
|
||||
def test_empty_layer_plan_posts_no_rdma_write():
|
||||
from unittest.mock import Mock
|
||||
|
||||
wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock())
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
plan = LayerTransferPlan(
|
||||
request_id="req",
|
||||
transfer_id="tA",
|
||||
layer_name="layer0",
|
||||
sess_idx=0,
|
||||
transfer_local_offsets=[],
|
||||
transfer_remote_offsets=[],
|
||||
transfer_sizes=[],
|
||||
)
|
||||
|
||||
assert writer._do_layer_write(plan, ["session"]) == []
|
||||
wrapper.write_remote_data.assert_not_called()
|
||||
wrapper.write_remote_data_single.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_state", [[], [1, 2]])
|
||||
def test_hybrid_write_requires_exactly_one_running_state_slot(local_state):
|
||||
worker = SimpleNamespace(
|
||||
_mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1],
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("tA", "10.0.0.1")
|
||||
task.layer_name = "kda"
|
||||
task.local_block_ids = [[], local_state]
|
||||
|
||||
with pytest.raises(MoRIIOError, match="one running-state slot"):
|
||||
writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]]))
|
||||
|
||||
@@ -805,6 +805,23 @@ class TestMultiConnectorStats:
|
||||
|
||||
child_metrics.observe.assert_called_once_with({"transfer_duration": [1.5]}, 2)
|
||||
|
||||
@pytest.mark.parametrize("with_exporter", [False, True])
|
||||
def test_prom_metrics_skips_stats_only_connectors(self, with_exporter):
|
||||
"""Stats-only children must not prevent supported metrics from exporting."""
|
||||
child_metrics = MagicMock()
|
||||
metrics = object.__new__(MultiKVConnectorPromMetrics)
|
||||
metrics._prom_metrics = (
|
||||
{"MooncakeStoreConnector": child_metrics} if with_exporter else {}
|
||||
)
|
||||
payload = {"MooncakeConnector": {"bytes_transferred": [1024]}}
|
||||
if with_exporter:
|
||||
payload["MooncakeStoreConnector"] = {"load_get_bytes": [2048]}
|
||||
|
||||
metrics.observe(payload, engine_idx=2)
|
||||
|
||||
if with_exporter:
|
||||
child_metrics.observe.assert_called_once_with({"load_get_bytes": [2048]}, 2)
|
||||
|
||||
def test_aggregate_same_connector(self):
|
||||
"""Test aggregating stats from the same connector type."""
|
||||
stats1 = MultiKVConnectorStats(
|
||||
|
||||
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p=None)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topp_only(self, batch_size: int, vocab_size: int):
|
||||
"""Test top-p only (k=None)."""
|
||||
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k=None, p=p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
|
||||
"""Test combined top-k and top-p."""
|
||||
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48, 64])
|
||||
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
|
||||
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
|
||||
"""Mixed batches at vocab sizes where the split top-p pipeline uses
|
||||
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
|
||||
k[1::4] = vocab_size
|
||||
p[2::4] = 1.0
|
||||
k[3::4] = vocab_size
|
||||
p[3::4] = 1.0
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
def test_both_disabled(self):
|
||||
"""Test when both k and p are None (should be no-op)."""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
|
||||
f"{kept_topk_topp.tolist()}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48])
|
||||
def test_skip_split_without_topp_only_rows(
|
||||
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""has_topp_only_rows=False skips the split pipeline; the result must
|
||||
not change when every top-p row also has a top-k, grammar rows
|
||||
(k >= #finite logits) included."""
|
||||
import vllm.v1.sample.ops.topk_topp_triton as tt
|
||||
|
||||
vocab_size = 32000
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
logits[1] = float("-inf")
|
||||
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
|
||||
logits[1, idx] = torch.randn(30, generator=self.generator)
|
||||
k = torch.randint(
|
||||
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
|
||||
)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
k[::4] = vocab_size
|
||||
p[::4] = 1.0
|
||||
|
||||
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
|
||||
monkeypatch.setattr(
|
||||
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
|
||||
)
|
||||
actual = tt.apply_top_k_top_p_triton(
|
||||
logits.clone(), k, p, has_topp_only_rows=False
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
def test_topp_tie_break_deterministic(self):
|
||||
"""Boundary-duplicate trimming must be deterministic across calls.
|
||||
|
||||
@@ -821,6 +871,36 @@ class TestTritonTopkTopp:
|
||||
)
|
||||
assert len(kept_idx) == pytorch_kept
|
||||
|
||||
def test_topp_ties_across_blocks(self):
|
||||
"""The split top-p tie budget must carry across a slice's blocks.
|
||||
|
||||
Regression: the split top-p mask kernel restarted its count of
|
||||
boundary-value copies at every 8192-entry block, so each block of a
|
||||
slice could keep up to `numkeep` copies of its own.
|
||||
"""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
|
||||
batch_size, vocab_size, num_ties = 64, 151936, 4096
|
||||
# 4096 tied tokens spread over many blocks, between 64 tokens above
|
||||
# and a low tail; the p=0.75 boundary lands mid-copy in the tie group.
|
||||
tie_idx = torch.arange(num_ties) * 37
|
||||
logits = torch.full((batch_size, vocab_size), -8.0)
|
||||
logits[:, tie_idx] = 0.0
|
||||
logits[:, -64:] = 4.0
|
||||
p = torch.full((batch_size,), 0.75, dtype=torch.float32)
|
||||
|
||||
kept = apply_top_k_top_p_triton(logits.clone(), None, p) != float("-inf")
|
||||
expected = apply_top_k_top_p_pytorch(logits.clone(), None, p) != float("-inf")
|
||||
num_kept, num_expected = kept.sum(dim=-1), expected.sum(dim=-1)
|
||||
assert torch.equal(num_kept, num_expected), (
|
||||
f"kept {num_kept.unique().tolist()}, "
|
||||
f"expected {num_expected.unique().tolist()}"
|
||||
)
|
||||
# Kept copies of the boundary value are the first ones in index order.
|
||||
kept_ties = kept[:, tie_idx]
|
||||
num_kept_ties = kept_ties.sum(dim=-1, keepdim=True)
|
||||
assert torch.equal(kept_ties, torch.arange(num_ties) < num_kept_ties)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# FlashInfer top-k/top-p robustness tests
|
||||
|
||||
@@ -245,6 +245,55 @@ def test_connector_reports_configured_cache_source(
|
||||
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
@pytest.mark.parametrize("backend", ["cpu", "disk"])
|
||||
def test_connector_explicit_cpu_blocks_align_uneven_stages(
|
||||
tmp_path, num_layers: int, backend: str
|
||||
):
|
||||
"""The same configured pool size must fit lighter and heavier PP stages."""
|
||||
config = _make_kv_cache_config(num_blocks=16)
|
||||
names = [f"layer_{i}" for i in range(num_layers)]
|
||||
config.kv_cache_groups[0].layer_names = names
|
||||
config.kv_cache_tensors[0].layers = names
|
||||
config.kv_cache_tensors[0].size *= num_layers
|
||||
vllm_config = _make_vllm_config()
|
||||
extra = {
|
||||
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8,
|
||||
"num_cpu_blocks": 4,
|
||||
"kv_offload_backend": backend,
|
||||
}
|
||||
if backend == "disk":
|
||||
extra.update(
|
||||
disk_path=str(tmp_path / "offload.bin"),
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = extra
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, config
|
||||
)
|
||||
assert connector.scheduler_manager is not None
|
||||
assert connector.scheduler_manager.num_cpu_blocks == 4
|
||||
assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count, capacity_bytes",
|
||||
[(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]]
|
||||
+ [(1, _BYTES_PER_BLOCK // 2)],
|
||||
)
|
||||
def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes):
|
||||
config = _make_kv_cache_config(num_blocks=16)
|
||||
vllm_config = _make_vllm_config()
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = {
|
||||
"cpu_bytes_to_use_per_rank": capacity_bytes,
|
||||
"num_cpu_blocks": count,
|
||||
}
|
||||
with pytest.raises(ValueError, match="num_cpu_blocks"):
|
||||
SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config)
|
||||
|
||||
|
||||
def make_scheduler(
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
|
||||
@@ -371,6 +371,69 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_layers", [1, 2])
|
||||
def test_explicit_cpu_blocks_bound_live_worker_allocation(monkeypatch, num_layers):
|
||||
"""Different live KV strides must honor the same explicit CPU pool size."""
|
||||
num_blocks = 4
|
||||
names = [f"layer.{i}" for i in range(num_layers)]
|
||||
raw = torch.zeros(
|
||||
num_layers * num_blocks * BLOCK_BYTES, dtype=torch.int8, device="cuda"
|
||||
)
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=names,
|
||||
layer_stride=num_blocks * BLOCK_BYTES,
|
||||
block_stride=BLOCK_BYTES,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
caches = dict(zip(names, raw.view(num_layers, num_blocks, BLOCK_BYTES)))
|
||||
worker = SimpleCPUOffloadWorker(None, config, 8 * BLOCK_BYTES, num_cpu_blocks=4)
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
try:
|
||||
worker.register_kv_caches(caches)
|
||||
assert worker.num_cpu_blocks == 4
|
||||
assert worker.cpu_kv_caches is not None
|
||||
assert all(t.shape[0] == 4 for t in worker.cpu_kv_caches.values())
|
||||
assert sum(t.nbytes for t in worker.cpu_kv_caches.values()) <= 8 * BLOCK_BYTES
|
||||
finally:
|
||||
if worker._backend is not None:
|
||||
worker._backend.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"capacity_bytes, count, local_capacity",
|
||||
[(8 * BLOCK_BYTES, 5, 4), (BLOCK_BYTES, 1, 0)],
|
||||
)
|
||||
def test_explicit_cpu_blocks_reject_heavier_worker_before_allocation(
|
||||
capacity_bytes, count, local_capacity
|
||||
):
|
||||
"""A count that fits the scheduler's stage must fail on a smaller worker."""
|
||||
names = ["layer.0", "layer.1"]
|
||||
raw = torch.zeros(2 * 4 * BLOCK_BYTES, dtype=torch.int8, device="cuda")
|
||||
config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=raw.nbytes,
|
||||
layers=names,
|
||||
layer_stride=4 * BLOCK_BYTES,
|
||||
block_stride=BLOCK_BYTES,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[],
|
||||
)
|
||||
worker = SimpleCPUOffloadWorker(None, config, capacity_bytes, num_cpu_blocks=count)
|
||||
caches = dict(zip(names, raw.view(2, 4, BLOCK_BYTES)))
|
||||
with pytest.raises(ValueError, match=f"local offload capacity {local_capacity}"):
|
||||
worker.register_kv_caches(caches)
|
||||
assert worker.cpu_kv_caches is None
|
||||
|
||||
|
||||
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
|
||||
num_blocks = 4
|
||||
block_bytes = 32
|
||||
|
||||
@@ -1157,6 +1157,91 @@ def test_chunked_requests_match_full_batch(has_draft_logits: bool):
|
||||
assert torch.equal(chunked_sampled[valid], sampled[valid])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_block_verification", [False, True])
|
||||
@pytest.mark.parametrize("has_draft_logits", [False, True])
|
||||
def test_adaptive_draft_lengths_match_and_bonus_oracle(
|
||||
use_block_verification: bool, has_draft_logits: bool
|
||||
):
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
K = 3
|
||||
num_drafts = [3, 1, 0, 2, 3, 2]
|
||||
temperatures = [1.0, 0.0, 0.0, 0.6, 0.0, 1.0]
|
||||
greedy_num_matches = {1: 1, 2: 0, 4: 2}
|
||||
num_reqs = len(num_drafts)
|
||||
|
||||
num_logits_per_req = torch.tensor(num_drafts, device=device) + 1
|
||||
cu_num_logits = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
cu_num_logits[1:] = num_logits_per_req.cumsum(0)
|
||||
num_logits = int(cu_num_logits[-1])
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
expanded_idx_mapping = idx_mapping.repeat_interleave(num_logits_per_req)
|
||||
expanded_local_pos = (
|
||||
torch.arange(num_logits, dtype=torch.int32, device=device)
|
||||
- cu_num_logits[expanded_idx_mapping]
|
||||
)
|
||||
|
||||
draft_logits = torch.randn(num_reqs, K, VOCAB_SIZE, device=device)
|
||||
target_logits = torch.randn(num_logits, VOCAB_SIZE, device=device)
|
||||
# Allocate at exactly num_logits so a lookahead past the batch is OOB.
|
||||
draft_sampled = torch.randint(VOCAB_SIZE, (num_logits,), device=device)
|
||||
target_argmax = target_logits.argmax(dim=-1)
|
||||
for req_idx, num_matches in greedy_num_matches.items():
|
||||
lo = int(cu_num_logits[req_idx])
|
||||
for step in range(num_drafts[req_idx]):
|
||||
token = target_argmax[lo + step]
|
||||
if step >= num_matches:
|
||||
token = (token + 1) % VOCAB_SIZE
|
||||
draft_sampled[lo + step + 1] = token
|
||||
|
||||
inputs = dict(
|
||||
target_logits=target_logits,
|
||||
draft_logits=draft_logits if has_draft_logits else None,
|
||||
draft_sampled=draft_sampled,
|
||||
cu_num_logits=cu_num_logits,
|
||||
pos=torch.arange(num_logits, dtype=torch.int32, device=device),
|
||||
idx_mapping=idx_mapping,
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
expanded_local_pos=expanded_local_pos,
|
||||
temperature=torch.tensor(temperatures, device=device),
|
||||
seed=torch.arange(num_reqs, dtype=torch.int64, device=device),
|
||||
num_speculative_steps=K,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
sampled, num_sampled = rejection_sample(**inputs)
|
||||
|
||||
for req_idx, num_matches in greedy_num_matches.items():
|
||||
lo = int(cu_num_logits[req_idx])
|
||||
n = num_matches + 1
|
||||
assert int(num_sampled[req_idx]) == n, f"greedy request {req_idx}"
|
||||
assert torch.equal(sampled[req_idx, :n], target_argmax[lo : lo + n]), (
|
||||
f"greedy request {req_idx}"
|
||||
)
|
||||
|
||||
for req_idx in range(num_reqs):
|
||||
lo, hi = int(cu_num_logits[req_idx]), int(cu_num_logits[req_idx + 1])
|
||||
req_inputs = dict(inputs)
|
||||
for name in (
|
||||
"target_logits",
|
||||
"draft_sampled",
|
||||
"pos",
|
||||
"expanded_idx_mapping",
|
||||
"expanded_local_pos",
|
||||
):
|
||||
req_inputs[name] = inputs[name][lo:hi]
|
||||
req_inputs["cu_num_logits"] = cu_num_logits[req_idx : req_idx + 2] - lo
|
||||
req_inputs["idx_mapping"] = idx_mapping[req_idx : req_idx + 1]
|
||||
|
||||
req_sampled, req_num_sampled = rejection_sample(**req_inputs)
|
||||
|
||||
n = int(req_num_sampled[0])
|
||||
assert n <= num_drafts[req_idx] + 1
|
||||
assert int(num_sampled[req_idx]) == n, f"request {req_idx} mismatch"
|
||||
assert torch.equal(sampled[req_idx, :n], req_sampled[0, :n]), (
|
||||
f"request {req_idx} mismatch"
|
||||
)
|
||||
|
||||
|
||||
def _residual_recovery_logits(
|
||||
inputs: dict,
|
||||
num_sampled: torch.Tensor,
|
||||
|
||||
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
|
||||
assert sampler.needs_logits_processing[3] == admitted
|
||||
|
||||
|
||||
def test_has_topp_only_rows():
|
||||
"""A row is top-p-only when it has top-p and no effective top-k; top_k
|
||||
of 0 (the default) or at least the logits width disables top-k."""
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
|
||||
sampler.add_request(1, SamplingParams(top_k=10))
|
||||
sampler.add_request(2, SamplingParams(top_p=0.9))
|
||||
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
|
||||
states = sampler.sampling_states
|
||||
states.apply_staged_writes()
|
||||
|
||||
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
|
||||
idx_np = np.array(idx, dtype=np.int32)
|
||||
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
|
||||
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
|
||||
|
||||
assert not has_topp_only_rows([0, 1])
|
||||
assert has_topp_only_rows([0, 2])
|
||||
assert has_topp_only_rows([3])
|
||||
# The kernel compares top_k against the logits width, so a top_k at or
|
||||
# above a narrower logits width counts as no top-k.
|
||||
assert has_topp_only_rows([0], vocab_size=10)
|
||||
|
||||
|
||||
def test_logits_processing_cache_only_checks_active_requests():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, SamplingParams(temperature=0.0))
|
||||
|
||||
@@ -368,6 +368,7 @@ def test_gpu_sampler_respects_mixed_request_watermarking(
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 7]))
|
||||
@@ -383,7 +384,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
|
||||
# reference path; the assertion is about the call order, not the kernel.
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.gpu_sampler.apply_top_k_top_p",
|
||||
lambda logits, k, p: apply_top_k_top_p_pytorch(
|
||||
lambda logits, k, p, **kwargs: apply_top_k_top_p_pytorch(
|
||||
logits, k, p, allow_cpu_sync=True
|
||||
),
|
||||
)
|
||||
@@ -415,6 +416,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
|
||||
torch.tensor([2]),
|
||||
torch.tensor([0.8]),
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert watermarker.captured_logits is not None
|
||||
@@ -482,6 +484,7 @@ def test_gpu_sampler_skips_watermarking_for_repeated_contexts(monkeypatch):
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 7]))
|
||||
@@ -517,6 +520,7 @@ def test_gpu_sampler_can_disable_context_deduplication(monkeypatch):
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([7, 7]))
|
||||
@@ -956,6 +960,7 @@ def test_gpu_sampler_uses_fused_gumbel_for_repeated_contexts():
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(repeated, torch.tensor([True, False], device=device))
|
||||
@@ -992,6 +997,7 @@ def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert actual is expected
|
||||
|
||||
Reference in New Issue
Block a user