Merge remote-tracking branch 'upstream/main' into patches

This commit is contained in:
2026-10-05 08:52:31 +07:00
85 changed files with 3146 additions and 769 deletions
@@ -423,3 +423,58 @@ def test_cumem_with_cudagraph():
# output content is as expected
assert torch.allclose(y, x + 1)
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
def test_cudagraph_pool_sleep(level):
"""Routing, and the graph pool backed up at both sleep levels and restored
in place by any wake."""
from contextlib import nullcontext
from types import SimpleNamespace
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
from vllm.compilation.cudagraph_pool import (
capture_outside_cumem_pool,
capture_pool,
)
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
handle = current_platform.graph_pool_handle()
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
with ctx, capture_pool(handle, cfg) as used:
assert used == handle and allocator.current_tag != "cudagraph"
with allocator.use_memory_pool("weights"):
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
x = torch.ones_like(weight)
def capture() -> list:
graph = torch.cuda.CUDAGraph()
stream = torch.cuda.Stream()
with (
capture_pool(handle, on) as pool,
torch.cuda.graph(graph, pool=pool, stream=stream),
):
assert nccl_alloc._graph_pool_id == pool != handle
const = torch.empty_like(x) # Never written by replay: needs a backup.
y = x * weight + const
return [graph, y, const.fill_(3.0)]
def graph_pool() -> dict[int, int]:
data = allocator.pointer_to_data.items()
return {
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
}
held, backend = capture(), CuMemBackend()
mapped = graph_pool()
backend.suspend(level=level)
backend.resume(tags=["kv_cache"])
assert graph_pool() == mapped
backend.resume(tags=["weights"])
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
held[0].replay()
assert torch.equal(held[1], torch.full_like(x, 5.0))
@@ -3,6 +3,7 @@
import asyncio
import math
import os
import pytest
import torch
@@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
from ....utils import (
create_new_process_for_each_test,
multi_gpu_marks,
multi_gpu_test,
requires_fp8,
)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch):
assert output[0].outputs[0].text == output2[0].outputs[0].text
def _cudagraph_bytes(worker) -> int:
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
def _custom_ar_active(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
ca_comm = get_tp_group().device_communicator.ca_comm
return ca_comm is not None and not ca_comm.disabled
@pytest.mark.parametrize(
("mode", "breakable", "tp", "offload"),
[
("FULL", False, 1, True),
("PIECEWISE", False, 1, True),
("PIECEWISE", True, 1, True),
pytest.param(
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
),
("FULL_AND_PIECEWISE", False, 1, False),
],
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
)
@create_new_process_for_each_test()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
unregistered custom AR; off changes nothing."""
for name, value in [
("VLLM_USE_V2_MODEL_RUNNER", "1"),
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
]:
monkeypatch.setenv(name, value)
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
llm = LLM(
"hmellor/tiny-random-LlamaForCausalLM",
enable_sleep_mode=True,
sleep_mode_offload_cudagraph=offload,
tensor_parallel_size=tp,
compilation_config={
"cudagraph_mode": mode,
"pass_config": {"fuse_allreduce_rms": False},
},
)
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
assert all(graph_bytes) if offload else not any(graph_bytes)
if not offload:
return
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("Custom allreduce is unavailable on these GPUs")
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
llm.sleep(level=level)
llm.wake_up(tags=["weights"])
if level == 2:
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
assert llm.generate(prompt, params)[0].outputs[0].text == expected
@create_new_process_for_each_test()
def test_deep_sleep_async(gpu_memory_cleared):
async def test():
+28 -2
View File
@@ -376,7 +376,6 @@ def test_flashinfer_standalone_workspace_size(
"_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", False)),
)
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 2)
monkeypatch.setattr(
flashinfer_all_reduce,
"_get_tuned_standalone_max_size",
@@ -389,6 +388,34 @@ def test_flashinfer_standalone_workspace_size(
assert create_workspace.call_args.args[3] == expected
@pytest.mark.parametrize(
("node_count", "is_blackwell", "expected"),
[
(1, False, ("trtllm", False)),
(1, True, ("trtllm", False)),
(2, False, (None, False)),
(2, True, ("mnnvl", False)),
],
)
def test_flashinfer_auto_backend_uses_group_topology(
monkeypatch: pytest.MonkeyPatch,
node_count: int,
is_blackwell: bool,
expected: tuple[str | None, bool],
) -> None:
monkeypatch.setattr(
flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto"
)
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count)
monkeypatch.setattr(
flashinfer_all_reduce.current_platform,
"has_device_capability",
lambda capability: is_blackwell,
)
assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected
def test_flashinfer_workspace_failure_is_not_retried(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -402,7 +429,6 @@ def test_flashinfer_workspace_failure_is_not_retried(
"_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", True)),
)
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
monkeypatch.setattr(
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
)
+3 -1
View File
@@ -177,7 +177,9 @@ def _worker(rank: int, tp_size: int, port: int) -> None:
# Supply the resolved Engram settings without constructing a full model.
vllm_config.engram_config = EngramConfig()
vllm_config.model_config = SimpleNamespace(
architecture="DeepseekV41ForCausalLM", is_moe=True
architecture="DeepseekV41ForCausalLM",
is_moe=True,
sleep_mode_offload_cudagraph=False,
)
try:
init_distributed_environment()
@@ -248,7 +248,8 @@ async def test_renderer_only_chat_request_skips_mm_cache():
@pytest.mark.asyncio
async def test_chat_error_stream():
@pytest.mark.parametrize("include_usage", [False, True])
async def test_chat_error_stream(include_usage: bool):
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
@@ -313,6 +314,67 @@ async def test_chat_error_stream():
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=True,
stream_options={"include_usage": include_usage},
)
response = await serving_chat.create_chat_completion(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
assert len(chunks) >= 2
assert any("Internal server error" in chunk for chunk in chunks), (
f"Expected error message in chunks: {chunks}"
)
assert chunks[-1] == "data: [DONE]\n\n"
@pytest.mark.asyncio
@pytest.mark.parametrize("include_usage", [False, True])
async def test_chat_error_stream_first_chunk(include_usage: bool):
"""An error before the first token must reach the streaming client."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
completion_output = CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=None,
logprobs=None,
finish_reason="error",
)
request_output = RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
)
async def mock_generate(*args, **kwargs):
yield request_output
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=True,
stream_options={"include_usage": include_usage},
)
response = await serving_chat.create_chat_completion(request)
@@ -383,10 +383,14 @@ async def test_completion_suffix_supports_text_prompt_list():
assert result == [{"ok": True}]
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
(
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}"
),
(
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}"
),
]
@@ -619,7 +623,8 @@ async def test_renderer_only_completion_request_skips_mm_cache():
@pytest.mark.asyncio
async def test_completion_error_stream():
@pytest.mark.parametrize("include_usage", [False, True])
async def test_completion_error_stream(include_usage: bool):
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
@@ -684,6 +689,67 @@ async def test_completion_error_stream():
prompt="Test prompt",
max_tokens=10,
stream=True,
stream_options={"include_usage": include_usage},
)
response = await serving_completion.create_completion(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
assert len(chunks) >= 2
assert any("Internal server error" in chunk for chunk in chunks), (
f"Expected error message in chunks: {chunks}"
)
assert chunks[-1] == "data: [DONE]\n\n"
@pytest.mark.asyncio
@pytest.mark.parametrize("include_usage", [False, True])
async def test_completion_error_stream_first_chunk(include_usage: bool):
"""An error before the first token must reach the streaming client."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
completion_output = CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=None,
logprobs=None,
finish_reason="error",
)
request_output = RequestOutput(
request_id="test-id",
prompt="Test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
)
async def mock_generate(*args, **kwargs):
yield request_output
mock_engine.generate = MagicMock(side_effect=mock_generate)
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
max_tokens=10,
stream=True,
stream_options={"include_usage": include_usage},
)
response = await serving_completion.create_completion(request)
@@ -67,7 +67,7 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.inputs import tokens_input
from vllm.logprobs import Logprob as SampleLogprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser.harmony import Segment
from vllm.parser.harmony import ChunkResult, HarmonyParser, Segment
from vllm.renderers import TokenizeParams
from vllm.renderers.online_renderer import (
OnlineRenderer,
@@ -2129,6 +2129,87 @@ async def test_stream_completed_response_reuses_streamed_items(monkeypatch):
assert function_call.call_id == "chatcmpl-tool-parser-id"
def _harmony_msg(channel: str, text: str, recipient: str | None = None):
msg = OpenAIHarmonyMessage.from_role_and_content(Role.ASSISTANT, text)
msg = msg.with_channel(channel)
return msg.with_recipient(recipient) if recipient else msg
def _harmony_segments(msg, *, streamed: bool = True) -> list[Segment]:
"""Segments the parser yields for one message: a content delta (absent
for a zero-delta message), then the completed message."""
delta = [Segment(msg.channel, msg.recipient, msg.content[0].text)]
completed = Segment(msg.channel, msg.recipient, "", completed_message=msg)
return (delta if streamed else []) + [completed]
async def _harmony_stream_events(segment_chunks: list[list[Segment]]):
serving = _make_serving_instance()
serving.use_harmony = True
parser = MagicMock(spec=HarmonyParser)
parser.process_chunk.side_effect = [
ChunkResult(segments=segments, reasoning_token_count=0)
for segments in segment_chunks
]
context = HarmonyContext([], [], frozenset({"get_weather"}), parser)
async def result_generator():
for token_id in range(len(segment_chunks)):
context.append_output(_make_request_output("", [token_id]))
yield context
return [
event
async for event in serving.responses_stream_generator(
request=ResponsesRequest(input="hi", stream=True, store=False),
sampling_params=SamplingParams(max_tokens=16),
result_generator=result_generator(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
@pytest.mark.asyncio
async def test_harmony_stream_completed_response_reuses_streamed_ids():
"""response.completed must carry the id and call_id each item was streamed
with, not ones minted when the harmony messages are converted again."""
messages = [
_harmony_msg("analysis", "think"),
_harmony_msg("commentary", '{"city": "Paris"}', "functions.get_weather"),
_harmony_msg("final", "Sunny"),
]
events = await _harmony_stream_events([_harmony_segments(msg) for msg in messages])
streamed = [e.item for e in events if e.type == "response.output_item.done"]
output = events[-1].response.output
assert [item.type for item in output] == ["reasoning", "function_call", "message"]
assert [item.id for item in output] == [item.id for item in streamed]
assert output[1].call_id == streamed[1].call_id
assert output[2].content[0].text == "Sunny"
@pytest.mark.asyncio
async def test_harmony_stream_unstreamed_item_keeps_own_id():
"""An item with no done event (zero-delta) must not take the streamed id
of a later item of the same type."""
unstreamed = _harmony_msg("analysis", "skipped")
reasoning = _harmony_msg("analysis", "think")
events = await _harmony_stream_events(
[_harmony_segments(unstreamed, streamed=False), _harmony_segments(reasoning)]
)
(streamed,) = [e.item for e in events if e.type == "response.output_item.done"]
first, second = events[-1].response.output
assert first.content[0].text == "skipped"
assert first.id != streamed.id
assert second.id == streamed.id
@pytest.mark.asyncio
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
async def test_streaming_parallel_tool_calls(
@@ -456,11 +456,13 @@ async def test_stream_returns_spec_decode_metrics_on_empty_terminal_output(
data_chunks = [chunk for chunk in _parse_sse_chunks(chunks) if chunk != "[DONE]"]
assert "metrics" not in data_chunks[0]
assert len(data_chunks) == 1 + int(include_usage)
assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
assert "metrics" not in data_chunks[1]
assert len(data_chunks) == 2 + int(include_usage)
if include_usage:
assert data_chunks[1]["choices"] == []
assert data_chunks[2]["choices"] == []
assert (
data_chunks[1]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
data_chunks[2]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
)
@@ -823,30 +825,76 @@ async def test_stream_zero_token_completion_still_delivers_prompt_metadata():
@pytest.mark.asyncio
async def test_stream_zero_token_completion_emits_no_chunk_by_default():
@pytest.mark.parametrize("finish_reason", ["stop", "length", "abort"])
async def test_stream_emits_terminal_output_without_new_tokens(finish_reason):
"""The final output may carry no new tokens (always for an abort); the
client must still see why the choice finished."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1", token_ids=[], finish_reason="stop", finished=True
"req-1", token_ids=[], finish_reason=finish_reason, finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
choices = [c["choices"][0] for c in parsed[:-1]]
assert not [c for c in parsed if isinstance(c, dict) and c.get("choices")]
assert [c["token_ids"] for c in choices] == [[10], []]
assert [c["finish_reason"] for c in choices] == [None, finish_reason]
assert parsed[-1] == "[DONE]"
@pytest.mark.asyncio
async def test_stream_parallel_sampling_first_output_missing_a_choice():
"""With n=2 the first output may carry only one choice; the other must
still stream, and an abort finishes both."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], index=0)
yield _make_request_output("req-1", token_ids=[11], index=1)
aborted = _make_request_output(
"req-1", token_ids=[], finish_reason="abort", finished=True
)
aborted.outputs += _make_request_output(
"req-1", token_ids=[], finish_reason="abort", index=1
).outputs
yield aborted
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10, n=2),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(include_usage=True),
)
response = await serving.serve_tokens(request)
parsed = _parse_sse_chunks([chunk async for chunk in response])
choices = [c["choices"][0] for c in parsed[:-2]]
assert [(c["index"], c["token_ids"], c["finish_reason"]) for c in choices] == [
(0, [10], None),
(1, [11], None),
(0, [], "abort"),
(1, [], "abort"),
]
assert parsed[-2]["usage"]["completion_tokens"] == 2
@pytest.mark.asyncio
async def test_stream_error_mid_generation():
"""finish_reason='error' mid-stream yields error chunk then [DONE]."""
@@ -710,21 +710,23 @@ async def test_text_mode_rejected_when_tokens_only(client):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_text_mode_stream_delivers_abort_finish_chunk(
client, tokenizer, messages
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
async def test_stream_delivers_abort_finish_chunk(
client, tokenizer, messages, output_mode
):
"""The final output after an abort has no new token IDs, so it
reaches a text stream only because text mode emits finish-only chunks."""
"""The final output after an abort has no new token IDs, but the
stream must still deliver its finish reason."""
payload = {
**_text_mode_payload(
_chat_prompt_token_ids(tokenizer, messages),
max_tokens=900,
ignore_eos=True,
),
"output_mode": output_mode,
"stream": True,
"request_id": "text-abort-e2e",
"request_id": f"{output_mode}-abort-e2e",
}
chunks = []
@@ -736,7 +738,7 @@ async def test_text_mode_stream_delivers_abort_finish_chunk(
continue
chunk = json.loads(line[len("data: ") :])
chunks.append(chunk)
if not aborted and chunk["choices"] and chunk["choices"][0]["text"]:
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
abort = await client.post(
ABORT_ENDPOINT,
json={"request_ids": [chunk["request_id"]]},
@@ -745,7 +747,7 @@ async def test_text_mode_stream_delivers_abort_finish_chunk(
aborted = True
assert aborted
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
final_choice = chunks[-1]["choices"][0]
assert final_choice["finish_reason"] == "abort"
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
@@ -2,11 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
from pathlib import Path
import requests
import torch
from prometheus_client.multiprocess import MultiProcessCollector
from prometheus_client.parser import text_string_to_metric_families
from transformers import AutoModelForCausalLM
@@ -21,10 +18,7 @@ from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
# The API servers accept from one shared listening socket, so the kernel may
# hand one process most connections. Sync until both have recorded, within a cap.
MIN_SYNCS = 8
MAX_SYNCS = 64
SYNCS = 8
def scrape(url: str) -> dict[tuple[str, str], float]:
@@ -37,17 +31,6 @@ def scrape(url: str) -> dict[tuple[str, str], float]:
}
def pids_that_recorded(multiproc_dir: Path) -> set[str]:
return {
path.stem.rsplit("_", 1)[1]
for path in multiproc_dir.glob("histogram_*.db")
if any(
metric.name == DURATION
for metric in MultiProcessCollector.merge([str(path)])
)
}
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
args = [
"--load-format",
@@ -77,17 +60,11 @@ def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
client=HTTPVLLMWeightSyncClient(url),
source=ModuleSource(trainer),
)
syncs = 0
while syncs < MIN_SYNCS or (
len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS
):
for _ in range(SYNCS):
engine.send_weights()
syncs += 1
metrics = scrape(url)
calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs}
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
for operation, count in calls.items():
assert metrics[(f"{DURATION}_count", operation)] == count, operation
assert metrics[(IN_FLIGHT, operation)] == 0, operation
# The counts above are sums over processes only if both servers recorded.
assert len(pids_that_recorded(tmp_path)) == 2
@@ -5,6 +5,7 @@ import math
import random
import time
from collections.abc import Callable
from types import SimpleNamespace
import pytest
import torch
@@ -323,6 +324,63 @@ def test_contexted_kv_attention(
torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend")
@pytest.mark.parametrize("sliding_window", [None, 1, 4])
@pytest.mark.parametrize("query_len", [1, 3])
@torch.inference_mode()
def test_rocm_backend_sliding_window_includes_boundary_token(
sliding_window: int | None, query_len: int
) -> None:
"""A window of W must include the current token and W - 1 preceding tokens."""
from vllm.v1.attention.backends.rocm_attn import RocmAttentionImpl
from vllm.v1.attention.ops.paged_attn import PagedAttention
device, dtype = "cuda:0", torch.float16
num_heads, head_size, seq_len = 4, 128, 8
query = torch.zeros(query_len, num_heads, head_size, dtype=dtype, device=device)
key = torch.zeros(seq_len, 1, head_size, dtype=dtype, device=device)
values = torch.arange(1, seq_len + 1, dtype=dtype, device=device)
value = values[:, None, None].expand_as(key)
kv_cache = torch.zeros(2, 1, 16, head_size, dtype=dtype, device=device)
_, value_cache = PagedAttention.split_kv_cache(kv_cache, 1, head_size)
value_cache[..., :seq_len] = value.permute(1, 2, 0)
scale = torch.tensor(1.0, device=device)
metadata = SimpleNamespace(
use_cascade=False,
num_actual_tokens=query_len,
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32, device=device),
seq_lens=torch.tensor([seq_len], dtype=torch.int32, device=device),
max_query_len=query_len,
max_seq_len=seq_len,
block_table=torch.zeros(1, 1, dtype=torch.int32, device=device),
causal=True,
)
impl = RocmAttentionImpl(
num_heads=num_heads,
head_size=head_size,
scale=head_size**-0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
)
output = torch.empty_like(query)
impl.forward(
SimpleNamespace(_k_scale=scale, _v_scale=scale),
query,
key[-query_len:],
value[-query_len:],
kv_cache.transpose(0, 1),
metadata,
output,
)
mask = create_causal_attention_mask_for_sdpa(
[query_len], [seq_len], sliding_window or 0, device=device, dtype=dtype
)
expected = (mask.float().softmax(dim=-1) @ values.float())[:, None, None]
torch.testing.assert_close(output, expected.to(dtype).expand_as(output))
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -0,0 +1,585 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Determinism + FP32-reference correctness tests for the RDNA3 W4A16 GEMM.
The split-K epilogues of ``gptq_gemm_rdna3`` (scalar path) and the WMMA
path previously accumulated per-split low-precision partials directly into
the output via CAS atomics. With multiple writers per output element the
accumulation order — and therefore the rounded result — varied per call:
repeated fixed-input calls produced a different output almost every time on
gfx1100 once more than a few split blocks contended.
Both paths now store FP32 split partials and reduce them in a fixed order
with a single final low-precision rounding, which is deterministic by
construction. When the split count is 1 every path stores directly
(single writer per output element).
These tests assert, through the public op path:
A. Repeatability — ``torch.equal`` across repeated identical calls.
B. Numerical correctness — against an FP32 dequantized reference.
(B) exists because repeatability alone cannot see the failure mode the
deterministic epilogue introduces: the FP32 partials scratch is
``at::empty`` (its coverage invariant is documented at
``alloc_wmma_partials`` in ``csrc/rocm/q_gemm_rdna3_wmma.cu``), and across
repeated calls the caching allocator hands back the same block — a slot
that ever went unwritten would be bitwise-repeatable *and* wrong.
Reference semantics (uint4b8, GPTQv1, synthesized zero points):
W[k, n] = (q[k, n] - 8) * float(round_to_dtype(scale[g, n])), g = k//G
ref[m, n] = x[m, :] @ W[:, n] (FP32 matmul)
Tolerances are derived from the kernel's rounding structure, not tuned to
pass (measured max errors on W7900/gfx1100 in parentheses):
* scalar bf16 — dequant keeps FP32 precision end-to-end (magic-value
v_dot2 path); the only rounding is the final cast. Measured max ~0.06
on the tested shapes (|ref| up to ~30); tolerance 0.10 with headroom
for larger |ref| at the final-cast ulp.
* scalar fp16 — the dequant uses the exact factored form (magic values in
fp16, (y, z) correction in fp32; see qdq_4_rdna3.cuh), so the only fp16
roundings are the final output cast and the B-tile-free dot inputs:
max_abs <= 0.5 * ulp_fp16(max|ref|) * 1.5 + 1e-3 (measured ~0.015 at
|ref|~37; the pre-fix baked-constant form measured ~1.0-1.4 on the same
shapes — 94-200x worse).
* WMMA bf16 — B is narrowed to bf16 per cell (one rounding,
<= 0.5*ulp_bf16(|B|)), accumulated in FP32; measured max ≈ 0.09 at
K=4096 — tolerance 0.25.
* WMMA fp16 — B narrowed to fp16 via the precise sub-then-mul variant
(one rounding, ~1e-4); measured max ≈ 0.015 — tolerance 0.05.
All four are orders of magnitude below what an unwritten scratch slot
(caching-allocator garbage / NaN) or an indexing bug (errors of the same
scale as |ref| ≈ 7) would produce.
Test hygiene: the op path needs a single-process distributed group and a
current VLLM config; tests use the repo-standard ``dist_init`` fixture from
``tests/conftest.py`` (real context manager, temp-file rendezvous so no
hardcoded MASTER_PORT can collide in parallel CI, and the suite's standard
post-test cleanup).
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
RDNA3W4A16LinearKernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
pack_quantized_values_into_int32,
)
from vllm.model_executor.parameter import ( # noqa: E402
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
from vllm.scalar_type import scalar_types # noqa: E402
device = "cuda"
WEIGHT_TYPE = scalar_types.uint4b8
gfx1100_only = pytest.mark.skipif(
not (
on_gfx1100()
and hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
),
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
)
REPEATS = 20
GROUP = 128
DTYPES = [torch.bfloat16, torch.float16]
# max_abs_error tolerances vs the FP32 reference, per (path, dtype).
# Derivation in the module docstring; ~2-3x headroom over measured maxima.
TOL = {
("scalar", torch.bfloat16): 0.10,
("scalar", torch.float16): 0.025,
("wmma", torch.bfloat16): 0.25,
("wmma", torch.float16): 0.05,
}
def _build_layer(k, n, seed, dtype, group=GROUP):
torch.manual_seed(seed)
q_int4_kn = torch.randint(0, 16, (k, n), dtype=torch.int32)
scales_gn = (torch.randn(k // group, n) * 0.01 + 0.02).to(dtype)
qweight = pack_quantized_values_into_int32(q_int4_kn, WEIGHT_TYPE, packed_dim=0)
no_loader = lambda *a, **kw: None # noqa: E731
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.register_parameter(
"qweight",
PackedvLLMParameter(
data=qweight,
weight_loader=no_loader,
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=8,
),
)
layer.register_parameter(
"scales",
GroupQuantScaleParameter(
data=scales_gn, weight_loader=no_loader, input_dim=0, output_dim=1
),
)
layer.to(device)
return layer, q_int4_kn, scales_gn
def _prepare(layer, dtype, k, n, group=GROUP):
cfg = MPLinearLayerConfig(
full_weight_shape=(k, n),
partition_weight_shape=(k, n),
weight_type=WEIGHT_TYPE,
act_type=dtype,
group_size=group,
zero_points=False,
)
kernel = RDNA3W4A16LinearKernel(
cfg, w_q_param_name="qweight", w_s_param_name="scales", w_zp_param_name=None
)
kernel.process_weights_after_loading(layer)
return kernel._get_weight_params(layer)
def _reference(m, k, dtype, seed, q_int4_kn, scales_gn, group=GROUP):
torch.manual_seed(seed + 1)
x = torch.randn(m, k, device=device, dtype=dtype)
# FP32 reference; see module docstring for the dequant semantics.
w_f32 = (q_int4_kn.to(device).float() - 8.0) * scales_gn.to(
device
).repeat_interleave(group, dim=0).float()
return x, x.float() @ w_f32
def _run_op(x, w_q, w_zp, w_s):
return torch.ops._rocm_C.gptq_gemm_rdna3(x, w_q, w_zp, w_s, False)
def _outputs_and_ref(M, K, N, seed, dtype, repeats=1, group=GROUP):
layer, q_int4_kn, scales_gn = _build_layer(K, N, seed, dtype, group)
w_q, w_s, w_zp = _prepare(layer, dtype, K, N, group)
x, ref = _reference(M, K, dtype, seed, q_int4_kn, scales_gn, group)
outs = [_run_op(x, w_q, w_zp, w_s) for _ in range(repeats)]
return outs, ref
def _path_for(dtype, m):
"""Public dispatch: both dtypes reach WMMA at M>=12 (see the
WMMA_MIN_M crossover table in csrc/rocm/q_gemm_rdna3.cu)."""
if m >= 12:
return "wmma"
return "scalar"
def _assert_repeatable(outs):
for o in outs[1:]:
assert torch.equal(o, outs[0]), (
"RDNA3 W4A16 split-K produced differing outputs for identical inputs"
)
def _assert_close_to_ref(out, ref, path, dtype):
err = (out.float() - ref).abs()
max_abs = err.max().item()
mean_abs = err.mean().item()
tol = TOL[(path, dtype)]
assert torch.isfinite(out.float()).all(), "non-finite output"
assert max_abs <= tol, (
f"RDNA3 W4A16 {path} output deviates from the FP32 dequantized "
f"reference: max_abs={max_abs:.4f} > tol={tol} (mean_abs="
f"{mean_abs:.4f}). This catches values that are bitwise-repeatable "
"but wrong (e.g. an unwritten FP32 scratch slot)."
)
# ---------------------------------------------------------------------------
# A. Repeatability (bit-exact across identical calls)
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_splitk_bit_repeatable(dist_init, dtype):
"""Scalar path at production-like contention: K=4096 splits into
ceil(4096/256) = 16 concurrent writers per output element (the old CAS
bug's onset was between 2 and 4 writers, so this sits well inside the
failure regime rather than at the empirical threshold)."""
outs, _ = _outputs_and_ref(
M=1, K=4096, N=4096, seed=1234, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_small_m_bit_repeatable(dist_init, dtype):
"""Scalar M>1 tiles share the same deterministic epilogue."""
outs, _ = _outputs_and_ref(
M=8, K=4096, N=4096, seed=1235, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_single_split_bit_repeatable(dist_init, dtype):
"""z_count == 1 (K=256 < BLOCK_KN_SIZE): direct-store epilogue, no
scratch, no reduce pass."""
outs, _ = _outputs_and_ref(
M=1, K=256, N=4096, seed=1238, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_splitk_bit_repeatable(dist_init, dtype):
"""WMMA path (both dtypes M >= 12), split-K active.
K=6656 gives K_SPLIT=4 under the upstream heuristic; the deterministic
epilogue must be bit-repeatable regardless of the split count.
"""
m = 12 # smallest M on the WMMA side of the dispatch boundary
outs, _ = _outputs_and_ref(
M=m, K=6656, N=4096, seed=1236, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_large_m_bit_repeatable(dist_init, dtype):
"""Large-M WMMA tiles (128x64 kernels) with the deterministic epilogue."""
outs, _ = _outputs_and_ref(
M=128, K=6656, N=4096, seed=1237, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
# ---------------------------------------------------------------------------
# B. Numerical correctness against the FP32 dequantized reference
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_matches_fp32_reference(dist_init, dtype):
"""Scalar split-K (K=4096, 16 writers) vs the FP32 reference."""
m = 1
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1240, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("m", [2, 3, 4, 5, 8])
def test_scalar_multi_m_matches_fp32_reference(dist_init, m, dtype):
"""The exact factored dequant across every M_COUNT instantiation
(M=2,3 -> M_COUNT=2; M=4,5 -> M_COUNT=4 with a padded row; M=8 -> 8):
the (y, z) correction and the nibble-slot activation sums must hold
for every tile width, not just the M=1 fast path."""
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1247, dtype=dtype)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_single_split_matches_fp32_reference(dist_init, dtype):
"""Scalar z_count == 1 direct-store path vs the FP32 reference."""
m = 1
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1241, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_matches_fp32_reference(dist_init, dtype):
"""WMMA split-K path (16x16_1w at the M=12 boundary) vs FP32 reference."""
m = 12 # smallest M on the WMMA side of the dispatch boundary
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1242, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_single_split_matches_fp32_reference(dist_init, dtype):
"""WMMA k_split == 1 direct-store path (K=256 -> compute_wmma_k_split
returns 1 for every dispatch level) vs the FP32 reference."""
m = 12 # smallest M on the WMMA side of the dispatch boundary
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1243, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_large_m_matches_fp32_reference(dist_init, dtype):
"""Large-M WMMA (128x64_k32, row-tiled scratch + reduce) vs FP32."""
m = 128
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1244, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
# ---------------------------------------------------------------------------
# C. V7/V8 (128x64) no-split path: k_split == 1 -> direct store, no scratch,
# no reduce pass (partials == nullptr).
# ---------------------------------------------------------------------------
# The measured production-like shape from the W7900 benchmark campaign.
# Routing arithmetic (see _assert_v7v8_no_split_routing): with (128, 64)
# tiles, blocks_xy = ceil(25600/64) * ceil(512/128) = 400 * 4 = 1600 >=
# 1500, so compute_wmma_k_split_mn returns 1. Smaller no-split shapes are
# not practical: N would have to grow past ~24k at M=512 (or M past ~1k at
# N=12k) to reach the 1500-block threshold, so this is effectively the
# smallest production-realistic V7/V8 no-split shape.
V7V8_NOSPLIT = (512, 25600, 6656) # (M, N, K)
def _compute_wmma_k_split_mn(m, n, k, m_tile, n_tile):
"""Faithful replica of compute_wmma_k_split_mn (C++ heuristic)."""
blocks_xy = ((n + n_tile - 1) // n_tile) * ((m + m_tile - 1) // m_tile)
if blocks_xy >= 1500:
return 1
if blocks_xy * 2 >= 1500 and k >= 512 and k % 32 == 0:
return 2
if blocks_xy * 4 >= 1500 and k >= 1024 and k % 64 == 0:
return 4
if k >= 1024 and k % 64 == 0:
return 4
if k >= 512 and k % 32 == 0:
return 2
return 1
def _assert_v7v8_no_split_routing(dtype, m, n, k):
"""Assert the shape deterministically routes to the V7/V8 128x64
kernel with k_split == 1: WMMA dispatch (both dtypes M >= 12), the
M >= 128 branch, and the no-split threshold."""
assert m >= 12, "not the WMMA path"
assert m >= 128, "below the V7/V8 (128x64) branch"
assert _compute_wmma_k_split_mn(m, n, k, 128, 64) == 1, (
"shape does not route to the k_split == 1 no-split path"
)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_v7v8_no_split_bit_repeatable(dist_init, dtype):
"""V7/V8 128x64_k32 with k_split == 1: single direct writer per output
cell — no FP32 scratch, no reduce pass."""
m, n, k = V7V8_NOSPLIT
_assert_v7v8_no_split_routing(dtype, m, n, k)
outs, _ = _outputs_and_ref(M=m, K=k, N=n, seed=1245, dtype=dtype, repeats=REPEATS)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_v7v8_no_split_matches_fp32_reference(dist_init, dtype):
"""V7/V8 128x64_k32 no-split direct-store path vs the FP32 reference."""
m, n, k = V7V8_NOSPLIT
_assert_v7v8_no_split_routing(dtype, m, n, k)
outs, ref = _outputs_and_ref(M=m, K=k, N=n, seed=1246, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
# ---------------------------------------------------------------------------
# D. Dispatch boundary: M=11 must stay scalar, M=12 must route to WMMA
# (WMMA_MIN_M in csrc/rocm/q_gemm_rdna3.cu). Observed via the CUDA
# profiler: the scalar and WMMA kernels have disjoint names.
# ---------------------------------------------------------------------------
def _launched_kernel_names(fn) -> set:
from torch.profiler import ProfilerActivity, profile
fn()
torch.accelerator.synchronize()
with profile(activities=[ProfilerActivity.CUDA]) as prof:
fn()
torch.accelerator.synchronize()
return {
e.name
for e in prof.events()
if "gemm_q4" in e.name or "reduce_partials" in e.name
}
@pytest.mark.parametrize("dtype", DTYPES)
@gfx1100_only
def test_dispatch_boundary(dist_init, dtype):
"""The scalar/WMMA crossover sits between M=11 and M=12 for both
dtypes; the boundary must be exact (see the WMMA_MIN_M table)."""
def run(m):
outs, _ = _outputs_and_ref(M=m, K=4096, N=512, seed=1248, dtype=dtype)
return outs[0]
m11 = run(11)
names11 = _launched_kernel_names(lambda: run(11))
assert any("gemm_q4_kernel_rdna3" in n for n in names11), names11
assert not any("wmma" in n for n in names11), names11
m12 = run(12)
names12 = _launched_kernel_names(lambda: run(12))
assert any("wmma" in n for n in names12), names12
assert not any("gemm_q4_kernel_rdna3" in n for n in names12), names12
# Both sides of the boundary stay correct against the reference.
_, ref11 = _outputs_and_ref(M=11, K=4096, N=512, seed=1248, dtype=dtype)
_, ref12 = _outputs_and_ref(M=12, K=4096, N=512, seed=1248, dtype=dtype)
_assert_close_to_ref(m11, ref11, "scalar", dtype)
_assert_close_to_ref(m12, ref12, "wmma", dtype)
# ---------------------------------------------------------------------------
# E. In-kernel split-K reduction: stress the per-call ticket epilogue.
# The fused reduction must stay deterministic and correct under
# back-to-back unsynchronized calls, concurrent invocations on two
# streams (per-call tickets must never share state), and CUDA-graph
# capture/replay.
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_splitk_100x_bit_repeatable(dist_init, dtype):
"""100 back-to-back calls (no intermediate sync) on the split-K path."""
outs, _ = _outputs_and_ref(M=1, K=4096, N=512, seed=1249, dtype=dtype, repeats=100)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_multistream_concurrent(dist_init, dtype):
"""Two concurrent invocations on separate streams, distinct weights and
activations: outputs must be bit-identical to the serial invocation of
each (the arrival tickets are per-call, so cross-stream interference
would show up here as a corrupted or unreduced output)."""
layer_a, q_a, s_a = _build_layer(4096, 512, 4321, dtype)
layer_b, q_b, s_b = _build_layer(4096, 512, 4322, dtype)
wq_a, ws_a, wz_a = _prepare(layer_a, dtype, 4096, 512)
wq_b, ws_b, wz_b = _prepare(layer_b, dtype, 4096, 512)
torch.manual_seed(4323)
x_a = torch.randn(1, 4096, device=device, dtype=dtype)
x_b = torch.randn(1, 4096, device=device, dtype=dtype)
serial_a = _run_op(x_a, wq_a, wz_a, ws_a)
serial_b = _run_op(x_b, wq_b, wz_b, ws_b)
torch.accelerator.synchronize()
s1, s2 = torch.cuda.Stream(), torch.cuda.Stream()
for _ in range(25):
with torch.cuda.stream(s1):
out_a = _run_op(x_a, wq_a, wz_a, ws_a)
with torch.cuda.stream(s2):
out_b = _run_op(x_b, wq_b, wz_b, ws_b)
torch.accelerator.synchronize()
assert torch.equal(out_a, serial_a), "concurrent stream A corrupted"
assert torch.equal(out_b, serial_b), "concurrent stream B corrupted"
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_graph_capture_replay(dist_init, dtype):
"""The op must survive CUDA-graph capture: the replayed output must be
bit-identical to the eager output and stable across repeated replays."""
layer, q_int4, scales_gn = _build_layer(4096, 512, 4324, dtype)
wq, ws, wz = _prepare(layer, dtype, 4096, 512)
torch.manual_seed(4325)
x = torch.randn(1, 4096, device=device, dtype=dtype)
eager = _run_op(x, wq, wz, ws)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
_run_op(x, wq, wz, ws) # warm-up side streams/allocator on capture path
torch.accelerator.synchronize()
with torch.cuda.graph(graph):
captured = _run_op(x, wq, wz, ws)
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(captured, eager), "graph replay differs from eager"
graph.replay()
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(captured, eager), "repeated replay not repeatable"
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_multi_row_tile_bit_repeatable(dist_init, dtype):
"""Row tiling with ticket reuse: N % 16 == 8 bypasses the WMMA
dispatch (N % 16 != 0), so M=100 runs scalar across two 64-row tiles
(the second with gy > 1 y-blocks). Every tile's tickets must be armed,
consumed, and re-armed correctly for all outputs to be written exactly
once — and the result must be bit-repeatable."""
outs, ref = _outputs_and_ref(
M=100, K=4096, N=2056, seed=1250, dtype=dtype, repeats=20
)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
# ---------------------------------------------------------------------------
# F. Split-K geometry corners: clipped non-first z blocks, small split
# counts, and non-128 group sizes.
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("k,group", [(4224, 128), (2112, 32)])
def test_scalar_clipped_tail_splitk(dist_init, dtype, k, group):
"""K % 256 != 0 with z_count > 1: the LAST z block is clipped by end_k,
its group prefetch guard (`k + 32 < end_k`) is what keeps the raw
zero/scale loads in bounds, and the reduce tail loop runs (z=17 -> one
remainder slice; z=9 -> one batch + one). K=2112/G=32 also puts two
group switches inside the clipped tail block."""
outs, ref = _outputs_and_ref(M=1, K=k, N=512, seed=1251, dtype=dtype, group=group)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("group", [32, 64])
def test_scalar_small_group_matches_fp32_reference(dist_init, dtype, group):
"""Group sizes below 128 make the group constants change every 32-K
round (G=32) or three times per z block (G=64); the factored-dequant
correction must track every change. G=32 is also the production MoE
group size."""
outs, ref = _outputs_and_ref(
M=1, K=2048, N=512, seed=1252, dtype=dtype, group=group
)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_two_way_split_bit_repeatable(dist_init, dtype):
"""z_count == 2 (K=512): the minimal split-K configuration — the old
CAS bug's onset regime — and the pure-remainder reduce tail."""
outs, _ = _outputs_and_ref(
M=1, K=512, N=512, seed=1253, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
+15 -13
View File
@@ -711,7 +711,8 @@ def test_v41_rope_insert_plain_row(compress_ratio: int, store_fp8: bool):
def test_v41_compressor_metadata_maps_tokens_to_their_ring():
"""The ring group's generic slot mapping is disabled (all PAD), so the
builder must map every real token to ``ring_block * capacity + pos %
capacity`` and keep padding tokens at PAD."""
capacity``. Padding tokens and requests on the null block (block 0, as in
dummy batches) must stay at PAD."""
from unittest.mock import MagicMock
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
@@ -731,29 +732,30 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
device = torch.device("cuda")
builder = CompressorMetadataBuilder(spec, ["state"], vllm_config, device)
# Two requests: 3 tokens at positions 13..15 on ring block 5, then 2
# tokens at positions 7..8 on ring block 2; three padding tokens.
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
positions = torch.tensor([13, 14, 15, 7, 8, 0, 0, 0], device=device)
block_table = torch.tensor([[5], [2]], dtype=torch.int32, device=device)
# Three requests: 3 tokens at positions 13..15 on ring block 5, 2 tokens
# at positions 7..8 on ring block 2, then 2 tokens on the null block;
# three padding tokens.
query_start_loc = torch.tensor([0, 3, 5, 7], dtype=torch.int32, device=device)
positions = torch.tensor([13, 14, 15, 7, 8, 0, 1, 0, 0, 0], device=device)
block_table = torch.tensor([[5], [2], [0]], dtype=torch.int32, device=device)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([16, 9], dtype=torch.int32, device=device),
num_reqs=2,
num_actual_tokens=5,
seq_lens=torch.tensor([16, 9, 2], dtype=torch.int32, device=device),
num_reqs=3,
num_actual_tokens=7,
max_query_len=3,
max_seq_len=16,
block_table_tensor=block_table,
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
slot_mapping=torch.full((10,), -1, dtype=torch.int64, device=device),
positions=positions,
)
metadata = builder.build(0, common)
expected = [5 * 8 + 5, 5 * 8 + 6, 5 * 8 + 7, 2 * 8 + 7, 2 * 8 + 0, -1, -1, -1]
assert metadata.slot_mapping.tolist() == expected
ring = [5 * 8 + 5, 5 * 8 + 6, 5 * 8 + 7, 2 * 8 + 7, 2 * 8 + 0]
assert metadata.slot_mapping.tolist() == ring + [-1] * 5
assert metadata.query_start_loc is query_start_loc
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1]
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
+19
View File
@@ -26,6 +26,7 @@ from vllm.config import SpeculativeConfig, VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptMixedPrecisionConfig,
ModelOptNvFp4Config,
@@ -573,6 +574,24 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
)
def test_ple_embedding_respects_inc_layer_config() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = INCConfig(
weight_bits=4,
group_size=128,
block_name_to_quantize="model.layers",
extra_config={".*ple.*": {"bits": 16, "data_type": "float"}},
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
quant_config.extra_config = None
with pytest.raises(NotImplementedError, match="INCConfig"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
+1 -5
View File
@@ -380,12 +380,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Llama4ForCausalLM": _HfExamplesInfo(
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
),
"LongcatFlashForCausalLM": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
),
"LongcatFlashForCausalLM": _HfExamplesInfo("meituan-longcat/LongCat-Flash-Chat"),
"LongcatFlashNgramForCausalLM": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Lite",
trust_remote_code=True,
# Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size)
# so the dummy-weight init test fits in CI memory.
hf_overrides={"ngram_vocab_size_ratio": 1},
@@ -1752,7 +1749,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
),
"LongCatFlashMTPModel": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Chat",
trust_remote_code=True,
speculative_model="meituan-longcat/LongCat-Flash-Chat",
),
"MiMoMTPModel": _HfExamplesInfo(
+19 -13
View File
@@ -216,17 +216,19 @@ class TestNormalizeAudio:
assert result.ndim == 1
torch.testing.assert_close(result, torch.tensor([0.0, 1.0]))
def test_mono_passthrough_for_1d_numpy(self):
"""1D numpy array should pass through unchanged with mono spec."""
mono = np.array([1.0, 2.0, 3.0], dtype=np.float32)
result = normalize_audio(mono, MONO_AUDIO_SPEC)
@pytest.mark.parametrize("shape", [(3,), (1, 3), (3, 1), (1, 1)])
def test_mono_passthrough_for_numpy(self, shape):
"""Mono output is 1D, including audio with an explicit channel axis."""
mono = np.arange(max(shape), dtype=np.float32)
result = normalize_audio(mono.reshape(shape), MONO_AUDIO_SPEC)
assert result.ndim == 1
np.testing.assert_array_equal(result, mono)
def test_mono_passthrough_for_1d_torch(self):
"""1D torch tensor should pass through unchanged with mono spec."""
mono = torch.tensor([1.0, 2.0, 3.0])
result = normalize_audio(mono, MONO_AUDIO_SPEC)
@pytest.mark.parametrize("shape", [(3,), (1, 3), (3, 1), (1, 1)])
def test_mono_passthrough_for_torch(self, shape):
"""Mono output is 1D, including audio with an explicit channel axis."""
mono = torch.arange(max(shape), dtype=torch.float32)
result = normalize_audio(mono.reshape(shape), MONO_AUDIO_SPEC)
assert result.ndim == 1
torch.testing.assert_close(result, mono)
@@ -410,8 +412,9 @@ class TestMultiModalDataParserChannelNormalization:
# When target_channels=None, stereo audio should be preserved
assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D"
def test_parser_mono_passthrough_when_target_channels_1(self):
"""Parser should pass through mono audio unchanged when target_channels=1."""
@pytest.mark.parametrize("shape", [(16000,), (1, 16000), (16000, 1)])
def test_parser_mono_passthrough_when_target_channels_1(self, shape):
"""Mono layouts must preserve samples and report the waveform length."""
from vllm.multimodal.parse import MultiModalDataParser
# Create parser with mono normalization enabled
@@ -420,17 +423,20 @@ class TestMultiModalDataParserChannelNormalization:
target_channels=1,
)
# Create mono audio (already 1D)
mono_audio = np.random.randn(16000).astype(np.float32)
mono_audio = np.linspace(-1.0, 1.0, 16000, dtype=np.float32)
# Parse audio data
result = parser._parse_audio_data((mono_audio, 16000))
result = parser.parse_mm_data({"audio": (mono_audio.reshape(shape), 16000)})[
"audio"
]
# Check that result is still mono (1D)
assert result is not None
audio_item = result.get(0)
assert audio_item.ndim == 1
assert audio_item.shape == (16000,)
assert len(audio_item) == 16000
np.testing.assert_array_equal(audio_item, mono_audio)
def test_parser_with_target_channels_2(self):
"""Parser should reduce 6-channel to 2-channel when target_channels=2."""
+146 -2
View File
@@ -1098,8 +1098,14 @@ def test_flashinfer_mla_dspark_dcp_supports_target_and_draft(monkeypatch):
pytest.param(False, 3, 1, 0, id="noncausal-multi-token"),
],
)
@pytest.mark.parametrize("cp_interleave_size", [1, 16, 896])
def test_tokenspeed_mla_decode_contract(
monkeypatch, causal, tokens_per_decode, dcp_world_size, dcp_rank
monkeypatch,
causal,
tokens_per_decode,
dcp_world_size,
dcp_rank,
cp_interleave_size,
):
decode_call = None
num_decodes = 2
@@ -1135,7 +1141,7 @@ def test_tokenspeed_mla_decode_contract(
impl = object.__new__(tokenspeed_mla_module.TokenspeedMLAImpl)
impl.dcp_world_size = dcp_world_size
impl.dcp_rank = dcp_rank
impl.cp_kv_cache_interleave_size = 1
impl._parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=1)
impl.need_to_return_lse_for_decode = True
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = qk_rope_head_dim
@@ -1164,6 +1170,8 @@ def test_tokenspeed_mla_decode_contract(
dtype=torch.float8_e4m3fn,
)
# NIXL may resolve the interleave size after the implementation is created.
impl._parallel_config.cp_kv_cache_interleave_size = cp_interleave_size
out, lse = impl.forward_mqa(
q,
kv_cache,
@@ -1194,6 +1202,142 @@ def test_tokenspeed_mla_decode_contract(
assert decode_call["return_lse"] is True
assert decode_call["cp_world"] == dcp_world_size
assert decode_call["cp_rank"] == dcp_rank
assert decode_call.get("cp_interleave_size", 1) == cp_interleave_size
@pytest.mark.skipif(
AttentionBackendEnum.TOKENSPEED_MLA not in BACKENDS_TO_TEST,
reason="TokenSpeed MLA requires an SM100-family GPU and tokenspeed-mla",
)
@pytest.mark.parametrize(
("cp_interleave_size", "seq_len"),
[
pytest.param(1, 5, id="token-interleave-empty-local-ranks"),
pytest.param(1, 8 + 17, id="token-interleave-full-cycle"),
pytest.param(896, 5, id="block-interleave-empty-local-ranks"),
pytest.param(896, 8 * 896 + 17, id="block-interleave-full-cycle"),
],
)
def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len):
from tokenspeed_mla import tokenspeed_mla_decode
from vllm.v1.attention.ops.dcp import (
_lse_weighted_combine,
mask_dcp_empty_shards_,
)
torch.manual_seed(7)
device = torch.device("cuda")
dcp_world_size = 8
kernel_block_size = 64
num_heads = 128
kv_lora_rank = 512
qk_rope_head_dim = 64
head_size = kv_lora_rank + qk_rope_head_dim
def make_paged_cache(tokens: torch.Tensor):
# Empty DCP ranks still have the null block in vLLM's physical cache.
num_pages = max(1, cdiv(tokens.shape[0], kernel_block_size))
cache = torch.zeros(
num_pages,
kernel_block_size,
head_size,
dtype=tokens.dtype,
device=device,
)
cache.view(-1, head_size)[: tokens.shape[0]].copy_(tokens)
block_table = torch.arange(
num_pages, dtype=torch.int32, device=device
).unsqueeze(0)
return cache, block_table
# Four query positions exercise the MTP/DSpark causal-mask path.
query = (torch.randn(1, 4, num_heads, head_size, device=device) * 0.1).to(
torch.float8_e4m3fn
)
global_tokens = (torch.randn(seq_len, head_size, device=device) * 0.1).to(
torch.float8_e4m3fn
)
workspace = tokenspeed_mla_module._get_workspace(device, num_heads, kv_lora_rank)
global_cache, global_block_table = make_paged_cache(global_tokens)
global_seq_len = torch.tensor([seq_len], dtype=torch.int32, device=device)
reference, _ = tokenspeed_mla_decode(
query=query,
kv_cache=global_cache,
workspace_buffer=workspace,
kv_lora_rank=kv_lora_rank,
qk_rope_head_dim=qk_rope_head_dim,
block_tables=global_block_table,
seq_lens=global_seq_len,
max_seq_len=seq_len,
softmax_scale=head_size**-0.5,
return_lse=True,
)
positions = torch.arange(seq_len, device=device)
owners = positions.div(cp_interleave_size, rounding_mode="floor").remainder(
dcp_world_size
)
max_local_seq_len = (
cdiv(seq_len, dcp_world_size * cp_interleave_size) * cp_interleave_size
)
partial_outputs = []
partial_lses = []
for dcp_rank in range(dcp_world_size):
local_tokens = global_tokens[owners == dcp_rank]
local_cache, local_block_table = make_paged_cache(local_tokens)
local_seq_len = torch.tensor(
[local_tokens.shape[0]], dtype=torch.int32, device=device
)
impl = object.__new__(tokenspeed_mla_module.TokenspeedMLAImpl)
impl.dcp_world_size = dcp_world_size
impl.dcp_rank = dcp_rank
impl._parallel_config = SimpleNamespace(
cp_kv_cache_interleave_size=cp_interleave_size
)
impl.need_to_return_lse_for_decode = True
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = qk_rope_head_dim
impl.num_heads = num_heads
impl.scale = head_size**-0.5
impl.softmax_scale = None
impl.output_scale = None
impl._workspace_buffer = workspace
metadata = SimpleNamespace(
num_decodes=1,
num_decode_tokens=query.shape[1],
max_seq_len=max_local_seq_len,
causal=True,
decode=SimpleNamespace(
block_table=local_block_table,
seq_lens=local_seq_len,
dcp_tot_seq_lens=global_seq_len,
),
)
output, lse = impl.forward_mqa(
query.view(-1, num_heads, head_size),
local_cache,
metadata,
SimpleNamespace(_q_scale_float=1.0, _k_scale_float=1.0),
)
# Empty-shard normalization belongs to the downstream DCP combine path.
mask_dcp_empty_shards_(
lse,
local_seq_len,
torch.tensor([0, query.shape[1]], dtype=torch.int32, device=device),
)
partial_outputs.append(
output.view(query.shape[1], num_heads, kv_lora_rank).float()
)
partial_lses.append(lse.view(query.shape[1], num_heads).float())
# TokenSpeed returns base-2 LSE. Merge the per-rank partial attention states
# the same way MLADCPManager does before comparing to the unsharded result.
merged = _lse_weighted_combine(
torch.stack(partial_outputs), torch.stack(partial_lses), is_lse_base_on_e=False
).unsqueeze(0)
# FP8 attention rounds differently when KV is partitioned across DCP ranks.
torch.testing.assert_close(merged, reference.float(), atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("is_fp8_kvcache", [False, True], ids=["bf16", "fp8"])
@@ -50,6 +50,7 @@ def _create_vllm_config() -> MagicMock:
vllm_config.parallel_config = ParallelConfig()
vllm_config.speculative_config = None
vllm_config.num_speculative_tokens = 0
vllm_config.use_cumem_cudagraph_pool = False
return vllm_config
+20 -31
View File
@@ -18,6 +18,7 @@ from utils import (
import vllm.envs as envs
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt
@skip_unsupported
@@ -778,32 +779,16 @@ def test_decode_logprobs_match_prefill_logprobs(
print(f"[Prompt {prompt_idx}] Generated {len(token_ids)} tokens: {token_ids}")
print(f"[Prompt {prompt_idx}] Decode logprobs: {decode_logprobs.tolist()}")
# Step 2: For each token position, run prefill and compare
# Step 2: For each token position, run prefill and compare.
prompt_token_ids = list(decode_output.prompt_token_ids or [])
print(f"\n[Prompt {prompt_idx}] Verifying each token via prefill...")
for token_idx in range(len(token_ids)):
# Construct the prefix up to (but not including) this token
# Construct the prefix up to (but not including) this token using
# token ids directly.
current_token = token_ids[token_idx]
# We need to detokenize to get the text prefix
# For this, we'll use the tokenizer from the LLM
# However, the LLM API doesn't expose tokenizer easily, so we'll
# construct the prefix by decoding from the original prompt
# Get text up to this point by using the output text
# This is approximate but should work for verification
if token_idx == 0:
prefix_prompt = prompt
else:
# No per-token text, so regenerate with max_tokens = token_idx
# to get the prefix text
prefix_sp = SamplingParams(
temperature=0.0,
max_tokens=token_idx,
logprobs=1,
)
prefix_output = llm.generate([prompt], prefix_sp, use_tqdm=False)[0]
prefix_prompt = prompt + prefix_output.outputs[0].text
prefix_token_ids = prompt_token_ids + list(token_ids[:token_idx])
# Now run prefill with max_tokens=1 to get the logprob of the next token
prefill_sp = SamplingParams(
@@ -814,19 +799,23 @@ def test_decode_logprobs_match_prefill_logprobs(
print(
f" [Token {token_idx}] Running prefill for prefix "
f"(len={len(prefix_prompt)})..."
f"(num_tokens={len(prefix_token_ids)})..."
)
prefill_output = llm.generate(
[TokensPrompt(prompt_token_ids=prefix_token_ids)],
prefill_sp,
use_tqdm=False,
)[0]
prefill_logprobs, prefill_token_ids_out = _extract_step_logprobs(
prefill_output
)
prefill_output = llm.generate([prefix_prompt], prefill_sp, use_tqdm=False)[
0
]
prefill_logprobs, prefill_token_ids = _extract_step_logprobs(prefill_output)
if prefill_logprobs is None:
print(f" [Token {token_idx}] Warning: No prefill logprobs available")
continue
# The first token from prefill should match the current token
prefill_token = prefill_token_ids[0]
prefill_token = prefill_token_ids_out[0]
prefill_logprob = prefill_logprobs[0].item()
decode_logprob = decode_logprobs[token_idx].item()
@@ -851,7 +840,7 @@ def test_decode_logprobs_match_prefill_logprobs(
"decode_logprob": decode_logprob,
"prefill_logprob": prefill_logprob,
"prompt_text": prompt[:100],
"prefix_text": prefix_prompt[:100],
"prefix_token_ids": prefix_token_ids[:50],
}
)
print(f" [Token {token_idx}] ✗ TOKEN MISMATCH!")
@@ -871,7 +860,7 @@ def test_decode_logprobs_match_prefill_logprobs(
"prefill_logprob": prefill_logprob,
"diff": diff,
"prompt_text": prompt[:100],
"prefix_text": prefix_prompt[:100],
"prefix_token_ids": prefix_token_ids[:50],
"decode_all_tokens": token_ids,
"decode_all_logprobs": decode_logprobs.tolist(),
}
@@ -908,7 +897,7 @@ def test_decode_logprobs_match_prefill_logprobs(
for i, fail in enumerate(failures[:5]): # Show first 5 failures per prompt
print(f"\n [Failure {i + 1}] Token position {fail['token_idx']}:")
print(f" Reason: {fail['reason']}")
print(f" Prefix text: '{fail['prefix_text']}...'")
print(f" Prefix token ids: {fail['prefix_token_ids']}...")
print(
f" Decode: token={fail['decode_token']}, "
f"logprob={fail['decode_logprob']:.10f}"
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-invariant LoRA shrink: fixed split-K partials, fixed-order reduction."""
import pytest
import torch
from utils import skip_if_not_cuda
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
import vllm.lora.ops.triton_ops.utils as lora_utils
from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_shrink
from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT
from vllm.utils.torch_utils import set_random_seed
DEVICE = "cuda"
@pytest.fixture(autouse=True)
def batch_invariant(monkeypatch):
monkeypatch.setattr(lora_utils, "is_batch_invariant", True)
monkeypatch.setattr(shrink_op, "is_batch_invariant", True)
lora_utils.get_lora_op_configs.cache_clear()
yield
lora_utils.get_lora_op_configs.cache_clear()
def _shrink(inputs, weights, mapping, scaling):
num_tokens = inputs.size(0)
meta = LoRAKernelMeta.make(weights[0].size(0), num_tokens, device=DEVICE)
meta.prepare_tensors(mapping)
out = torch.zeros(
(len(weights), num_tokens, weights[0].size(1)),
dtype=torch.float32,
device=DEVICE,
)
_LORA_A_PTR_DICT.clear()
args = meta.meta_args(token_nums=num_tokens, specialize_active_lora=False)
lora_shrink(inputs, weights, out, *args, scaling)
return out
@skip_if_not_cuda
@pytest.mark.parametrize(
"hidden_size,rank,nslices,dtype",
[
(2560, 8, 3, torch.bfloat16),
(9728, 16, 1, torch.float16),
(4097, 64, 1, torch.bfloat16), # K tail block
(1024, 8, 1, torch.float16), # K < 8 * BLOCK_K: some splits are empty
],
)
def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
set_random_seed(0)
inputs = torch.randn((33, hidden_size), dtype=dtype, device=DEVICE)
weights = [
torch.randn((2, rank, hidden_size), dtype=dtype, device=DEVICE)
for _ in range(nslices)
]
mapping = torch.randint(-1, 2, (33,), dtype=torch.int32, device=DEVICE)
out = _shrink(inputs, weights, mapping, 0.5).double().cpu()
ids, x = mapping.cpu(), inputs.double().cpu()
ref = torch.zeros_like(out)
for s, weight in enumerate(weights):
for lora_id in (0, 1):
rows = ids == lora_id
ref[s, rows] = 0.5 * x[rows] @ weight[lora_id].double().cpu().T
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
@skip_if_not_cuda
@pytest.mark.parametrize("nslices", [1, 3])
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
class RecordReduce:
def __getitem__(self, grid):
reduce_grids.append(grid)
return reduce_kernel[grid]
monkeypatch.setattr(shrink_op, "_lora_shrink_reduce_kernel", RecordReduce())
set_random_seed(0)
# K = 2560 is ten BLOCK_K=256 blocks, so all 8 splits contribute.
dtype, hidden_size = torch.bfloat16, 2560
weights = [
torch.randn((3, 16, hidden_size), dtype=dtype, device=DEVICE)
for _ in range(nslices)
]
target = torch.randn((1, hidden_size), dtype=dtype, device=DEVICE)
target_id = torch.zeros(1, dtype=torch.int32, device=DEVICE)
expected = _shrink(target, weights, target_id, 0.7)[:, 0]
for num_tokens in (31, 32, 33, 127, 128, 129):
inputs = torch.randn((num_tokens, hidden_size), dtype=dtype, device=DEVICE)
mapping = torch.randint(-1, 3, (num_tokens,), dtype=torch.int32, device=DEVICE)
pos = num_tokens // 2
inputs[pos], mapping[pos] = target[0], 0
out = _shrink(inputs, weights, mapping, 0.7)
assert torch.equal(out[:, pos], expected), num_tokens
assert len(reduce_grids) == 7
@@ -1864,6 +1864,31 @@ def test_reset_pending_lazy_stores() -> None:
assert hit_tokens == 0, "CPU cache should be empty after reset"
@pytest.mark.parametrize("num_groups", [1, 2])
@pytest.mark.parametrize("lazy", [False, True])
def test_reset_releases_pending_cpu_hits(num_groups: int, lazy: bool) -> None:
"""A lookup awaiting GPU allocation must not prevent a cache reset."""
fix = make_scheduler(num_groups=num_groups, lazy=lazy)
sched = fix.scheduler
req = make_request(num_blocks=2)
cpu_blocks = []
for group_id in range(num_groups):
cpu_blocks.extend(
_allocate_gpu_blocks(sched.cpu_block_pool, req, 2, group_id=group_id)
)
sched.cpu_block_pool.free_blocks(cpu_blocks)
assert sched.get_num_new_matched_tokens(req, 0) == (2 * BLOCK_SIZE, True)
assert all(block.ref_cnt == 1 for block in cpu_blocks)
assert not sched._reqs_to_load
assert sched.reset()
assert all(block.ref_cnt == 0 for block in cpu_blocks)
assert not sched._pending_cpu_hits
assert sched.get_num_new_matched_tokens(req, 0) == (0, False)
assert sched.reset()
# ---------------------------------------------------------------------------
# Test 14: Reset with pending loads waits for completion
# ---------------------------------------------------------------------------
+61
View File
@@ -486,6 +486,67 @@ def test_preferred_block_size_rejects_backends_with_no_common_size():
Platform._preferred_block_size_for_backends(classes, 16, None)
@pytest.mark.parametrize("cpu", [False, True])
def test_alignment_rejected_by_a_sibling_backend_raises(monkeypatch, cpu):
# Alignment that lands on a size an exact-size sibling rejects must fail
# here, not later in select_common_block_size().
backends = [_mock_backend([MultipleOf(16)]), _mock_backend([16], exact=True)]
monkeypatch.setattr(
Platform, "_find_non_ssm_backends", classmethod(lambda cls, c: backends)
)
def bump(cls, vllm_config, backend_cls):
vllm_config.cache_config.block_size = 32
monkeypatch.setattr(Platform, "_align_hybrid_block_size", classmethod(bump))
cache_config = SimpleNamespace(
block_size=16, user_specified_block_size=True, kv_cache_dtype_skip_layers=None
)
vllm_config = SimpleNamespace(
cache_config=cache_config, model_config=SimpleNamespace(is_hybrid=True)
)
from vllm.platforms.cpu import CpuPlatform # overrides the method; must check too
platform = CpuPlatform if cpu else Platform
with pytest.raises(ValueError, match="MOCK_EXACT"):
platform.update_block_size_for_backend(vllm_config)
def test_xpu_gdn_rounding_rejected_by_a_sibling_backend_raises(monkeypatch):
# XPU's GDN rounding runs after super()'s check, so it must check again.
try:
from vllm.platforms.xpu import XPUPlatform
except ImportError:
pytest.skip("vllm_xpu_kernels not importable outside an XPU stack")
backends = [_mock_backend([MultipleOf(16)]), _mock_backend([16], exact=True)]
monkeypatch.setattr(
Platform, "update_block_size_for_backend", classmethod(lambda cls, c: None)
)
monkeypatch.setattr(
Platform, "_find_non_ssm_backends", classmethod(lambda cls, c: backends)
)
gdn_layer = SimpleNamespace(
get_attn_backend=lambda: SimpleNamespace(get_name=lambda: "GDN_ATTN")
)
monkeypatch.setattr(
"vllm.config.vllm.get_layers_from_vllm_config",
lambda vllm_config, layer_type: {"gdn": gdn_layer},
)
cache_config = SimpleNamespace(
block_size=16,
mamba_cache_mode="none",
mamba_page_size_padded=None,
user_specified_block_size=True,
kv_cache_dtype_skip_layers=None,
)
vllm_config = SimpleNamespace(
cache_config=cache_config, model_config=SimpleNamespace(is_hybrid=True)
)
with pytest.raises(ValueError, match="MOCK_EXACT"):
XPUPlatform.update_block_size_for_backend(vllm_config)
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
model = Mock()
model.get_mm_lora_token_counts.side_effect = (