Merge remote-tracking branch 'upstream/main' into patches
This commit is contained in:
@@ -423,3 +423,58 @@ def test_cumem_with_cudagraph():
|
||||
|
||||
# output content is as expected
|
||||
assert torch.allclose(y, x + 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
def test_cudagraph_pool_sleep(level):
|
||||
"""Routing, and the graph pool backed up at both sleep levels and restored
|
||||
in place by any wake."""
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
|
||||
from vllm.compilation.cudagraph_pool import (
|
||||
capture_outside_cumem_pool,
|
||||
capture_pool,
|
||||
)
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
|
||||
handle = current_platform.graph_pool_handle()
|
||||
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
|
||||
with ctx, capture_pool(handle, cfg) as used:
|
||||
assert used == handle and allocator.current_tag != "cudagraph"
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
|
||||
x = torch.ones_like(weight)
|
||||
|
||||
def capture() -> list:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
with (
|
||||
capture_pool(handle, on) as pool,
|
||||
torch.cuda.graph(graph, pool=pool, stream=stream),
|
||||
):
|
||||
assert nccl_alloc._graph_pool_id == pool != handle
|
||||
const = torch.empty_like(x) # Never written by replay: needs a backup.
|
||||
y = x * weight + const
|
||||
return [graph, y, const.fill_(3.0)]
|
||||
|
||||
def graph_pool() -> dict[int, int]:
|
||||
data = allocator.pointer_to_data.items()
|
||||
return {
|
||||
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
|
||||
}
|
||||
|
||||
held, backend = capture(), CuMemBackend()
|
||||
mapped = graph_pool()
|
||||
backend.suspend(level=level)
|
||||
backend.resume(tags=["kv_cache"])
|
||||
assert graph_pool() == mapped
|
||||
backend.resume(tags=["weights"])
|
||||
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
|
||||
held[0].replay()
|
||||
assert torch.equal(held[1], torch.full_like(x, 5.0))
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
|
||||
from ....utils import (
|
||||
create_new_process_for_each_test,
|
||||
multi_gpu_marks,
|
||||
multi_gpu_test,
|
||||
requires_fp8,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch):
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
|
||||
def _cudagraph_bytes(worker) -> int:
|
||||
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
|
||||
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
|
||||
|
||||
|
||||
def _custom_ar_active(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
ca_comm = get_tp_group().device_communicator.ca_comm
|
||||
return ca_comm is not None and not ca_comm.disabled
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "breakable", "tp", "offload"),
|
||||
[
|
||||
("FULL", False, 1, True),
|
||||
("PIECEWISE", False, 1, True),
|
||||
("PIECEWISE", True, 1, True),
|
||||
pytest.param(
|
||||
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
|
||||
),
|
||||
("FULL_AND_PIECEWISE", False, 1, False),
|
||||
],
|
||||
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
|
||||
unregistered custom AR; off changes nothing."""
|
||||
for name, value in [
|
||||
("VLLM_USE_V2_MODEL_RUNNER", "1"),
|
||||
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
|
||||
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
|
||||
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
|
||||
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
|
||||
]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
|
||||
llm = LLM(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
enable_sleep_mode=True,
|
||||
sleep_mode_offload_cudagraph=offload,
|
||||
tensor_parallel_size=tp,
|
||||
compilation_config={
|
||||
"cudagraph_mode": mode,
|
||||
"pass_config": {"fuse_allreduce_rms": False},
|
||||
},
|
||||
)
|
||||
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
|
||||
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
|
||||
assert all(graph_bytes) if offload else not any(graph_bytes)
|
||||
if not offload:
|
||||
return
|
||||
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("Custom allreduce is unavailable on these GPUs")
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
llm.sleep(level=level)
|
||||
llm.wake_up(tags=["weights"])
|
||||
if level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_async(gpu_memory_cleared):
|
||||
async def test():
|
||||
|
||||
@@ -376,7 +376,6 @@ def test_flashinfer_standalone_workspace_size(
|
||||
"_resolve_fi_ar_backend",
|
||||
Mock(return_value=("mnnvl", False)),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 2)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_get_tuned_standalone_max_size",
|
||||
@@ -389,6 +388,34 @@ def test_flashinfer_standalone_workspace_size(
|
||||
assert create_workspace.call_args.args[3] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("node_count", "is_blackwell", "expected"),
|
||||
[
|
||||
(1, False, ("trtllm", False)),
|
||||
(1, True, ("trtllm", False)),
|
||||
(2, False, (None, False)),
|
||||
(2, True, ("mnnvl", False)),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_auto_backend_uses_group_topology(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
node_count: int,
|
||||
is_blackwell: bool,
|
||||
expected: tuple[str | None, bool],
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto"
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce.current_platform,
|
||||
"has_device_capability",
|
||||
lambda capability: is_blackwell,
|
||||
)
|
||||
|
||||
assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected
|
||||
|
||||
|
||||
def test_flashinfer_workspace_failure_is_not_retried(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -402,7 +429,6 @@ def test_flashinfer_workspace_failure_is_not_retried(
|
||||
"_resolve_fi_ar_backend",
|
||||
Mock(return_value=("mnnvl", True)),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
|
||||
)
|
||||
|
||||
@@ -177,7 +177,9 @@ def _worker(rank: int, tp_size: int, port: int) -> None:
|
||||
# Supply the resolved Engram settings without constructing a full model.
|
||||
vllm_config.engram_config = EngramConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
architecture="DeepseekV41ForCausalLM", is_moe=True
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
try:
|
||||
init_distributed_environment()
|
||||
|
||||
@@ -248,7 +248,8 @@ async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_error_stream():
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream(include_usage: bool):
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
@@ -313,6 +314,67 @@ async def test_chat_error_stream():
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_chat.create_chat_completion(request)
|
||||
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_chat_error_stream_first_chunk(include_usage: bool):
|
||||
"""An error before the first token must reach the streaming client."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
completion_output = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_chat.create_chat_completion(request)
|
||||
|
||||
@@ -383,10 +383,14 @@ async def test_completion_suffix_supports_text_prompt_list():
|
||||
|
||||
assert result == [{"ok": True}]
|
||||
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
|
||||
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
|
||||
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
|
||||
(
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
|
||||
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}"
|
||||
),
|
||||
(
|
||||
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
|
||||
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -619,7 +623,8 @@ async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_error_stream():
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_completion_error_stream(include_usage: bool):
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
@@ -684,6 +689,67 @@ async def test_completion_error_stream():
|
||||
prompt="Test prompt",
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_completion.create_completion(request)
|
||||
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_usage", [False, True])
|
||||
async def test_completion_error_stream_first_chunk(include_usage: bool):
|
||||
"""An error before the first token must reach the streaming client."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
|
||||
completion_output = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="error",
|
||||
)
|
||||
|
||||
request_output = RequestOutput(
|
||||
request_id="test-id",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion_output],
|
||||
finished=True,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield request_output
|
||||
|
||||
mock_engine.generate = MagicMock(side_effect=mock_generate)
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
stream_options={"include_usage": include_usage},
|
||||
)
|
||||
|
||||
response = await serving_completion.create_completion(request)
|
||||
|
||||
@@ -67,7 +67,7 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser.harmony import Segment
|
||||
from vllm.parser.harmony import ChunkResult, HarmonyParser, Segment
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.online_renderer import (
|
||||
OnlineRenderer,
|
||||
@@ -2129,6 +2129,87 @@ async def test_stream_completed_response_reuses_streamed_items(monkeypatch):
|
||||
assert function_call.call_id == "chatcmpl-tool-parser-id"
|
||||
|
||||
|
||||
def _harmony_msg(channel: str, text: str, recipient: str | None = None):
|
||||
msg = OpenAIHarmonyMessage.from_role_and_content(Role.ASSISTANT, text)
|
||||
msg = msg.with_channel(channel)
|
||||
return msg.with_recipient(recipient) if recipient else msg
|
||||
|
||||
|
||||
def _harmony_segments(msg, *, streamed: bool = True) -> list[Segment]:
|
||||
"""Segments the parser yields for one message: a content delta (absent
|
||||
for a zero-delta message), then the completed message."""
|
||||
delta = [Segment(msg.channel, msg.recipient, msg.content[0].text)]
|
||||
completed = Segment(msg.channel, msg.recipient, "", completed_message=msg)
|
||||
return (delta if streamed else []) + [completed]
|
||||
|
||||
|
||||
async def _harmony_stream_events(segment_chunks: list[list[Segment]]):
|
||||
serving = _make_serving_instance()
|
||||
serving.use_harmony = True
|
||||
parser = MagicMock(spec=HarmonyParser)
|
||||
parser.process_chunk.side_effect = [
|
||||
ChunkResult(segments=segments, reasoning_token_count=0)
|
||||
for segments in segment_chunks
|
||||
]
|
||||
context = HarmonyContext([], [], frozenset({"get_weather"}), parser)
|
||||
|
||||
async def result_generator():
|
||||
for token_id in range(len(segment_chunks)):
|
||||
context.append_output(_make_request_output("", [token_id]))
|
||||
yield context
|
||||
|
||||
return [
|
||||
event
|
||||
async for event in serving.responses_stream_generator(
|
||||
request=ResponsesRequest(input="hi", stream=True, store=False),
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
result_generator=result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_harmony_stream_completed_response_reuses_streamed_ids():
|
||||
"""response.completed must carry the id and call_id each item was streamed
|
||||
with, not ones minted when the harmony messages are converted again."""
|
||||
messages = [
|
||||
_harmony_msg("analysis", "think"),
|
||||
_harmony_msg("commentary", '{"city": "Paris"}', "functions.get_weather"),
|
||||
_harmony_msg("final", "Sunny"),
|
||||
]
|
||||
|
||||
events = await _harmony_stream_events([_harmony_segments(msg) for msg in messages])
|
||||
|
||||
streamed = [e.item for e in events if e.type == "response.output_item.done"]
|
||||
output = events[-1].response.output
|
||||
assert [item.type for item in output] == ["reasoning", "function_call", "message"]
|
||||
assert [item.id for item in output] == [item.id for item in streamed]
|
||||
assert output[1].call_id == streamed[1].call_id
|
||||
assert output[2].content[0].text == "Sunny"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_harmony_stream_unstreamed_item_keeps_own_id():
|
||||
"""An item with no done event (zero-delta) must not take the streamed id
|
||||
of a later item of the same type."""
|
||||
unstreamed = _harmony_msg("analysis", "skipped")
|
||||
reasoning = _harmony_msg("analysis", "think")
|
||||
|
||||
events = await _harmony_stream_events(
|
||||
[_harmony_segments(unstreamed, streamed=False), _harmony_segments(reasoning)]
|
||||
)
|
||||
|
||||
(streamed,) = [e.item for e in events if e.type == "response.output_item.done"]
|
||||
first, second = events[-1].response.output
|
||||
assert first.content[0].text == "skipped"
|
||||
assert first.id != streamed.id
|
||||
assert second.id == streamed.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
|
||||
async def test_streaming_parallel_tool_calls(
|
||||
|
||||
@@ -456,11 +456,13 @@ async def test_stream_returns_spec_decode_metrics_on_empty_terminal_output(
|
||||
data_chunks = [chunk for chunk in _parse_sse_chunks(chunks) if chunk != "[DONE]"]
|
||||
|
||||
assert "metrics" not in data_chunks[0]
|
||||
assert len(data_chunks) == 1 + int(include_usage)
|
||||
assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
|
||||
assert "metrics" not in data_chunks[1]
|
||||
assert len(data_chunks) == 2 + int(include_usage)
|
||||
if include_usage:
|
||||
assert data_chunks[1]["choices"] == []
|
||||
assert data_chunks[2]["choices"] == []
|
||||
assert (
|
||||
data_chunks[1]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
|
||||
data_chunks[2]["metrics"]["speculative_decoding"]["num_draft_tokens"] == 3
|
||||
)
|
||||
|
||||
|
||||
@@ -823,30 +825,76 @@ async def test_stream_zero_token_completion_still_delivers_prompt_metadata():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_zero_token_completion_emits_no_chunk_by_default():
|
||||
@pytest.mark.parametrize("finish_reason", ["stop", "length", "abort"])
|
||||
async def test_stream_emits_terminal_output_without_new_tokens(finish_reason):
|
||||
"""The final output may carry no new tokens (always for an abort); the
|
||||
client must still see why the choice finished."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[], finish_reason="stop", finished=True
|
||||
"req-1", token_ids=[], finish_reason=finish_reason, finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
parsed = _parse_sse_chunks([chunk async for chunk in response])
|
||||
choices = [c["choices"][0] for c in parsed[:-1]]
|
||||
|
||||
assert not [c for c in parsed if isinstance(c, dict) and c.get("choices")]
|
||||
assert [c["token_ids"] for c in choices] == [[10], []]
|
||||
assert [c["finish_reason"] for c in choices] == [None, finish_reason]
|
||||
assert parsed[-1] == "[DONE]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_parallel_sampling_first_output_missing_a_choice():
|
||||
"""With n=2 the first output may carry only one choice; the other must
|
||||
still stream, and an abort finishes both."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], index=0)
|
||||
yield _make_request_output("req-1", token_ids=[11], index=1)
|
||||
aborted = _make_request_output(
|
||||
"req-1", token_ids=[], finish_reason="abort", finished=True
|
||||
)
|
||||
aborted.outputs += _make_request_output(
|
||||
"req-1", token_ids=[], finish_reason="abort", index=1
|
||||
).outputs
|
||||
yield aborted
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10, n=2),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=True),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
parsed = _parse_sse_chunks([chunk async for chunk in response])
|
||||
choices = [c["choices"][0] for c in parsed[:-2]]
|
||||
|
||||
assert [(c["index"], c["token_ids"], c["finish_reason"]) for c in choices] == [
|
||||
(0, [10], None),
|
||||
(1, [11], None),
|
||||
(0, [], "abort"),
|
||||
(1, [], "abort"),
|
||||
]
|
||||
assert parsed[-2]["usage"]["completion_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_error_mid_generation():
|
||||
"""finish_reason='error' mid-stream yields error chunk then [DONE]."""
|
||||
|
||||
@@ -710,21 +710,23 @@ async def test_text_mode_rejected_when_tokens_only(client):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
async def test_text_mode_stream_delivers_abort_finish_chunk(
|
||||
client, tokenizer, messages
|
||||
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
|
||||
async def test_stream_delivers_abort_finish_chunk(
|
||||
client, tokenizer, messages, output_mode
|
||||
):
|
||||
"""The final output after an abort has no new token IDs, so it
|
||||
reaches a text stream only because text mode emits finish-only chunks."""
|
||||
"""The final output after an abort has no new token IDs, but the
|
||||
stream must still deliver its finish reason."""
|
||||
payload = {
|
||||
**_text_mode_payload(
|
||||
_chat_prompt_token_ids(tokenizer, messages),
|
||||
max_tokens=900,
|
||||
ignore_eos=True,
|
||||
),
|
||||
"output_mode": output_mode,
|
||||
"stream": True,
|
||||
"request_id": "text-abort-e2e",
|
||||
"request_id": f"{output_mode}-abort-e2e",
|
||||
}
|
||||
|
||||
chunks = []
|
||||
@@ -736,7 +738,7 @@ async def test_text_mode_stream_delivers_abort_finish_chunk(
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
chunks.append(chunk)
|
||||
if not aborted and chunk["choices"] and chunk["choices"][0]["text"]:
|
||||
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
|
||||
abort = await client.post(
|
||||
ABORT_ENDPOINT,
|
||||
json={"request_ids": [chunk["request_id"]]},
|
||||
@@ -745,7 +747,7 @@ async def test_text_mode_stream_delivers_abort_finish_chunk(
|
||||
aborted = True
|
||||
|
||||
assert aborted
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
|
||||
final_choice = chunks[-1]["choices"][0]
|
||||
assert final_choice["finish_reason"] == "abort"
|
||||
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
|
||||
|
||||
@@ -2,11 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from prometheus_client.multiprocess import MultiProcessCollector
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
@@ -21,10 +18,7 @@ from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
# The API servers accept from one shared listening socket, so the kernel may
|
||||
# hand one process most connections. Sync until both have recorded, within a cap.
|
||||
MIN_SYNCS = 8
|
||||
MAX_SYNCS = 64
|
||||
SYNCS = 8
|
||||
|
||||
|
||||
def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||
@@ -37,17 +31,6 @@ def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||
}
|
||||
|
||||
|
||||
def pids_that_recorded(multiproc_dir: Path) -> set[str]:
|
||||
return {
|
||||
path.stem.rsplit("_", 1)[1]
|
||||
for path in multiproc_dir.glob("histogram_*.db")
|
||||
if any(
|
||||
metric.name == DURATION
|
||||
for metric in MultiProcessCollector.merge([str(path)])
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
||||
args = [
|
||||
"--load-format",
|
||||
@@ -77,17 +60,11 @@ def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer),
|
||||
)
|
||||
syncs = 0
|
||||
while syncs < MIN_SYNCS or (
|
||||
len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS
|
||||
):
|
||||
for _ in range(SYNCS):
|
||||
engine.send_weights()
|
||||
syncs += 1
|
||||
metrics = scrape(url)
|
||||
|
||||
calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs}
|
||||
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
|
||||
for operation, count in calls.items():
|
||||
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
||||
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
||||
# The counts above are sums over processes only if both servers recorded.
|
||||
assert len(pids_that_recorded(tmp_path)) == 2
|
||||
|
||||
@@ -5,6 +5,7 @@ import math
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -323,6 +324,63 @@ def test_contexted_kv_attention(
|
||||
torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend")
|
||||
@pytest.mark.parametrize("sliding_window", [None, 1, 4])
|
||||
@pytest.mark.parametrize("query_len", [1, 3])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_backend_sliding_window_includes_boundary_token(
|
||||
sliding_window: int | None, query_len: int
|
||||
) -> None:
|
||||
"""A window of W must include the current token and W - 1 preceding tokens."""
|
||||
from vllm.v1.attention.backends.rocm_attn import RocmAttentionImpl
|
||||
from vllm.v1.attention.ops.paged_attn import PagedAttention
|
||||
|
||||
device, dtype = "cuda:0", torch.float16
|
||||
num_heads, head_size, seq_len = 4, 128, 8
|
||||
query = torch.zeros(query_len, num_heads, head_size, dtype=dtype, device=device)
|
||||
key = torch.zeros(seq_len, 1, head_size, dtype=dtype, device=device)
|
||||
values = torch.arange(1, seq_len + 1, dtype=dtype, device=device)
|
||||
value = values[:, None, None].expand_as(key)
|
||||
kv_cache = torch.zeros(2, 1, 16, head_size, dtype=dtype, device=device)
|
||||
_, value_cache = PagedAttention.split_kv_cache(kv_cache, 1, head_size)
|
||||
value_cache[..., :seq_len] = value.permute(1, 2, 0)
|
||||
scale = torch.tensor(1.0, device=device)
|
||||
metadata = SimpleNamespace(
|
||||
use_cascade=False,
|
||||
num_actual_tokens=query_len,
|
||||
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([seq_len], dtype=torch.int32, device=device),
|
||||
max_query_len=query_len,
|
||||
max_seq_len=seq_len,
|
||||
block_table=torch.zeros(1, 1, dtype=torch.int32, device=device),
|
||||
causal=True,
|
||||
)
|
||||
impl = RocmAttentionImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
scale=head_size**-0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
)
|
||||
output = torch.empty_like(query)
|
||||
impl.forward(
|
||||
SimpleNamespace(_k_scale=scale, _v_scale=scale),
|
||||
query,
|
||||
key[-query_len:],
|
||||
value[-query_len:],
|
||||
kv_cache.transpose(0, 1),
|
||||
metadata,
|
||||
output,
|
||||
)
|
||||
mask = create_causal_attention_mask_for_sdpa(
|
||||
[query_len], [seq_len], sliding_window or 0, device=device, dtype=dtype
|
||||
)
|
||||
expected = (mask.float().softmax(dim=-1) @ values.float())[:, None, None]
|
||||
torch.testing.assert_close(output, expected.to(dtype).expand_as(output))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
|
||||
@@ -0,0 +1,585 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Determinism + FP32-reference correctness tests for the RDNA3 W4A16 GEMM.
|
||||
|
||||
The split-K epilogues of ``gptq_gemm_rdna3`` (scalar path) and the WMMA
|
||||
path previously accumulated per-split low-precision partials directly into
|
||||
the output via CAS atomics. With multiple writers per output element the
|
||||
accumulation order — and therefore the rounded result — varied per call:
|
||||
repeated fixed-input calls produced a different output almost every time on
|
||||
gfx1100 once more than a few split blocks contended.
|
||||
|
||||
Both paths now store FP32 split partials and reduce them in a fixed order
|
||||
with a single final low-precision rounding, which is deterministic by
|
||||
construction. When the split count is 1 every path stores directly
|
||||
(single writer per output element).
|
||||
|
||||
These tests assert, through the public op path:
|
||||
|
||||
A. Repeatability — ``torch.equal`` across repeated identical calls.
|
||||
B. Numerical correctness — against an FP32 dequantized reference.
|
||||
|
||||
(B) exists because repeatability alone cannot see the failure mode the
|
||||
deterministic epilogue introduces: the FP32 partials scratch is
|
||||
``at::empty`` (its coverage invariant is documented at
|
||||
``alloc_wmma_partials`` in ``csrc/rocm/q_gemm_rdna3_wmma.cu``), and across
|
||||
repeated calls the caching allocator hands back the same block — a slot
|
||||
that ever went unwritten would be bitwise-repeatable *and* wrong.
|
||||
|
||||
Reference semantics (uint4b8, GPTQv1, synthesized zero points):
|
||||
W[k, n] = (q[k, n] - 8) * float(round_to_dtype(scale[g, n])), g = k//G
|
||||
ref[m, n] = x[m, :] @ W[:, n] (FP32 matmul)
|
||||
|
||||
Tolerances are derived from the kernel's rounding structure, not tuned to
|
||||
pass (measured max errors on W7900/gfx1100 in parentheses):
|
||||
|
||||
* scalar bf16 — dequant keeps FP32 precision end-to-end (magic-value
|
||||
v_dot2 path); the only rounding is the final cast. Measured max ~0.06
|
||||
on the tested shapes (|ref| up to ~30); tolerance 0.10 with headroom
|
||||
for larger |ref| at the final-cast ulp.
|
||||
* scalar fp16 — the dequant uses the exact factored form (magic values in
|
||||
fp16, (y, z) correction in fp32; see qdq_4_rdna3.cuh), so the only fp16
|
||||
roundings are the final output cast and the B-tile-free dot inputs:
|
||||
max_abs <= 0.5 * ulp_fp16(max|ref|) * 1.5 + 1e-3 (measured ~0.015 at
|
||||
|ref|~37; the pre-fix baked-constant form measured ~1.0-1.4 on the same
|
||||
shapes — 94-200x worse).
|
||||
* WMMA bf16 — B is narrowed to bf16 per cell (one rounding,
|
||||
<= 0.5*ulp_bf16(|B|)), accumulated in FP32; measured max ≈ 0.09 at
|
||||
K=4096 — tolerance 0.25.
|
||||
* WMMA fp16 — B narrowed to fp16 via the precise sub-then-mul variant
|
||||
(one rounding, ~1e-4); measured max ≈ 0.015 — tolerance 0.05.
|
||||
|
||||
All four are orders of magnitude below what an unwritten scratch slot
|
||||
(caching-allocator garbage / NaN) or an indexing bug (errors of the same
|
||||
scale as |ref| ≈ 7) would produce.
|
||||
|
||||
Test hygiene: the op path needs a single-process distributed group and a
|
||||
current VLLM config; tests use the repo-standard ``dist_init`` fixture from
|
||||
``tests/conftest.py`` (real context manager, temp-file rendezvous so no
|
||||
hardcoded MASTER_PORT can collide in parallel CI, and the suite's standard
|
||||
post-test cleanup).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.model_executor.parameter import ( # noqa: E402
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
from vllm.scalar_type import scalar_types # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
WEIGHT_TYPE = scalar_types.uint4b8
|
||||
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not (
|
||||
on_gfx1100()
|
||||
and hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
|
||||
),
|
||||
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
|
||||
)
|
||||
|
||||
REPEATS = 20
|
||||
GROUP = 128
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
|
||||
# max_abs_error tolerances vs the FP32 reference, per (path, dtype).
|
||||
# Derivation in the module docstring; ~2-3x headroom over measured maxima.
|
||||
TOL = {
|
||||
("scalar", torch.bfloat16): 0.10,
|
||||
("scalar", torch.float16): 0.025,
|
||||
("wmma", torch.bfloat16): 0.25,
|
||||
("wmma", torch.float16): 0.05,
|
||||
}
|
||||
|
||||
|
||||
def _build_layer(k, n, seed, dtype, group=GROUP):
|
||||
torch.manual_seed(seed)
|
||||
q_int4_kn = torch.randint(0, 16, (k, n), dtype=torch.int32)
|
||||
scales_gn = (torch.randn(k // group, n) * 0.01 + 0.02).to(dtype)
|
||||
qweight = pack_quantized_values_into_int32(q_int4_kn, WEIGHT_TYPE, packed_dim=0)
|
||||
no_loader = lambda *a, **kw: None # noqa: E731
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"qweight",
|
||||
PackedvLLMParameter(
|
||||
data=qweight,
|
||||
weight_loader=no_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=8,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"scales",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_gn, weight_loader=no_loader, input_dim=0, output_dim=1
|
||||
),
|
||||
)
|
||||
layer.to(device)
|
||||
return layer, q_int4_kn, scales_gn
|
||||
|
||||
|
||||
def _prepare(layer, dtype, k, n, group=GROUP):
|
||||
cfg = MPLinearLayerConfig(
|
||||
full_weight_shape=(k, n),
|
||||
partition_weight_shape=(k, n),
|
||||
weight_type=WEIGHT_TYPE,
|
||||
act_type=dtype,
|
||||
group_size=group,
|
||||
zero_points=False,
|
||||
)
|
||||
kernel = RDNA3W4A16LinearKernel(
|
||||
cfg, w_q_param_name="qweight", w_s_param_name="scales", w_zp_param_name=None
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
return kernel._get_weight_params(layer)
|
||||
|
||||
|
||||
def _reference(m, k, dtype, seed, q_int4_kn, scales_gn, group=GROUP):
|
||||
torch.manual_seed(seed + 1)
|
||||
x = torch.randn(m, k, device=device, dtype=dtype)
|
||||
# FP32 reference; see module docstring for the dequant semantics.
|
||||
w_f32 = (q_int4_kn.to(device).float() - 8.0) * scales_gn.to(
|
||||
device
|
||||
).repeat_interleave(group, dim=0).float()
|
||||
return x, x.float() @ w_f32
|
||||
|
||||
|
||||
def _run_op(x, w_q, w_zp, w_s):
|
||||
return torch.ops._rocm_C.gptq_gemm_rdna3(x, w_q, w_zp, w_s, False)
|
||||
|
||||
|
||||
def _outputs_and_ref(M, K, N, seed, dtype, repeats=1, group=GROUP):
|
||||
layer, q_int4_kn, scales_gn = _build_layer(K, N, seed, dtype, group)
|
||||
w_q, w_s, w_zp = _prepare(layer, dtype, K, N, group)
|
||||
x, ref = _reference(M, K, dtype, seed, q_int4_kn, scales_gn, group)
|
||||
outs = [_run_op(x, w_q, w_zp, w_s) for _ in range(repeats)]
|
||||
return outs, ref
|
||||
|
||||
|
||||
def _path_for(dtype, m):
|
||||
"""Public dispatch: both dtypes reach WMMA at M>=12 (see the
|
||||
WMMA_MIN_M crossover table in csrc/rocm/q_gemm_rdna3.cu)."""
|
||||
if m >= 12:
|
||||
return "wmma"
|
||||
return "scalar"
|
||||
|
||||
|
||||
def _assert_repeatable(outs):
|
||||
for o in outs[1:]:
|
||||
assert torch.equal(o, outs[0]), (
|
||||
"RDNA3 W4A16 split-K produced differing outputs for identical inputs"
|
||||
)
|
||||
|
||||
|
||||
def _assert_close_to_ref(out, ref, path, dtype):
|
||||
err = (out.float() - ref).abs()
|
||||
max_abs = err.max().item()
|
||||
mean_abs = err.mean().item()
|
||||
tol = TOL[(path, dtype)]
|
||||
assert torch.isfinite(out.float()).all(), "non-finite output"
|
||||
assert max_abs <= tol, (
|
||||
f"RDNA3 W4A16 {path} output deviates from the FP32 dequantized "
|
||||
f"reference: max_abs={max_abs:.4f} > tol={tol} (mean_abs="
|
||||
f"{mean_abs:.4f}). This catches values that are bitwise-repeatable "
|
||||
"but wrong (e.g. an unwritten FP32 scratch slot)."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# A. Repeatability (bit-exact across identical calls)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_splitk_bit_repeatable(dist_init, dtype):
|
||||
"""Scalar path at production-like contention: K=4096 splits into
|
||||
ceil(4096/256) = 16 concurrent writers per output element (the old CAS
|
||||
bug's onset was between 2 and 4 writers, so this sits well inside the
|
||||
failure regime rather than at the empirical threshold)."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=1, K=4096, N=4096, seed=1234, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_small_m_bit_repeatable(dist_init, dtype):
|
||||
"""Scalar M>1 tiles share the same deterministic epilogue."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=8, K=4096, N=4096, seed=1235, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_single_split_bit_repeatable(dist_init, dtype):
|
||||
"""z_count == 1 (K=256 < BLOCK_KN_SIZE): direct-store epilogue, no
|
||||
scratch, no reduce pass."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=1, K=256, N=4096, seed=1238, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_splitk_bit_repeatable(dist_init, dtype):
|
||||
"""WMMA path (both dtypes M >= 12), split-K active.
|
||||
|
||||
K=6656 gives K_SPLIT=4 under the upstream heuristic; the deterministic
|
||||
epilogue must be bit-repeatable regardless of the split count.
|
||||
"""
|
||||
m = 12 # smallest M on the WMMA side of the dispatch boundary
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=m, K=6656, N=4096, seed=1236, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_large_m_bit_repeatable(dist_init, dtype):
|
||||
"""Large-M WMMA tiles (128x64 kernels) with the deterministic epilogue."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=128, K=6656, N=4096, seed=1237, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# B. Numerical correctness against the FP32 dequantized reference
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_matches_fp32_reference(dist_init, dtype):
|
||||
"""Scalar split-K (K=4096, 16 writers) vs the FP32 reference."""
|
||||
m = 1
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1240, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("m", [2, 3, 4, 5, 8])
|
||||
def test_scalar_multi_m_matches_fp32_reference(dist_init, m, dtype):
|
||||
"""The exact factored dequant across every M_COUNT instantiation
|
||||
(M=2,3 -> M_COUNT=2; M=4,5 -> M_COUNT=4 with a padded row; M=8 -> 8):
|
||||
the (y, z) correction and the nibble-slot activation sums must hold
|
||||
for every tile width, not just the M=1 fast path."""
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1247, dtype=dtype)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_single_split_matches_fp32_reference(dist_init, dtype):
|
||||
"""Scalar z_count == 1 direct-store path vs the FP32 reference."""
|
||||
m = 1
|
||||
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1241, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_matches_fp32_reference(dist_init, dtype):
|
||||
"""WMMA split-K path (16x16_1w at the M=12 boundary) vs FP32 reference."""
|
||||
m = 12 # smallest M on the WMMA side of the dispatch boundary
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1242, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_single_split_matches_fp32_reference(dist_init, dtype):
|
||||
"""WMMA k_split == 1 direct-store path (K=256 -> compute_wmma_k_split
|
||||
returns 1 for every dispatch level) vs the FP32 reference."""
|
||||
m = 12 # smallest M on the WMMA side of the dispatch boundary
|
||||
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1243, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_large_m_matches_fp32_reference(dist_init, dtype):
|
||||
"""Large-M WMMA (128x64_k32, row-tiled scratch + reduce) vs FP32."""
|
||||
m = 128
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1244, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C. V7/V8 (128x64) no-split path: k_split == 1 -> direct store, no scratch,
|
||||
# no reduce pass (partials == nullptr).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The measured production-like shape from the W7900 benchmark campaign.
|
||||
# Routing arithmetic (see _assert_v7v8_no_split_routing): with (128, 64)
|
||||
# tiles, blocks_xy = ceil(25600/64) * ceil(512/128) = 400 * 4 = 1600 >=
|
||||
# 1500, so compute_wmma_k_split_mn returns 1. Smaller no-split shapes are
|
||||
# not practical: N would have to grow past ~24k at M=512 (or M past ~1k at
|
||||
# N=12k) to reach the 1500-block threshold, so this is effectively the
|
||||
# smallest production-realistic V7/V8 no-split shape.
|
||||
V7V8_NOSPLIT = (512, 25600, 6656) # (M, N, K)
|
||||
|
||||
|
||||
def _compute_wmma_k_split_mn(m, n, k, m_tile, n_tile):
|
||||
"""Faithful replica of compute_wmma_k_split_mn (C++ heuristic)."""
|
||||
blocks_xy = ((n + n_tile - 1) // n_tile) * ((m + m_tile - 1) // m_tile)
|
||||
if blocks_xy >= 1500:
|
||||
return 1
|
||||
if blocks_xy * 2 >= 1500 and k >= 512 and k % 32 == 0:
|
||||
return 2
|
||||
if blocks_xy * 4 >= 1500 and k >= 1024 and k % 64 == 0:
|
||||
return 4
|
||||
if k >= 1024 and k % 64 == 0:
|
||||
return 4
|
||||
if k >= 512 and k % 32 == 0:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _assert_v7v8_no_split_routing(dtype, m, n, k):
|
||||
"""Assert the shape deterministically routes to the V7/V8 128x64
|
||||
kernel with k_split == 1: WMMA dispatch (both dtypes M >= 12), the
|
||||
M >= 128 branch, and the no-split threshold."""
|
||||
assert m >= 12, "not the WMMA path"
|
||||
assert m >= 128, "below the V7/V8 (128x64) branch"
|
||||
assert _compute_wmma_k_split_mn(m, n, k, 128, 64) == 1, (
|
||||
"shape does not route to the k_split == 1 no-split path"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_v7v8_no_split_bit_repeatable(dist_init, dtype):
|
||||
"""V7/V8 128x64_k32 with k_split == 1: single direct writer per output
|
||||
cell — no FP32 scratch, no reduce pass."""
|
||||
m, n, k = V7V8_NOSPLIT
|
||||
_assert_v7v8_no_split_routing(dtype, m, n, k)
|
||||
outs, _ = _outputs_and_ref(M=m, K=k, N=n, seed=1245, dtype=dtype, repeats=REPEATS)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_v7v8_no_split_matches_fp32_reference(dist_init, dtype):
|
||||
"""V7/V8 128x64_k32 no-split direct-store path vs the FP32 reference."""
|
||||
m, n, k = V7V8_NOSPLIT
|
||||
_assert_v7v8_no_split_routing(dtype, m, n, k)
|
||||
outs, ref = _outputs_and_ref(M=m, K=k, N=n, seed=1246, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# D. Dispatch boundary: M=11 must stay scalar, M=12 must route to WMMA
|
||||
# (WMMA_MIN_M in csrc/rocm/q_gemm_rdna3.cu). Observed via the CUDA
|
||||
# profiler: the scalar and WMMA kernels have disjoint names.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _launched_kernel_names(fn) -> set:
|
||||
from torch.profiler import ProfilerActivity, profile
|
||||
|
||||
fn()
|
||||
torch.accelerator.synchronize()
|
||||
with profile(activities=[ProfilerActivity.CUDA]) as prof:
|
||||
fn()
|
||||
torch.accelerator.synchronize()
|
||||
return {
|
||||
e.name
|
||||
for e in prof.events()
|
||||
if "gemm_q4" in e.name or "reduce_partials" in e.name
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@gfx1100_only
|
||||
def test_dispatch_boundary(dist_init, dtype):
|
||||
"""The scalar/WMMA crossover sits between M=11 and M=12 for both
|
||||
dtypes; the boundary must be exact (see the WMMA_MIN_M table)."""
|
||||
|
||||
def run(m):
|
||||
outs, _ = _outputs_and_ref(M=m, K=4096, N=512, seed=1248, dtype=dtype)
|
||||
return outs[0]
|
||||
|
||||
m11 = run(11)
|
||||
names11 = _launched_kernel_names(lambda: run(11))
|
||||
assert any("gemm_q4_kernel_rdna3" in n for n in names11), names11
|
||||
assert not any("wmma" in n for n in names11), names11
|
||||
|
||||
m12 = run(12)
|
||||
names12 = _launched_kernel_names(lambda: run(12))
|
||||
assert any("wmma" in n for n in names12), names12
|
||||
assert not any("gemm_q4_kernel_rdna3" in n for n in names12), names12
|
||||
|
||||
# Both sides of the boundary stay correct against the reference.
|
||||
_, ref11 = _outputs_and_ref(M=11, K=4096, N=512, seed=1248, dtype=dtype)
|
||||
_, ref12 = _outputs_and_ref(M=12, K=4096, N=512, seed=1248, dtype=dtype)
|
||||
_assert_close_to_ref(m11, ref11, "scalar", dtype)
|
||||
_assert_close_to_ref(m12, ref12, "wmma", dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E. In-kernel split-K reduction: stress the per-call ticket epilogue.
|
||||
# The fused reduction must stay deterministic and correct under
|
||||
# back-to-back unsynchronized calls, concurrent invocations on two
|
||||
# streams (per-call tickets must never share state), and CUDA-graph
|
||||
# capture/replay.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_splitk_100x_bit_repeatable(dist_init, dtype):
|
||||
"""100 back-to-back calls (no intermediate sync) on the split-K path."""
|
||||
outs, _ = _outputs_and_ref(M=1, K=4096, N=512, seed=1249, dtype=dtype, repeats=100)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_multistream_concurrent(dist_init, dtype):
|
||||
"""Two concurrent invocations on separate streams, distinct weights and
|
||||
activations: outputs must be bit-identical to the serial invocation of
|
||||
each (the arrival tickets are per-call, so cross-stream interference
|
||||
would show up here as a corrupted or unreduced output)."""
|
||||
layer_a, q_a, s_a = _build_layer(4096, 512, 4321, dtype)
|
||||
layer_b, q_b, s_b = _build_layer(4096, 512, 4322, dtype)
|
||||
wq_a, ws_a, wz_a = _prepare(layer_a, dtype, 4096, 512)
|
||||
wq_b, ws_b, wz_b = _prepare(layer_b, dtype, 4096, 512)
|
||||
torch.manual_seed(4323)
|
||||
x_a = torch.randn(1, 4096, device=device, dtype=dtype)
|
||||
x_b = torch.randn(1, 4096, device=device, dtype=dtype)
|
||||
|
||||
serial_a = _run_op(x_a, wq_a, wz_a, ws_a)
|
||||
serial_b = _run_op(x_b, wq_b, wz_b, ws_b)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
s1, s2 = torch.cuda.Stream(), torch.cuda.Stream()
|
||||
for _ in range(25):
|
||||
with torch.cuda.stream(s1):
|
||||
out_a = _run_op(x_a, wq_a, wz_a, ws_a)
|
||||
with torch.cuda.stream(s2):
|
||||
out_b = _run_op(x_b, wq_b, wz_b, ws_b)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(out_a, serial_a), "concurrent stream A corrupted"
|
||||
assert torch.equal(out_b, serial_b), "concurrent stream B corrupted"
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_graph_capture_replay(dist_init, dtype):
|
||||
"""The op must survive CUDA-graph capture: the replayed output must be
|
||||
bit-identical to the eager output and stable across repeated replays."""
|
||||
layer, q_int4, scales_gn = _build_layer(4096, 512, 4324, dtype)
|
||||
wq, ws, wz = _prepare(layer, dtype, 4096, 512)
|
||||
torch.manual_seed(4325)
|
||||
x = torch.randn(1, 4096, device=device, dtype=dtype)
|
||||
|
||||
eager = _run_op(x, wq, wz, ws)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
_run_op(x, wq, wz, ws) # warm-up side streams/allocator on capture path
|
||||
torch.accelerator.synchronize()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = _run_op(x, wq, wz, ws)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(captured, eager), "graph replay differs from eager"
|
||||
graph.replay()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(captured, eager), "repeated replay not repeatable"
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_multi_row_tile_bit_repeatable(dist_init, dtype):
|
||||
"""Row tiling with ticket reuse: N % 16 == 8 bypasses the WMMA
|
||||
dispatch (N % 16 != 0), so M=100 runs scalar across two 64-row tiles
|
||||
(the second with gy > 1 y-blocks). Every tile's tickets must be armed,
|
||||
consumed, and re-armed correctly for all outputs to be written exactly
|
||||
once — and the result must be bit-repeatable."""
|
||||
outs, ref = _outputs_and_ref(
|
||||
M=100, K=4096, N=2056, seed=1250, dtype=dtype, repeats=20
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F. Split-K geometry corners: clipped non-first z blocks, small split
|
||||
# counts, and non-128 group sizes.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("k,group", [(4224, 128), (2112, 32)])
|
||||
def test_scalar_clipped_tail_splitk(dist_init, dtype, k, group):
|
||||
"""K % 256 != 0 with z_count > 1: the LAST z block is clipped by end_k,
|
||||
its group prefetch guard (`k + 32 < end_k`) is what keeps the raw
|
||||
zero/scale loads in bounds, and the reduce tail loop runs (z=17 -> one
|
||||
remainder slice; z=9 -> one batch + one). K=2112/G=32 also puts two
|
||||
group switches inside the clipped tail block."""
|
||||
outs, ref = _outputs_and_ref(M=1, K=k, N=512, seed=1251, dtype=dtype, group=group)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("group", [32, 64])
|
||||
def test_scalar_small_group_matches_fp32_reference(dist_init, dtype, group):
|
||||
"""Group sizes below 128 make the group constants change every 32-K
|
||||
round (G=32) or three times per z block (G=64); the factored-dequant
|
||||
correction must track every change. G=32 is also the production MoE
|
||||
group size."""
|
||||
outs, ref = _outputs_and_ref(
|
||||
M=1, K=2048, N=512, seed=1252, dtype=dtype, group=group
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_two_way_split_bit_repeatable(dist_init, dtype):
|
||||
"""z_count == 2 (K=512): the minimal split-K configuration — the old
|
||||
CAS bug's onset regime — and the pure-remainder reduce tail."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=1, K=512, N=512, seed=1253, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
@@ -711,7 +711,8 @@ def test_v41_rope_insert_plain_row(compress_ratio: int, store_fp8: bool):
|
||||
def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
"""The ring group's generic slot mapping is disabled (all PAD), so the
|
||||
builder must map every real token to ``ring_block * capacity + pos %
|
||||
capacity`` and keep padding tokens at PAD."""
|
||||
capacity``. Padding tokens and requests on the null block (block 0, as in
|
||||
dummy batches) must stay at PAD."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
|
||||
@@ -731,29 +732,30 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
device = torch.device("cuda")
|
||||
builder = CompressorMetadataBuilder(spec, ["state"], vllm_config, device)
|
||||
|
||||
# Two requests: 3 tokens at positions 13..15 on ring block 5, then 2
|
||||
# tokens at positions 7..8 on ring block 2; three padding tokens.
|
||||
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
positions = torch.tensor([13, 14, 15, 7, 8, 0, 0, 0], device=device)
|
||||
block_table = torch.tensor([[5], [2]], dtype=torch.int32, device=device)
|
||||
# Three requests: 3 tokens at positions 13..15 on ring block 5, 2 tokens
|
||||
# at positions 7..8 on ring block 2, then 2 tokens on the null block;
|
||||
# three padding tokens.
|
||||
query_start_loc = torch.tensor([0, 3, 5, 7], dtype=torch.int32, device=device)
|
||||
positions = torch.tensor([13, 14, 15, 7, 8, 0, 1, 0, 0, 0], device=device)
|
||||
block_table = torch.tensor([[5], [2], [0]], dtype=torch.int32, device=device)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([16, 9], dtype=torch.int32, device=device),
|
||||
num_reqs=2,
|
||||
num_actual_tokens=5,
|
||||
seq_lens=torch.tensor([16, 9, 2], dtype=torch.int32, device=device),
|
||||
num_reqs=3,
|
||||
num_actual_tokens=7,
|
||||
max_query_len=3,
|
||||
max_seq_len=16,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
|
||||
slot_mapping=torch.full((10,), -1, dtype=torch.int64, device=device),
|
||||
positions=positions,
|
||||
)
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
expected = [5 * 8 + 5, 5 * 8 + 6, 5 * 8 + 7, 2 * 8 + 7, 2 * 8 + 0, -1, -1, -1]
|
||||
assert metadata.slot_mapping.tolist() == expected
|
||||
ring = [5 * 8 + 5, 5 * 8 + 6, 5 * 8 + 7, 2 * 8 + 7, 2 * 8 + 0]
|
||||
assert metadata.slot_mapping.tolist() == ring + [-1] * 5
|
||||
assert metadata.query_start_loc is query_start_loc
|
||||
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1]
|
||||
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptNvFp4Config,
|
||||
@@ -573,6 +574,24 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_respects_inc_layer_config() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = INCConfig(
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
block_name_to_quantize="model.layers",
|
||||
extra_config={".*ple.*": {"bits": 16, "data_type": "float"}},
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.extra_config = None
|
||||
with pytest.raises(NotImplementedError, match="INCConfig"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
@@ -380,12 +380,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Llama4ForCausalLM": _HfExamplesInfo(
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
),
|
||||
"LongcatFlashForCausalLM": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
|
||||
),
|
||||
"LongcatFlashForCausalLM": _HfExamplesInfo("meituan-longcat/LongCat-Flash-Chat"),
|
||||
"LongcatFlashNgramForCausalLM": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Lite",
|
||||
trust_remote_code=True,
|
||||
# Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size)
|
||||
# so the dummy-weight init test fits in CI memory.
|
||||
hf_overrides={"ngram_vocab_size_ratio": 1},
|
||||
@@ -1752,7 +1749,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
),
|
||||
"LongCatFlashMTPModel": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat",
|
||||
trust_remote_code=True,
|
||||
speculative_model="meituan-longcat/LongCat-Flash-Chat",
|
||||
),
|
||||
"MiMoMTPModel": _HfExamplesInfo(
|
||||
|
||||
@@ -216,17 +216,19 @@ class TestNormalizeAudio:
|
||||
assert result.ndim == 1
|
||||
torch.testing.assert_close(result, torch.tensor([0.0, 1.0]))
|
||||
|
||||
def test_mono_passthrough_for_1d_numpy(self):
|
||||
"""1D numpy array should pass through unchanged with mono spec."""
|
||||
mono = np.array([1.0, 2.0, 3.0], dtype=np.float32)
|
||||
result = normalize_audio(mono, MONO_AUDIO_SPEC)
|
||||
@pytest.mark.parametrize("shape", [(3,), (1, 3), (3, 1), (1, 1)])
|
||||
def test_mono_passthrough_for_numpy(self, shape):
|
||||
"""Mono output is 1D, including audio with an explicit channel axis."""
|
||||
mono = np.arange(max(shape), dtype=np.float32)
|
||||
result = normalize_audio(mono.reshape(shape), MONO_AUDIO_SPEC)
|
||||
assert result.ndim == 1
|
||||
np.testing.assert_array_equal(result, mono)
|
||||
|
||||
def test_mono_passthrough_for_1d_torch(self):
|
||||
"""1D torch tensor should pass through unchanged with mono spec."""
|
||||
mono = torch.tensor([1.0, 2.0, 3.0])
|
||||
result = normalize_audio(mono, MONO_AUDIO_SPEC)
|
||||
@pytest.mark.parametrize("shape", [(3,), (1, 3), (3, 1), (1, 1)])
|
||||
def test_mono_passthrough_for_torch(self, shape):
|
||||
"""Mono output is 1D, including audio with an explicit channel axis."""
|
||||
mono = torch.arange(max(shape), dtype=torch.float32)
|
||||
result = normalize_audio(mono.reshape(shape), MONO_AUDIO_SPEC)
|
||||
assert result.ndim == 1
|
||||
torch.testing.assert_close(result, mono)
|
||||
|
||||
@@ -410,8 +412,9 @@ class TestMultiModalDataParserChannelNormalization:
|
||||
# When target_channels=None, stereo audio should be preserved
|
||||
assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D"
|
||||
|
||||
def test_parser_mono_passthrough_when_target_channels_1(self):
|
||||
"""Parser should pass through mono audio unchanged when target_channels=1."""
|
||||
@pytest.mark.parametrize("shape", [(16000,), (1, 16000), (16000, 1)])
|
||||
def test_parser_mono_passthrough_when_target_channels_1(self, shape):
|
||||
"""Mono layouts must preserve samples and report the waveform length."""
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
|
||||
# Create parser with mono normalization enabled
|
||||
@@ -420,17 +423,20 @@ class TestMultiModalDataParserChannelNormalization:
|
||||
target_channels=1,
|
||||
)
|
||||
|
||||
# Create mono audio (already 1D)
|
||||
mono_audio = np.random.randn(16000).astype(np.float32)
|
||||
mono_audio = np.linspace(-1.0, 1.0, 16000, dtype=np.float32)
|
||||
|
||||
# Parse audio data
|
||||
result = parser._parse_audio_data((mono_audio, 16000))
|
||||
result = parser.parse_mm_data({"audio": (mono_audio.reshape(shape), 16000)})[
|
||||
"audio"
|
||||
]
|
||||
|
||||
# Check that result is still mono (1D)
|
||||
assert result is not None
|
||||
audio_item = result.get(0)
|
||||
assert audio_item.ndim == 1
|
||||
assert audio_item.shape == (16000,)
|
||||
assert len(audio_item) == 16000
|
||||
np.testing.assert_array_equal(audio_item, mono_audio)
|
||||
|
||||
def test_parser_with_target_channels_2(self):
|
||||
"""Parser should reduce 6-channel to 2-channel when target_channels=2."""
|
||||
|
||||
@@ -1098,8 +1098,14 @@ def test_flashinfer_mla_dspark_dcp_supports_target_and_draft(monkeypatch):
|
||||
pytest.param(False, 3, 1, 0, id="noncausal-multi-token"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("cp_interleave_size", [1, 16, 896])
|
||||
def test_tokenspeed_mla_decode_contract(
|
||||
monkeypatch, causal, tokens_per_decode, dcp_world_size, dcp_rank
|
||||
monkeypatch,
|
||||
causal,
|
||||
tokens_per_decode,
|
||||
dcp_world_size,
|
||||
dcp_rank,
|
||||
cp_interleave_size,
|
||||
):
|
||||
decode_call = None
|
||||
num_decodes = 2
|
||||
@@ -1135,7 +1141,7 @@ def test_tokenspeed_mla_decode_contract(
|
||||
impl = object.__new__(tokenspeed_mla_module.TokenspeedMLAImpl)
|
||||
impl.dcp_world_size = dcp_world_size
|
||||
impl.dcp_rank = dcp_rank
|
||||
impl.cp_kv_cache_interleave_size = 1
|
||||
impl._parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=1)
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl.qk_rope_head_dim = qk_rope_head_dim
|
||||
@@ -1164,6 +1170,8 @@ def test_tokenspeed_mla_decode_contract(
|
||||
dtype=torch.float8_e4m3fn,
|
||||
)
|
||||
|
||||
# NIXL may resolve the interleave size after the implementation is created.
|
||||
impl._parallel_config.cp_kv_cache_interleave_size = cp_interleave_size
|
||||
out, lse = impl.forward_mqa(
|
||||
q,
|
||||
kv_cache,
|
||||
@@ -1194,6 +1202,142 @@ def test_tokenspeed_mla_decode_contract(
|
||||
assert decode_call["return_lse"] is True
|
||||
assert decode_call["cp_world"] == dcp_world_size
|
||||
assert decode_call["cp_rank"] == dcp_rank
|
||||
assert decode_call.get("cp_interleave_size", 1) == cp_interleave_size
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.TOKENSPEED_MLA not in BACKENDS_TO_TEST,
|
||||
reason="TokenSpeed MLA requires an SM100-family GPU and tokenspeed-mla",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("cp_interleave_size", "seq_len"),
|
||||
[
|
||||
pytest.param(1, 5, id="token-interleave-empty-local-ranks"),
|
||||
pytest.param(1, 8 + 17, id="token-interleave-full-cycle"),
|
||||
pytest.param(896, 5, id="block-interleave-empty-local-ranks"),
|
||||
pytest.param(896, 8 * 896 + 17, id="block-interleave-full-cycle"),
|
||||
],
|
||||
)
|
||||
def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len):
|
||||
from tokenspeed_mla import tokenspeed_mla_decode
|
||||
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
_lse_weighted_combine,
|
||||
mask_dcp_empty_shards_,
|
||||
)
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = torch.device("cuda")
|
||||
dcp_world_size = 8
|
||||
kernel_block_size = 64
|
||||
num_heads = 128
|
||||
kv_lora_rank = 512
|
||||
qk_rope_head_dim = 64
|
||||
head_size = kv_lora_rank + qk_rope_head_dim
|
||||
|
||||
def make_paged_cache(tokens: torch.Tensor):
|
||||
# Empty DCP ranks still have the null block in vLLM's physical cache.
|
||||
num_pages = max(1, cdiv(tokens.shape[0], kernel_block_size))
|
||||
cache = torch.zeros(
|
||||
num_pages,
|
||||
kernel_block_size,
|
||||
head_size,
|
||||
dtype=tokens.dtype,
|
||||
device=device,
|
||||
)
|
||||
cache.view(-1, head_size)[: tokens.shape[0]].copy_(tokens)
|
||||
block_table = torch.arange(
|
||||
num_pages, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
return cache, block_table
|
||||
|
||||
# Four query positions exercise the MTP/DSpark causal-mask path.
|
||||
query = (torch.randn(1, 4, num_heads, head_size, device=device) * 0.1).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
global_tokens = (torch.randn(seq_len, head_size, device=device) * 0.1).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
workspace = tokenspeed_mla_module._get_workspace(device, num_heads, kv_lora_rank)
|
||||
global_cache, global_block_table = make_paged_cache(global_tokens)
|
||||
global_seq_len = torch.tensor([seq_len], dtype=torch.int32, device=device)
|
||||
reference, _ = tokenspeed_mla_decode(
|
||||
query=query,
|
||||
kv_cache=global_cache,
|
||||
workspace_buffer=workspace,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
block_tables=global_block_table,
|
||||
seq_lens=global_seq_len,
|
||||
max_seq_len=seq_len,
|
||||
softmax_scale=head_size**-0.5,
|
||||
return_lse=True,
|
||||
)
|
||||
|
||||
positions = torch.arange(seq_len, device=device)
|
||||
owners = positions.div(cp_interleave_size, rounding_mode="floor").remainder(
|
||||
dcp_world_size
|
||||
)
|
||||
max_local_seq_len = (
|
||||
cdiv(seq_len, dcp_world_size * cp_interleave_size) * cp_interleave_size
|
||||
)
|
||||
partial_outputs = []
|
||||
partial_lses = []
|
||||
for dcp_rank in range(dcp_world_size):
|
||||
local_tokens = global_tokens[owners == dcp_rank]
|
||||
local_cache, local_block_table = make_paged_cache(local_tokens)
|
||||
local_seq_len = torch.tensor(
|
||||
[local_tokens.shape[0]], dtype=torch.int32, device=device
|
||||
)
|
||||
impl = object.__new__(tokenspeed_mla_module.TokenspeedMLAImpl)
|
||||
impl.dcp_world_size = dcp_world_size
|
||||
impl.dcp_rank = dcp_rank
|
||||
impl._parallel_config = SimpleNamespace(
|
||||
cp_kv_cache_interleave_size=cp_interleave_size
|
||||
)
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl.qk_rope_head_dim = qk_rope_head_dim
|
||||
impl.num_heads = num_heads
|
||||
impl.scale = head_size**-0.5
|
||||
impl.softmax_scale = None
|
||||
impl.output_scale = None
|
||||
impl._workspace_buffer = workspace
|
||||
metadata = SimpleNamespace(
|
||||
num_decodes=1,
|
||||
num_decode_tokens=query.shape[1],
|
||||
max_seq_len=max_local_seq_len,
|
||||
causal=True,
|
||||
decode=SimpleNamespace(
|
||||
block_table=local_block_table,
|
||||
seq_lens=local_seq_len,
|
||||
dcp_tot_seq_lens=global_seq_len,
|
||||
),
|
||||
)
|
||||
output, lse = impl.forward_mqa(
|
||||
query.view(-1, num_heads, head_size),
|
||||
local_cache,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale_float=1.0, _k_scale_float=1.0),
|
||||
)
|
||||
# Empty-shard normalization belongs to the downstream DCP combine path.
|
||||
mask_dcp_empty_shards_(
|
||||
lse,
|
||||
local_seq_len,
|
||||
torch.tensor([0, query.shape[1]], dtype=torch.int32, device=device),
|
||||
)
|
||||
partial_outputs.append(
|
||||
output.view(query.shape[1], num_heads, kv_lora_rank).float()
|
||||
)
|
||||
partial_lses.append(lse.view(query.shape[1], num_heads).float())
|
||||
|
||||
# TokenSpeed returns base-2 LSE. Merge the per-rank partial attention states
|
||||
# the same way MLADCPManager does before comparing to the unsharded result.
|
||||
merged = _lse_weighted_combine(
|
||||
torch.stack(partial_outputs), torch.stack(partial_lses), is_lse_base_on_e=False
|
||||
).unsqueeze(0)
|
||||
# FP8 attention rounds differently when KV is partitioned across DCP ranks.
|
||||
torch.testing.assert_close(merged, reference.float(), atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_fp8_kvcache", [False, True], ids=["bf16", "fp8"])
|
||||
|
||||
@@ -50,6 +50,7 @@ def _create_vllm_config() -> MagicMock:
|
||||
vllm_config.parallel_config = ParallelConfig()
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.num_speculative_tokens = 0
|
||||
vllm_config.use_cumem_cudagraph_pool = False
|
||||
return vllm_config
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from utils import (
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
|
||||
|
||||
@skip_unsupported
|
||||
@@ -778,32 +779,16 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
print(f"[Prompt {prompt_idx}] Generated {len(token_ids)} tokens: {token_ids}")
|
||||
print(f"[Prompt {prompt_idx}] Decode logprobs: {decode_logprobs.tolist()}")
|
||||
|
||||
# Step 2: For each token position, run prefill and compare
|
||||
# Step 2: For each token position, run prefill and compare.
|
||||
prompt_token_ids = list(decode_output.prompt_token_ids or [])
|
||||
|
||||
print(f"\n[Prompt {prompt_idx}] Verifying each token via prefill...")
|
||||
|
||||
for token_idx in range(len(token_ids)):
|
||||
# Construct the prefix up to (but not including) this token
|
||||
# Construct the prefix up to (but not including) this token using
|
||||
# token ids directly.
|
||||
current_token = token_ids[token_idx]
|
||||
|
||||
# We need to detokenize to get the text prefix
|
||||
# For this, we'll use the tokenizer from the LLM
|
||||
# However, the LLM API doesn't expose tokenizer easily, so we'll
|
||||
# construct the prefix by decoding from the original prompt
|
||||
|
||||
# Get text up to this point by using the output text
|
||||
# This is approximate but should work for verification
|
||||
if token_idx == 0:
|
||||
prefix_prompt = prompt
|
||||
else:
|
||||
# No per-token text, so regenerate with max_tokens = token_idx
|
||||
# to get the prefix text
|
||||
prefix_sp = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=token_idx,
|
||||
logprobs=1,
|
||||
)
|
||||
prefix_output = llm.generate([prompt], prefix_sp, use_tqdm=False)[0]
|
||||
prefix_prompt = prompt + prefix_output.outputs[0].text
|
||||
prefix_token_ids = prompt_token_ids + list(token_ids[:token_idx])
|
||||
|
||||
# Now run prefill with max_tokens=1 to get the logprob of the next token
|
||||
prefill_sp = SamplingParams(
|
||||
@@ -814,19 +799,23 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
|
||||
print(
|
||||
f" [Token {token_idx}] Running prefill for prefix "
|
||||
f"(len={len(prefix_prompt)})..."
|
||||
f"(num_tokens={len(prefix_token_ids)})..."
|
||||
)
|
||||
prefill_output = llm.generate(
|
||||
[TokensPrompt(prompt_token_ids=prefix_token_ids)],
|
||||
prefill_sp,
|
||||
use_tqdm=False,
|
||||
)[0]
|
||||
prefill_logprobs, prefill_token_ids_out = _extract_step_logprobs(
|
||||
prefill_output
|
||||
)
|
||||
prefill_output = llm.generate([prefix_prompt], prefill_sp, use_tqdm=False)[
|
||||
0
|
||||
]
|
||||
prefill_logprobs, prefill_token_ids = _extract_step_logprobs(prefill_output)
|
||||
|
||||
if prefill_logprobs is None:
|
||||
print(f" [Token {token_idx}] Warning: No prefill logprobs available")
|
||||
continue
|
||||
|
||||
# The first token from prefill should match the current token
|
||||
prefill_token = prefill_token_ids[0]
|
||||
prefill_token = prefill_token_ids_out[0]
|
||||
prefill_logprob = prefill_logprobs[0].item()
|
||||
decode_logprob = decode_logprobs[token_idx].item()
|
||||
|
||||
@@ -851,7 +840,7 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
"decode_logprob": decode_logprob,
|
||||
"prefill_logprob": prefill_logprob,
|
||||
"prompt_text": prompt[:100],
|
||||
"prefix_text": prefix_prompt[:100],
|
||||
"prefix_token_ids": prefix_token_ids[:50],
|
||||
}
|
||||
)
|
||||
print(f" [Token {token_idx}] ✗ TOKEN MISMATCH!")
|
||||
@@ -871,7 +860,7 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
"prefill_logprob": prefill_logprob,
|
||||
"diff": diff,
|
||||
"prompt_text": prompt[:100],
|
||||
"prefix_text": prefix_prompt[:100],
|
||||
"prefix_token_ids": prefix_token_ids[:50],
|
||||
"decode_all_tokens": token_ids,
|
||||
"decode_all_logprobs": decode_logprobs.tolist(),
|
||||
}
|
||||
@@ -908,7 +897,7 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
for i, fail in enumerate(failures[:5]): # Show first 5 failures per prompt
|
||||
print(f"\n [Failure {i + 1}] Token position {fail['token_idx']}:")
|
||||
print(f" Reason: {fail['reason']}")
|
||||
print(f" Prefix text: '{fail['prefix_text']}...'")
|
||||
print(f" Prefix token ids: {fail['prefix_token_ids']}...")
|
||||
print(
|
||||
f" Decode: token={fail['decode_token']}, "
|
||||
f"logprob={fail['decode_logprob']:.10f}"
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Batch-invariant LoRA shrink: fixed split-K partials, fixed-order reduction."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import skip_if_not_cuda
|
||||
|
||||
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
|
||||
import vllm.lora.ops.triton_ops.utils as lora_utils
|
||||
from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_shrink
|
||||
from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def batch_invariant(monkeypatch):
|
||||
monkeypatch.setattr(lora_utils, "is_batch_invariant", True)
|
||||
monkeypatch.setattr(shrink_op, "is_batch_invariant", True)
|
||||
lora_utils.get_lora_op_configs.cache_clear()
|
||||
yield
|
||||
lora_utils.get_lora_op_configs.cache_clear()
|
||||
|
||||
|
||||
def _shrink(inputs, weights, mapping, scaling):
|
||||
num_tokens = inputs.size(0)
|
||||
meta = LoRAKernelMeta.make(weights[0].size(0), num_tokens, device=DEVICE)
|
||||
meta.prepare_tensors(mapping)
|
||||
out = torch.zeros(
|
||||
(len(weights), num_tokens, weights[0].size(1)),
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
_LORA_A_PTR_DICT.clear()
|
||||
args = meta.meta_args(token_nums=num_tokens, specialize_active_lora=False)
|
||||
lora_shrink(inputs, weights, out, *args, scaling)
|
||||
return out
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,rank,nslices,dtype",
|
||||
[
|
||||
(2560, 8, 3, torch.bfloat16),
|
||||
(9728, 16, 1, torch.float16),
|
||||
(4097, 64, 1, torch.bfloat16), # K tail block
|
||||
(1024, 8, 1, torch.float16), # K < 8 * BLOCK_K: some splits are empty
|
||||
],
|
||||
)
|
||||
def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
|
||||
set_random_seed(0)
|
||||
inputs = torch.randn((33, hidden_size), dtype=dtype, device=DEVICE)
|
||||
weights = [
|
||||
torch.randn((2, rank, hidden_size), dtype=dtype, device=DEVICE)
|
||||
for _ in range(nslices)
|
||||
]
|
||||
mapping = torch.randint(-1, 2, (33,), dtype=torch.int32, device=DEVICE)
|
||||
out = _shrink(inputs, weights, mapping, 0.5).double().cpu()
|
||||
|
||||
ids, x = mapping.cpu(), inputs.double().cpu()
|
||||
ref = torch.zeros_like(out)
|
||||
for s, weight in enumerate(weights):
|
||||
for lora_id in (0, 1):
|
||||
rows = ids == lora_id
|
||||
ref[s, rows] = 0.5 * x[rows] @ weight[lora_id].double().cpu().T
|
||||
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("nslices", [1, 3])
|
||||
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
|
||||
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
|
||||
|
||||
class RecordReduce:
|
||||
def __getitem__(self, grid):
|
||||
reduce_grids.append(grid)
|
||||
return reduce_kernel[grid]
|
||||
|
||||
monkeypatch.setattr(shrink_op, "_lora_shrink_reduce_kernel", RecordReduce())
|
||||
set_random_seed(0)
|
||||
# K = 2560 is ten BLOCK_K=256 blocks, so all 8 splits contribute.
|
||||
dtype, hidden_size = torch.bfloat16, 2560
|
||||
weights = [
|
||||
torch.randn((3, 16, hidden_size), dtype=dtype, device=DEVICE)
|
||||
for _ in range(nslices)
|
||||
]
|
||||
target = torch.randn((1, hidden_size), dtype=dtype, device=DEVICE)
|
||||
target_id = torch.zeros(1, dtype=torch.int32, device=DEVICE)
|
||||
expected = _shrink(target, weights, target_id, 0.7)[:, 0]
|
||||
for num_tokens in (31, 32, 33, 127, 128, 129):
|
||||
inputs = torch.randn((num_tokens, hidden_size), dtype=dtype, device=DEVICE)
|
||||
mapping = torch.randint(-1, 3, (num_tokens,), dtype=torch.int32, device=DEVICE)
|
||||
pos = num_tokens // 2
|
||||
inputs[pos], mapping[pos] = target[0], 0
|
||||
out = _shrink(inputs, weights, mapping, 0.7)
|
||||
assert torch.equal(out[:, pos], expected), num_tokens
|
||||
assert len(reduce_grids) == 7
|
||||
@@ -1864,6 +1864,31 @@ def test_reset_pending_lazy_stores() -> None:
|
||||
assert hit_tokens == 0, "CPU cache should be empty after reset"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_groups", [1, 2])
|
||||
@pytest.mark.parametrize("lazy", [False, True])
|
||||
def test_reset_releases_pending_cpu_hits(num_groups: int, lazy: bool) -> None:
|
||||
"""A lookup awaiting GPU allocation must not prevent a cache reset."""
|
||||
fix = make_scheduler(num_groups=num_groups, lazy=lazy)
|
||||
sched = fix.scheduler
|
||||
req = make_request(num_blocks=2)
|
||||
cpu_blocks = []
|
||||
for group_id in range(num_groups):
|
||||
cpu_blocks.extend(
|
||||
_allocate_gpu_blocks(sched.cpu_block_pool, req, 2, group_id=group_id)
|
||||
)
|
||||
sched.cpu_block_pool.free_blocks(cpu_blocks)
|
||||
|
||||
assert sched.get_num_new_matched_tokens(req, 0) == (2 * BLOCK_SIZE, True)
|
||||
assert all(block.ref_cnt == 1 for block in cpu_blocks)
|
||||
assert not sched._reqs_to_load
|
||||
|
||||
assert sched.reset()
|
||||
assert all(block.ref_cnt == 0 for block in cpu_blocks)
|
||||
assert not sched._pending_cpu_hits
|
||||
assert sched.get_num_new_matched_tokens(req, 0) == (0, False)
|
||||
assert sched.reset()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 14: Reset with pending loads waits for completion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -486,6 +486,67 @@ def test_preferred_block_size_rejects_backends_with_no_common_size():
|
||||
Platform._preferred_block_size_for_backends(classes, 16, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cpu", [False, True])
|
||||
def test_alignment_rejected_by_a_sibling_backend_raises(monkeypatch, cpu):
|
||||
# Alignment that lands on a size an exact-size sibling rejects must fail
|
||||
# here, not later in select_common_block_size().
|
||||
backends = [_mock_backend([MultipleOf(16)]), _mock_backend([16], exact=True)]
|
||||
monkeypatch.setattr(
|
||||
Platform, "_find_non_ssm_backends", classmethod(lambda cls, c: backends)
|
||||
)
|
||||
|
||||
def bump(cls, vllm_config, backend_cls):
|
||||
vllm_config.cache_config.block_size = 32
|
||||
|
||||
monkeypatch.setattr(Platform, "_align_hybrid_block_size", classmethod(bump))
|
||||
cache_config = SimpleNamespace(
|
||||
block_size=16, user_specified_block_size=True, kv_cache_dtype_skip_layers=None
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=cache_config, model_config=SimpleNamespace(is_hybrid=True)
|
||||
)
|
||||
from vllm.platforms.cpu import CpuPlatform # overrides the method; must check too
|
||||
|
||||
platform = CpuPlatform if cpu else Platform
|
||||
with pytest.raises(ValueError, match="MOCK_EXACT"):
|
||||
platform.update_block_size_for_backend(vllm_config)
|
||||
|
||||
|
||||
def test_xpu_gdn_rounding_rejected_by_a_sibling_backend_raises(monkeypatch):
|
||||
# XPU's GDN rounding runs after super()'s check, so it must check again.
|
||||
try:
|
||||
from vllm.platforms.xpu import XPUPlatform
|
||||
except ImportError:
|
||||
pytest.skip("vllm_xpu_kernels not importable outside an XPU stack")
|
||||
|
||||
backends = [_mock_backend([MultipleOf(16)]), _mock_backend([16], exact=True)]
|
||||
monkeypatch.setattr(
|
||||
Platform, "update_block_size_for_backend", classmethod(lambda cls, c: None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
Platform, "_find_non_ssm_backends", classmethod(lambda cls, c: backends)
|
||||
)
|
||||
gdn_layer = SimpleNamespace(
|
||||
get_attn_backend=lambda: SimpleNamespace(get_name=lambda: "GDN_ATTN")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.vllm.get_layers_from_vllm_config",
|
||||
lambda vllm_config, layer_type: {"gdn": gdn_layer},
|
||||
)
|
||||
cache_config = SimpleNamespace(
|
||||
block_size=16,
|
||||
mamba_cache_mode="none",
|
||||
mamba_page_size_padded=None,
|
||||
user_specified_block_size=True,
|
||||
kv_cache_dtype_skip_layers=None,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=cache_config, model_config=SimpleNamespace(is_hybrid=True)
|
||||
)
|
||||
with pytest.raises(ValueError, match="MOCK_EXACT"):
|
||||
XPUPlatform.update_block_size_for_backend(vllm_config)
|
||||
|
||||
|
||||
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
|
||||
model = Mock()
|
||||
model.get_mm_lora_token_counts.side_effect = (
|
||||
|
||||
Reference in New Issue
Block a user