Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
@@ -216,15 +216,6 @@ def test_models_distributed(
pytest.skip(f"Skip test for {TARGET_TEST_SUITE}")
with monkeypatch.context() as monkeypatch_context:
if (
model == "meta-llama/Llama-3.2-1B-Instruct"
and distributed_executor_backend == "ray"
and attention_backend == ""
and TARGET_TEST_SUITE == "L4"
and enable_prompt_embeds
): # noqa
pytest.skip("enable_prompt_embeds does not work with ray compiled dag.")
for k, v in extra_env.items():
monkeypatch_context.setenv(k, v)
@@ -157,10 +157,17 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
)
if self.kv_cache_dtype != self.dtype:
self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
self.attn._k_scale_float = 1.0
self.attn._v_scale_float = 1.0
# Non-unit scales so the fused and unfused writers must agree on them.
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
self.attn._k_scale_cpu = torch.tensor(
0.5, dtype=torch.float32, device="cpu"
)
self.attn._v_scale_cpu = torch.tensor(
0.25, dtype=torch.float32, device="cpu"
)
self.attn._k_scale_float = 0.5
self.attn._v_scale_float = 0.25
else:
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
@@ -527,16 +534,6 @@ def _run_qk_norm_rope_kvcache_fusion_test(
else:
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
if not expect_fusion or (not is_fp8_cache and mrope_section is None):
# The AITER PTS kernel populates k_out only for non-FP8 caches.
# With FP8, the kernel writes quantized K directly to the cache
# and may leave k_out uninitialised. In production this is fine
# because downstream attention reads K from the cache.
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
# Allow one E4M3 quantization step while still comparing every block.
if is_fp8_cache:
@@ -544,6 +541,18 @@ def _run_qk_norm_rope_kvcache_fusion_test(
else:
cache_atol, cache_rtol = ATOL, RTOL
# The fused k_out is the dequantized cache write, so it carries the same
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
# from the cache and asks the kernel not to write k_out at all.
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
torch.testing.assert_close(
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
torch.testing.assert_close(
kv_cache_unfused.float(),
kv_cache_fused.float(),
@@ -570,20 +579,52 @@ _FUSION_CONFIGS = [
_FUSION_CONFIGS,
)
@pytest.mark.parametrize(
"attn_backend",
"attn_backend, use_shuffle_kv_layout, kv_layout",
[
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
# K/V side contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_ATTN,
"0",
KVCacheLayout.LBHNC,
id="rocm_attn",
),
# Unified never reads the shuffle env; its strided views take any layout.
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBHNC,
id="unified-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBNHC,
id="unified-token_major",
),
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
# contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBHNC,
id="fa-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBNHC,
id="fa-token_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"1",
KVCacheLayout.LBHNC,
id="fa-shuffle",
),
],
)
@pytest.mark.parametrize("num_tokens", [5, 2048])
@pytest.mark.parametrize(
"kv_layout",
[
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
],
)
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("block_size", [16, 32, 64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@@ -615,6 +656,7 @@ def test_qk_norm_rope_kvcache_fusion(
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
use_shuffle_kv_layout: str,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
@@ -627,7 +669,7 @@ def test_qk_norm_rope_kvcache_fusion(
rotary_dim=rotary_dim,
block_size=block_size,
is_neox=is_neox,
use_shuffle_kv_layout="0",
use_shuffle_kv_layout=use_shuffle_kv_layout,
kv_layout=kv_layout,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
+84
View File
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for cpu_all_reduce_ (gloo, CPU only)."""
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.distributed.utils import (
cpu_all_reduce_,
stateless_init_torch_distributed_process_group,
)
from vllm.utils.network_utils import get_open_port
pytestmark = pytest.mark.cpu_test
def _make_input(rank: int, world_size: int, step: int) -> torch.Tensor:
# Same shape as the V2 DP metadata tensor; values differ per rank/step.
t = torch.zeros(6, world_size, dtype=torch.int32)
t[:, rank] = torch.arange(6, dtype=torch.int32) * (rank + 1) + step
return t
def _worker(rank: int, world_size: int, port: int, stateless_port: int):
dist.init_process_group(
"gloo",
init_method=f"tcp://127.0.0.1:{port}",
rank=rank,
world_size=world_size,
)
coord = GroupCoordinator(
group_ranks=[list(range(world_size))],
local_rank=0,
torch_distributed_backend="gloo",
use_device_communicator=False,
group_name="cpu_all_reduce_test",
)
# Several steps to exercise tag reuse across consecutive calls.
for step in range(5):
expected = _make_input(rank, world_size, step)
dist.all_reduce(expected, group=coord.cpu_group)
t = _make_input(rank, world_size, step)
out = coord.cpu_all_reduce_(t)
assert out is t
torch.testing.assert_close(t, expected, rtol=0, atol=0)
# Float sums must be bitwise identical on every rank.
t = torch.randn(6, world_size, generator=torch.Generator().manual_seed(rank))
expected = t.clone()
dist.all_reduce(expected, group=coord.cpu_group)
coord.cpu_all_reduce_(t)
torch.testing.assert_close(t, expected)
gathered = [torch.empty_like(t) for _ in range(world_size)]
dist.all_gather(gathered, t, group=coord.cpu_group)
for other in gathered:
assert torch.equal(other, t)
# Unregistered (stateless) process group, as used by the engine-core DP
# group and elastic EP.
stateless_pg = stateless_init_torch_distributed_process_group(
"127.0.0.1", stateless_port, rank, world_size, backend="gloo"
)
expected = _make_input(rank, world_size, 0)
dist.all_reduce(expected)
t = _make_input(rank, world_size, 0)
cpu_all_reduce_(t, stateless_pg)
torch.testing.assert_close(t, expected, rtol=0, atol=0)
dist.destroy_process_group()
# Non-power-of-two sizes exercise the cross-block exchange.
@pytest.mark.parametrize("world_size", [2, 3, 4, 6, 8, 9, 12])
def test_cpu_all_reduce(world_size: int):
mp.spawn(
_worker,
args=(world_size, get_open_port(), get_open_port()),
nprocs=world_size,
join=True,
)
+18
View File
@@ -1,6 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.plugins_tests.test_oot_registration_online import (
run_and_test_dummy_opt_api_server,
)
@@ -8,3 +12,17 @@ from tests.plugins_tests.test_oot_registration_online import (
def test_distributed_oot(dummy_opt_path: str):
run_and_test_dummy_opt_api_server(dummy_opt_path, tp=2)
@pytest.mark.parametrize("enforce_eager", [True, False])
def test_distributed_oot_pp_raw_input_tokens(dummy_opt_path: str, enforce_eager: bool):
# The model asserts that input_ids reaches every pipeline stage.
overrides = {"architectures": ["MyOPTRawInputTokensForCausalLM"]}
extra_args = ["-pp", "2", "--hf-overrides", json.dumps(overrides)]
if enforce_eager:
extra_args.append("--enforce-eager")
run_and_test_dummy_opt_api_server(
dummy_opt_path,
extra_args=extra_args,
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
)
+22 -13
View File
@@ -51,7 +51,7 @@ def _traffic_loop(
stop: threading.Event,
finished: threading.Event,
is_probe: bool = False,
) -> list[tuple[float, float, int | None]]:
) -> list[tuple[float, float, int | None, str | None]]:
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
@@ -60,15 +60,18 @@ def _traffic_loop(
is_ready = False
while not stop.is_set():
request_start = time.perf_counter()
phase = None
try:
response = requests.post(
url, json=request_payload, headers=headers, timeout=120
)
status_code = response.status_code
if is_probe and status_code == 200:
phase = response.json()["phase"]
except requests.exceptions.RequestException:
status_code = None
responses.append((request_start, time.perf_counter(), status_code))
if status_code == 200:
responses.append((request_start, time.perf_counter(), status_code, phase))
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
if not is_ready:
ready.wait(timeout=120)
is_ready = True
@@ -78,14 +81,16 @@ def _traffic_loop(
return responses
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
rejected = [end for _, end, status in responses if status == 503]
if not rejected:
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
committing = [end for _, end, _, phase in responses if phase == "committing"]
if not committing:
return 0
recovered = next(
end for _, end, status in responses if status == 200 and end > rejected[-1]
end
for _, end, status, phase in responses
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
)
return recovered - rejected[0]
return recovered - committing[0]
def _scale_with_traffic(
@@ -124,16 +129,20 @@ def _scale_with_traffic(
bad_statuses = {
status
for responses in [probe_result, *results]
for _, _, status in responses
for _, _, status, _ in responses
if status not in (200, 503)
}
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
probe_503 = [start for start, _, status in probe_result if status == 503]
assert probe_503, "Scaling probe did not observe commit"
probe_committing = [
start for start, _, _, phase in probe_result if phase == "committing"
]
assert probe_committing, "Scaling probe did not observe commit"
assert not results or any(
status == 200 and start_time <= request_start and request_end < probe_503[0]
status == 200
and start_time <= request_start
and request_end < probe_committing[0]
for responses in results
for request_start, request_end, status in responses
for request_start, request_end, status, _ in responses
), "No request completed successfully during preparation"
print(
@@ -585,6 +585,7 @@ def _check_dummy_hash_model_forward(
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.parallel_config = vllm_config.parallel_config
model.use_native_mega_moe = False
model.use_sequence_parallel = False
model.fuse_mhc_all_reduce = False
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Make sure ray assigns GPU workers to the correct node.
Run:
```sh
cd $VLLM_PATH/tests
pytest distributed/test_multi_node_assignment.py
```
"""
import os
import time
import pytest
import ray
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
from vllm import initialize_ray_cluster
from vllm.config import ParallelConfig
from vllm.utils.network_utils import get_ip
from vllm.v1.executor.ray_utils import available_resources_per_node
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
@pytest.mark.skipif(
not VLLM_MULTI_NODE, reason="Need at least 2 nodes to run the test."
)
def test_multi_node_assignment() -> None:
# NOTE: important to keep this class definition here
# to let ray use cloudpickle to serialize it.
class Actor:
def get_ip(self):
return get_ip()
for _ in range(10):
config = ParallelConfig(1, 2)
initialize_ray_cluster(config)
current_ip = get_ip()
workers = []
for bundle_id, bundle in enumerate(config.placement_group.bundle_specs):
if not bundle.get("GPU", 0):
continue
scheduling_strategy = PlacementGroupSchedulingStrategy(
placement_group=config.placement_group,
placement_group_capture_child_tasks=True,
placement_group_bundle_index=bundle_id,
)
worker = ray.remote(
num_cpus=0,
num_gpus=1,
scheduling_strategy=scheduling_strategy,
)(Actor).remote()
worker_ip = ray.get(worker.get_ip.remote())
assert worker_ip == current_ip
workers.append(worker)
for worker in workers:
ray.kill(worker)
ray.util.remove_placement_group(config.placement_group)
gpus_needed = len(workers)
deadline = time.monotonic() + 60
node_id = ray.get_runtime_context().get_node_id()
while True:
free_gpus = available_resources_per_node().get(node_id, {}).get("GPU", 0)
if free_gpus >= gpus_needed:
break
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("placement group resources were not released")
time.sleep(min(1, remaining))
@@ -598,6 +598,21 @@ class TestCohereErrorEnvelope:
assert "message" in body
assert isinstance(body["message"], str) and body["message"]
@pytest.mark.parametrize("path", ["/cohere/v2/chat", "/cohere/v2/chat/render"])
def test_required_tool_choice_without_tools_returns_400(self, path):
"""Reject an impossible tool choice before reaching either handler."""
app = _build_app_with_vllm_handlers(handler=None)
with TestClient(app) as client:
response = client.post(
path,
json={**_minimal_request_body(), "tool_choice": "REQUIRED"},
)
assert response.status_code == HTTPStatus.BAD_REQUEST
body = response.json()
assert "error" not in body
assert "REQUIRED" in body["message"]
assert "tools" in body["message"]
def test_http_error_body_is_cohere_shaped(self):
# A raised ``HTTPException`` from anywhere in the request cycle
# is routed through ``http_exception_handler`` (producing the
+21
View File
@@ -95,6 +95,27 @@ class TestCohereChatV2Request:
tool_choice="ANY", # not REQUIRED/NONE
)
@pytest.mark.parametrize("tool_fields", [{}, {"tools": None}, {"tools": []}])
def test_required_tool_choice_requires_tools(self, tool_fields):
with pytest.raises(VLLMValidationError, match="REQUIRED.*tools") as exc_info:
CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
tool_choice="REQUIRED",
**tool_fields,
)
assert exc_info.value.parameter == "tool_choice"
@pytest.mark.parametrize("tool_choice", [None, "NONE"])
def test_nonrequired_tool_choice_allows_empty_tools(self, tool_choice):
request = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
tool_choice=tool_choice,
tools=[],
)
assert request.tool_choice == tool_choice
def test_stop_sequences_bounded_by_env_limit(self):
messages = [{"role": "user", "content": "hi"}]
limit = envs.VLLM_MAX_STOP_STRINGS
@@ -64,6 +64,7 @@ def test_load_prompt_embeds(
model_config = Mock(spec=ModelConfig)
model_config.enable_prompt_embeds = True
model_config.get_hidden_size.return_value = hidden_size
model_config.hf_text_config = object()
model_config.dtype = dtype
# construct arbitrary tensors of various dtypes, layouts, and sizes.
@@ -348,6 +348,85 @@ def test_non_streaming_derender_initializes_parser_prefix(
assert choices[0].finish_reason == "length"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("stop", "include_stop_str_in_output", "dropped"),
[
("eos", False, True),
("stop_token_id", False, True),
("stop_string", False, False),
("eos", True, False),
],
)
async def test_parsed_derender_leaves_out_stop_token_like_engine(
tokenizer, stop, include_stop_str_in_output, dropped
):
"""A parser keeps special tokens in the decoded text, so the token that
stopped generation must be dropped the way the engine drops it."""
eos = tokenizer.eos_token_id
answer_ids = tokenizer.encode("answer", add_special_tokens=False)
bang = tokenizer.encode("!", add_special_tokens=False)[-1]
last_id, stop_reason = {
"eos": (eos, None),
"stop_token_id": (bang, bang),
"stop_string": (bang, "!"),
}[stop]
parser = MagicMock()
parser.parse.side_effect = lambda text, *args, **kwargs: (None, text, [])
parser.parse_delta.side_effect = lambda text, *args, **kwargs: DeltaMessage(
content=text
)
request = _chat_request(include_stop_str_in_output=include_stop_str_in_output)
with ThreadPoolExecutor(max_workers=1) as executor:
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
renderer.get_eos_token_id.return_value = eos
renderer._executor = executor
model_config = MagicMock()
model_config.hf_config.model_type = "llama"
model_config.model = MODEL_NAME
model_config.try_get_generation_config.return_value = {}
derenderer = OnlineDerenderer(
model_config=model_config,
renderer=renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="string",
)
derenderer.parser = MagicMock(return_value=parser)
def batch_content(token_ids: list[int], finish_reason: str) -> str:
choice = GenerateTokensChoice(
index=0,
token_ids=token_ids,
finish_reason=finish_reason,
stop_reason=stop_reason if finish_reason == "stop" else None,
)
response = GenerateTokensResponse(
request_id="test", prompt_token_ids=[11, 12], choices=[choice]
)
return derenderer._derender_chat(response, request)[0].message.content
# Same decode window and seeding; "length" never drops a token.
expected = batch_content(
answer_ids if dropped else answer_ids + [last_id], "length"
)
batch = batch_content(answer_ids + [last_id], "stop")
stream, _ = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
answer_ids + [last_id], finish_reason="stop", stop_reason=stop_reason
),
chat_request=request,
prompt_token_ids=[11, 12],
)
assert batch == expected
assert stream.choices[0].delta.content == expected
class TestDetokenizeDelta:
"""_detokenize_delta: chunked decode must equal one shot decode."""
@@ -139,7 +139,7 @@ def test_sentencepiece_leading_space_matches_engine():
assert [t.token for t in entry.top_logprobs] == convert_ids_list_to_tokens(
tokenizer, [spaced, bare]
)
completion = _convert_chat_logprobs_to_completion_logprobs(resolved)
completion = _convert_chat_logprobs_to_completion_logprobs(resolved, logprobs)
assert completion.top_logprobs == [{" true": -0.5, "true": -1.5}]
@@ -0,0 +1,212 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""generate + derender must return the same `top_logprobs` as the coupled
`/v1/chat/completions` and `/v1/completions` endpoints (#59513).
Both sides start from the same per-position engine dicts, built with the
engine's own `append_logprobs_for_next_position` and detokenized the way the
engine does, and go through the real coupled serving objects on one side and
`/inference/v1/generate` + derender on the other. The cases include a sampled
token outside the top k, where chat keeps k entries and completions k + 1; the
greedy end-to-end parity tests never produce that case.
"""
import pytest
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
_build_mock_engine,
_build_serving_chat,
)
from tests.entrypoints.openai.completion.test_completion_error import (
_build_serving_completion,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProbs
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import append_logprobs_for_next_position
from vllm.renderers.online_derenderer import (
_chat_top_logprobs_limit,
_completion_top_logprobs_limit,
_convert_chat_logprobs_to_completion_logprobs,
_resolve_logprobs,
)
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@pytest.fixture(scope="module")
def tokenizer():
return get_tokenizer(MODEL_NAME)
@pytest.fixture(scope="module")
def serving_chat():
return _build_serving_chat(_build_mock_engine())
@pytest.fixture(scope="module")
def serving_completion():
return _build_serving_completion(_build_mock_engine())
@pytest.fixture(scope="module")
def vocab_ids(tokenizer):
"""Ordinary word pieces (no byte-fallback tokens, which need the U+FFFD
context repair that is compared elsewhere)."""
ids = tokenizer.encode(
"the quick brown fox jumps over a lazy dog while seven cats watch "
"from an old wooden fence near the river bank",
add_special_tokens=False,
)
pieces = tokenizer.convert_ids_to_tokens(ids)
return list(dict.fromkeys(i for i, p in zip(ids, pieces) if "<0x" not in p))
def _positions(tokenizer, vocab_ids, k, sampled_ranks):
"""Engine output for one completion: per position the sampled id and its
dict (sampled first, then ranks 1..k), with decoded tokens as the engine
fills them. k = -1 returns every candidate, as for the whole vocabulary."""
token_ids, dicts = [], []
for pos, sampled_rank in enumerate(sampled_ranks):
n_top = max(k, sampled_rank) if k != -1 else len(vocab_ids) - 1
ranked = [vocab_ids[(pos + j) % len(vocab_ids)] for j in range(n_top + 1)]
sampled = ranked[sampled_rank - 1]
top = ranked[: len(ranked) - 1] if k == -1 else ranked[:k]
ids = [sampled, *top]
logprobs = [-0.25 * sampled_rank] + [-0.25 * (r + 1) for r in range(len(top))]
per_position: list = []
append_logprobs_for_next_position(
per_position,
ids,
logprobs,
convert_ids_list_to_tokens(tokenizer, ids),
sampled_rank,
len(top),
)
token_ids.append(sampled)
dicts.append(per_position[0])
return token_ids, dicts
def _generate(token_ids, dicts) -> GenerateLogProbs:
out = ServingTokens._create_tokens_logprobs(
None, token_ids=token_ids, top_logprobs=dicts
)
# Over the wire, as a derender client would send it back.
return GenerateLogProbs.model_validate_json(out.model_dump_json())
# Sampled token rank per position: top-1, inside the top k, outside it.
RANKS = [1, 2, 1, 7, 3, 9, 1]
@pytest.mark.parametrize("k", [0, 1, 2, 5, -1])
def test_chat_derender_matches_coupled_chat(tokenizer, vocab_ids, serving_chat, k):
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
coupled = serving_chat._create_chat_logprobs(
token_ids=token_ids,
top_logprobs=dicts,
tokenizer=tokenizer,
num_output_top_logprobs=k,
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hi"}],
logprobs=True,
top_logprobs=k,
)
derendered = _resolve_logprobs(
_generate(token_ids, dicts),
tokenizer,
top_limit=_chat_top_logprobs_limit(request),
)
assert derendered.model_dump() == coupled.model_dump()
@pytest.mark.parametrize("k", [0, 1, 2, 5, -1])
def test_completion_derender_matches_coupled_completion(
tokenizer, vocab_ids, serving_completion, k
):
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
coupled = serving_completion._create_completion_logprobs(
token_ids=token_ids,
top_logprobs=dicts,
num_output_top_logprobs=k,
tokenizer=tokenizer,
)
request = CompletionRequest(model=MODEL_NAME, prompt="hi", logprobs=k)
generated = _generate(token_ids, dicts)
derendered = _convert_chat_logprobs_to_completion_logprobs(
_resolve_logprobs(
generated,
tokenizer,
top_limit=_completion_top_logprobs_limit(request),
),
generated,
)
assert derendered.model_dump() == coupled.model_dump()
if k > 0:
# The case #59513 is about: completions keeps k + 1 entries when the
# sampled token is outside the top k.
outside = [i for i, r in enumerate(RANKS) if r > k]
assert all(len(derendered.top_logprobs[i]) == k + 1 for i in outside)
if k == -1:
# /v1/completions returns every candidate for logprobs=-1 (#55717).
assert [len(top) for top in derendered.top_logprobs] == [len(d) for d in dicts]
@pytest.mark.parametrize(
"top_limit",
[
_chat_top_logprobs_limit(None),
# A request that does not ask for logprobs says nothing about the cut.
_chat_top_logprobs_limit(
ChatCompletionRequest(
model=MODEL_NAME, messages=[{"role": "user", "content": "hi"}]
)
),
_completion_top_logprobs_limit(None),
_completion_top_logprobs_limit(
CompletionRequest(model=MODEL_NAME, prompt="hi")
),
],
)
def test_without_a_logprobs_request_derender_keeps_every_candidate(
tokenizer, vocab_ids, top_limit
):
token_ids, dicts = _positions(tokenizer, vocab_ids, 2, RANKS)
derendered = _resolve_logprobs(
_generate(token_ids, dicts), tokenizer, top_limit=top_limit
)
assert [len(e.top_logprobs) for e in derendered.content] == [len(d) for d in dicts]
@pytest.mark.parametrize("k", [-1, 0, 2])
def test_completion_position_without_logprobs_stays_none(
tokenizer, vocab_ids, serving_completion, k
):
"""A position the engine returned no logprobs for is `None` in
`/v1/completions`, not `{}`; derender keeps it that way."""
token_ids, dicts = _positions(tokenizer, vocab_ids, k, RANKS)
dicts[3] = None
coupled = serving_completion._create_completion_logprobs(
token_ids=token_ids,
top_logprobs=dicts,
num_output_top_logprobs=k,
tokenizer=tokenizer,
)
request = CompletionRequest(model=MODEL_NAME, prompt="hi", logprobs=k)
generated = _generate(token_ids, dicts)
derendered = _convert_chat_logprobs_to_completion_logprobs(
_resolve_logprobs(
generated,
tokenizer,
top_limit=_completion_top_logprobs_limit(request),
),
generated,
)
assert derendered.top_logprobs == coupled.top_logprobs
assert derendered.top_logprobs[3] is None
@@ -14,11 +14,11 @@ from vllm.logprobs import Logprob
def test_top_logprobs_alternatives_have_own_token_ids():
"""Each top_logprobs alternative must carry its own integer token id."""
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
num_output_top_logprobs=2,
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
)
token_ids = {e.token_id for e in result.content[0].top_logprobs}
assert token_ids == {262, 257}, f"got {token_ids}"
@@ -32,7 +32,6 @@ def test_sampled_entry_carries_token_id_and_rank():
top_logprobs=[
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
],
num_output_top_logprobs=2,
)
entry = result.content[0]
assert entry.token_id == 262
@@ -50,7 +49,6 @@ def test_sampled_token_absent_from_topk_uses_sentinel():
None,
token_ids=[5],
top_logprobs=[{7: Logprob(-0.9)}],
num_output_top_logprobs=1,
)
entry = result.content[0]
assert entry.token_id == 5
@@ -60,14 +58,13 @@ def test_sampled_token_absent_from_topk_uses_sentinel():
def test_logprobs_zero_emits_sampled_token():
"""logprobs=0 must still emit 1 entry (the sampled token)."""
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=0,
top_logprobs=[{7: Logprob(-0.9)}],
)
assert len(result.content[0].top_logprobs) == 1
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
def test_logprobs_minus_one_emits_all_tokens():
@@ -75,7 +72,6 @@ def test_logprobs_minus_one_emits_all_tokens():
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=-1,
)
assert len(result.content[0].top_logprobs) == 2
@@ -86,7 +82,6 @@ def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=2,
)
entry = result.content[0]
assert entry.token == "token_id:7"
@@ -102,7 +97,6 @@ def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
top_logprobs=[
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
],
num_output_top_logprobs=2,
tokenizer=tokenizer,
)
entry = result.content[0]
@@ -121,7 +115,6 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
num_output_top_logprobs=1,
tokenizer=tokenizer,
)
assert [e.token for e in result.content] == ["<7>", "<9>"]
@@ -130,9 +123,9 @@ def test_resolved_logprobs_decode_tokens_the_engine_did_not():
def test_sampled_token_outside_topk_comes_first():
"""The engine puts the sampled token first, then ranks 1..k. When the
sampled token is outside the top k it takes one of the k slots, so rank k
is left out (same as the OpenAI endpoints)."""
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
all k + 1 candidates when the sampled token is outside the top k; derender
applies each endpoint's cut (#59513)."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[50],
@@ -143,9 +136,8 @@ def test_sampled_token_outside_topk_comes_first():
20: Logprob(-1.0, rank=2),
}
],
num_output_top_logprobs=2,
)
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1]
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
def test_logprob_is_required_on_the_wire():
@@ -160,7 +152,6 @@ def test_rank_zero_from_a_nan_logprob_is_none():
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
num_output_top_logprobs=1,
)
entry = result.content[0]
assert entry.rank is None
@@ -176,7 +167,6 @@ def test_text_logprobs_clamp_a_nan_logprob():
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
num_output_top_logprobs=1,
tokenizer=MagicMock(),
)
entry = result.content[0]
+25 -1
View File
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
def test_paged_attention(
kv_cache_factory,
num_seqs: int,
@@ -133,9 +134,14 @@ def test_paged_attention(
kv_cache_dtype: str,
seed: int,
device: str,
use_interleaved_v_cache: bool,
) -> None:
if current_platform.is_navi() and (
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
kv_cache_dtype == "fp8"
or head_size != 128
or block_size != 16
or use_alibi
or use_interleaved_v_cache
):
pytest.skip()
@@ -201,6 +207,20 @@ def test_paged_attention(
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
standard_value_cache = value_cache
if use_interleaved_v_cache:
# Hand the kernel V in the interleaved layout (same 4D shape); the
# reference below gets the standard layout back.
x_v = 16 // value_cache.element_size()
value_cache = (
value_cache.view(
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
)
# Call the paged attention kernel.
output = torch.empty_like(query)
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
@@ -234,6 +254,7 @@ def test_paged_attention(
kv_cache_dtype,
k_scale,
v_scale,
use_interleaved_v_cache=use_interleaved_v_cache,
)
opcheck(
@@ -259,10 +280,13 @@ def test_paged_attention(
v_scale,
None,
"f16",
use_interleaved_v_cache,
),
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
)
value_cache = standard_value_cache
# Run the reference implementation.
if kv_cache_dtype == "fp8":
# Convert cache data back to dtype.
+90 -2
View File
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention(
num_heads: int,
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv(
num_heads: int,
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
dtype: torch.dtype,
kv_cache_dtype: str,
device: str,
interleave_v: bool,
block_size: int = 32,
) -> None:
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
def test_contexted_kv_attention_cached_kv_block_table_boundary(
device: str, interleave_v: bool
) -> None:
# Boundary guard for the KV_FROM_CACHE block-table load. With an
# exact-sized block table (row length == number of blocks for the
# sequence) and a query that ends the sequence, the last K/V tile has
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(num_blocks, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi(
num_heads: int,
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_f32(
num_heads: int,
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention(
num_heads,
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi_f32(
num_heads: int,
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention_alibi(
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
num_heads,
num_queries_per_kv,
head_size,
dtype,
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_qwen3_nonstandard_block_size(
num_heads: int,
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
dtype: torch.dtype,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
if not current_platform.is_rocm():
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
kv_cache_dtype="auto",
device=device,
op=op,
interleave_v=interleave_v,
)
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
rtol=rtol,
)
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
def test_asm_kv_scale_tables_shared_across_builders():
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
return the same dense table, a different scale a different one."""
from vllm.v1.attention.backends.rocm_aiter_fa import (
AiterFlashAttentionMetadataBuilder,
AsmKvScaleTables,
)
tables = AsmKvScaleTables()
device = torch.device("cuda")
table = tables.get(device, 8, 2, 16, 0.5)
assert tables.get(device, 8, 2, 16, 0.5) is table
assert tables.get(device, 8, 2, 16, 1.0) is not table
assert table.shape == (8, 2, 16)
assert table.dtype == torch.float32
assert torch.all(table == 0.5)
# The builder class holds the single process-wide instance.
assert isinstance(
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
)
@@ -1989,8 +1989,8 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
"""Replay V4.1 ROCm decode after device-only query reallocation."""
from tests.v1.attention.utils import create_vllm_config
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
compute_global_topk_ragged_indices_and_indptr,
)
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
@@ -2044,7 +2044,7 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
mla_spec, ["c2a"], vllm_config, device
)
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_builder = DeepseekV41ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["c2a"], vllm_config, device
)
assert (
File diff suppressed because it is too large Load Diff
+179 -15
View File
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -65,7 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.models.llama4 import Llama4MoE
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import next_power_of_2, round_up
from vllm.utils.torch_utils import set_random_seed
try:
@@ -788,7 +789,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
def _make_mxfp8_moe_weights(
e: int, n: int, k: int
e: int, n: int, k: int, is_gated: bool = True
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q, s = zip(
@@ -796,8 +797,9 @@ def _make_mxfp8_moe_weights(
)
return torch.stack(q), torch.stack(s)
w1_rows = 2 * n if is_gated else n
w1, w1_scale = quantize(
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
torch.randn(e, w1_rows, k, device="cuda", dtype=torch.bfloat16) / 10
)
w2, w2_scale = quantize(
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
@@ -806,16 +808,20 @@ def _make_mxfp8_moe_weights(
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
@pytest.mark.parametrize("transposed", [True, False], ids=["transposed", "row-major"])
@pytest.mark.parametrize("m", [1, 16])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_fp8_block_moe_deferred_finalize(
m: int, block_shape: list[int], workspace_init
m: int, block_shape: list[int], transposed: bool, workspace_init
):
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
finalize to the caller."""
finalize to the caller. For fp8-block, ``transposed`` toggles the oracle's
is_scale_transposed column-major activation-scale layout."""
if block_shape != [128, 128] and transposed:
pytest.skip("column-major scales apply to fp8 [128, 128] block only")
e, topk, n, k = 32, 4, 1024, 1024
set_random_seed(7)
with set_current_vllm_config(vllm_config):
@@ -849,6 +855,9 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
a2_scale=None,
block_shape=block_shape,
)
is_block = block_shape == [128, 128]
if is_block and not transposed:
quant_config = replace(quant_config, is_scale_transposed=False)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
@@ -866,19 +875,27 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config, quant_config=quant_config, allow_new_interface=True
),
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
)
def make_kernel(quant_config: FusedMoEQuantConfig) -> mk.FusedMoEKernel:
return mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
),
TrtLlmFp8ExpertsModular(
moe_config=moe_config, quant_config=quant_config
),
)
kernel = make_kernel(quant_config)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
def apply(k_: mk.FusedMoEKernel) -> torch.Tensor:
return k_.apply(
hidden_states=a,
w1=w1,
w2=w2,
@@ -888,7 +905,34 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
)
if is_block and transposed:
# The oracle gates column-major scales on TRTLLM fp8-block.
assert quant_config.is_scale_transposed
assert not make_fp8_moe_quant_config(
Fp8MoeBackend.DEEPGEMM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
).is_scale_transposed
# no_dp_ep emits column-major [M, G] scales (physical [G, M]), so
# the experts' a1q_scale.t().contiguous() is a no-op.
_, a1q_scale, _, _, _ = kernel.prepare_finalize.prepare(
a, topk_weights, topk_ids, e, None, False, quant_config
)
assert a1q_scale.t().is_contiguous()
# Bitwise-equal to running with row-major scales.
row_major = replace(quant_config, is_scale_transposed=False)
torch.testing.assert_close(
apply(kernel), apply(make_kernel(row_major)), atol=0, rtol=0
)
check_deferred_moe_finalize(
moe_config,
lambda: apply(kernel),
router_weights=topk_weights,
)
@@ -986,3 +1030,123 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
)
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("activation", "n", "k"),
[
(MoEActivation.SILU, 1024, 1024),
(MoEActivation.SILU, 928, 1024),
(MoEActivation.SILU, 1024, 2688),
(MoEActivation.RELU2_NO_MUL, 1024, 1024),
# Nemotron-3.5 Lightning's non-gated MoE: intermediate 1856, hidden 2688.
(MoEActivation.RELU2_NO_MUL, 1856, 2688),
],
)
@pytest.mark.parametrize("m", [1, 32])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_mxfp8_moe_pads_intermediate(
m: int, activation: MoEActivation, n: int, k: int, workspace_init
):
"""TRTLLM-Gen MXFP8 experts need the intermediate size to be a multiple
of 128 and the hidden size a multiple of 256. Misaligned experts, gated or
not, are padded at load and, with the activations padded and the output
cropped as the MoE runner does, must still match the unquantized reference
on the dequantized weights."""
e, topk = 8, 2
is_gated = activation.is_gated
block_shape = [1, 32]
set_random_seed(7)
with set_current_vllm_config(vllm_config):
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k, is_gated)
w1_ref = dequant_mxfp8_to_bf16(w1, w1_scale)
w2_ref = dequant_mxfp8_to_bf16(w2, w2_scale)
layer = SimpleNamespace(
weight_block_size=block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=is_gated, intermediate_size_per_partition=n
),
activation=activation,
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.FLASHINFER_TRTLLM,
layer,
w1,
w2,
w1_scale,
w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
padded_n = layer.moe_config.intermediate_size_per_partition
padded_k = layer.moe_config.hidden_dim
assert padded_n == round_up(n, 128)
assert padded_k == round_up(k, 256)
quant_config = make_fp8_moe_quant_config(
Fp8MoeBackend.FLASHINFER_TRTLLM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=padded_k,
intermediate_size=padded_n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.Renormalize,
max_num_tokens=next_power_of_2(m),
)
# The monolithic experts pass the activation type to the kernel; the
# modular ones always run it gated.
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=True,
),
TrtLlmFp8ExpertsMonolithic(
moe_config=moe_config, quant_config=quant_config
),
)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
output = kernel.apply_monolithic(
hidden_states=torch.nn.functional.pad(a, (0, padded_k - k)),
w1=w1,
w2=w2,
router_logits=logits,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
routed_scaling_factor=1.0,
routing_sink=None,
)[:, :k]
topk_weights, topk_ids, _ = fused_topk(a, logits, topk, renormalize=True)
reference = fused_experts(
a,
w1_ref,
w2_ref,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=e,
expert_map=None,
)
check_accuracy(ref_output=reference, actual_output=output)
+14 -1
View File
@@ -42,13 +42,15 @@ ROCM_AVAILABLE = current_platform.is_rocm()
ROCM_TRITON_KERNELS_AVAILABLE = False
ROCM_AITER_AVAILABLE = is_aiter_found()
ROCM_GFX950 = False
ROCM_GFX1250 = False
if ROCM_AVAILABLE:
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
from vllm.utils.import_utils import has_triton_kernels
ROCM_TRITON_KERNELS_AVAILABLE = has_triton_kernels()
ROCM_GFX950 = on_gfx950()
ROCM_GFX1250 = on_gfx1250()
if ROCM_AITER_AVAILABLE:
from aiter.ops.triton.quant import dynamic_mxfp4_quant
@@ -1568,6 +1570,15 @@ ROCM_BACKEND_CONFIGS = {
"requires_aiter": True,
"requires_gfx950": True,
},
"AITER_TRITON_MXFP4_MXFP4": {
"activation": "SILU",
"act_type": "mxfp4",
"rtol": 1.0,
"percent": 0.8,
"requires_aiter": True,
"requires_gfx950": False,
"requires_gfx1250": True,
},
}
@@ -1607,6 +1618,8 @@ def test_rocm_mxfp4_moe_oracle(
pytest.skip(f"Backend {backend_name} requires AITER")
if config["requires_gfx950"] and not ROCM_GFX950:
pytest.skip(f"Backend {backend_name} requires GFX950")
if config.get("requires_gfx1250") and not ROCM_GFX1250:
pytest.skip(f"Backend {backend_name} requires GFX1250")
import vllm.distributed.parallel_state as ps
from vllm.config import VllmConfig, set_current_vllm_config
+166
View File
@@ -219,3 +219,169 @@ def test_rocm_aiter_grouped_topk_torch_compile_compatibility():
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
)
assert torch.allclose(topk_ids_original, topk_ids_compiled)
def test_rocm_aiter_topk_gating_custom_op_registration():
assert callable(torch.ops.vllm.rocm_aiter_topk_gating)
def _sort_routing(weights: torch.Tensor, ids: torch.Tensor):
ids_sorted, order = torch.sort(ids, dim=-1)
weights_sorted = torch.gather(weights, 1, order)
return weights_sorted, ids_sorted
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 64])
@pytest.mark.parametrize("num_experts", [256, 512])
@pytest.mark.parametrize("topk", [8, 10])
@pytest.mark.parametrize("renormalize", [True, False])
def test_rocm_aiter_topk_gating_matches_softmax_reference(
num_tokens: int,
num_experts: int,
topk: int,
renormalize: bool,
):
"""Numerical check vs softmax + topk, including Qwen3.8 (E=512, k=10)."""
torch.manual_seed(0)
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.bfloat16, device="cuda"
)
scores = torch.softmax(gating_output.float(), dim=-1)
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda")
topk_ids = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda")
token_expert_indices = torch.empty(
(num_tokens, topk), dtype=torch.int32, device="cuda"
)
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize,
0,
"",
)
# BF16 logits can tie at the top-k boundary, so a different tied expert is
# valid. Check selection against the cutoff and weights against the scores
# of the experts the kernel actually selected.
assert ((topk_ids >= 0) & (topk_ids < num_experts)).all()
sorted_ids = torch.sort(topk_ids, dim=-1).values
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
selected_scores = torch.gather(scores, 1, topk_ids.long())
assert (selected_scores >= cutoff).all()
expected_weights = selected_scores
if renormalize:
expected_weights = expected_weights / expected_weights.sum(dim=-1, keepdim=True)
torch.testing.assert_close(topk_weights, expected_weights, atol=2e-2, rtol=2e-2)
def test_rocm_aiter_topk_gating_shared_expert_suffix():
"""Fused shared expert: softmax top-k on the prefix, sigmoid appended.
Matches the Qwen3.8 FSE launch: 512 routed experts, top-10, one shared
column, weights wide enough for the suffix, and the shared id prefilled.
"""
torch.manual_seed(0)
num_tokens, num_experts, topk, num_shared = 4, 512, 10, 1
routed = torch.randn((num_tokens, num_experts), dtype=torch.bfloat16, device="cuda")
shared = torch.randn((num_tokens, num_shared), dtype=torch.bfloat16, device="cuda")
shared[0, 0] = 40
gating_output = torch.cat([routed, shared], dim=-1).contiguous()
width = topk + num_shared
shared_ids = torch.arange(
num_experts, num_experts + num_shared, device="cuda", dtype=torch.int32
)
topk_ids = torch.full((num_tokens, width), -1, dtype=torch.int32, device="cuda")
topk_ids[:, topk:] = shared_ids
topk_weights = torch.empty((num_tokens, width), dtype=torch.float32, device="cuda")
token_expert_indices = torch.empty(
(num_tokens, topk), dtype=torch.int32, device="cuda"
)
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids[:, :topk],
token_expert_indices,
gating_output,
True,
num_shared,
"sigmoid",
)
scores = torch.softmax(routed.float(), dim=-1)
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
routed_ids = topk_ids[:, :topk]
assert ((routed_ids >= 0) & (routed_ids < num_experts)).all()
sorted_ids = torch.sort(routed_ids, dim=-1).values
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
selected = torch.gather(scores, 1, routed_ids.long())
assert (selected >= cutoff).all()
expected = selected / selected.sum(dim=-1, keepdim=True)
torch.testing.assert_close(topk_weights[:, :topk], expected, atol=2e-2, rtol=2e-2)
torch.testing.assert_close(
topk_weights[:, topk:],
torch.sigmoid(shared.float()),
atol=2e-2,
rtol=2e-2,
)
assert torch.equal(topk_ids[:, topk:], shared_ids.expand_as(topk_ids[:, topk:]))
def test_rocm_aiter_topk_gating_torch_compile_compatibility():
token, expert, topk, renormalize = 4, 512, 10, True
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
device = gating_output.device
def gating_fn(gating_output, topk_weights, topk_ids, token_expert_indices):
return torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize,
0,
"",
)
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_topk_gating,
(
torch.empty((token, topk), dtype=torch.float32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
gating_output,
),
kwargs={
"renormalize": renormalize,
"num_shared_experts": 0,
"shared_expert_scoring_func": "",
},
test_utils=("test_faketensor"),
)
compiled_fn = torch.compile(
gating_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
def alloc():
return (
torch.empty((token, topk), dtype=torch.float32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
)
w0, i0, t0 = alloc()
w1, i1, t1 = alloc()
gating_fn(gating_output, w0, i0, t0)
compiled_fn(gating_output, w1, i1, t1)
w0, i0 = _sort_routing(w0, i0)
w1, i1 = _sort_routing(w1, i1)
assert torch.allclose(w0, w1, rtol=1e-2, atol=1e-2)
assert torch.equal(i0, i1)
+57 -29
View File
@@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt(
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables must reach backend selection; modular AITER remains eligible."""
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
from dataclasses import replace
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
from vllm.platforms import rocm
# Exercise both native MX and monolithic AITER selection on any ROCm device.
@@ -130,14 +109,63 @@ def test_hash_routing_backend_selection(
),
prefix="hash_routing_selection",
)
config = replace(layer.routed_experts.moe_config, moe_backend=backend)
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
if has_hash_routing and backend == "aiter_triton_mxfp4_bf16":
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
else:
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables reach backend selection and hash-capable kernels."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
layer, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, has_hash_routing, backend
)
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
# Monolithic kernels route internally, so they read the table from the layer.
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
def test_hash_routing_rejects_monolithic_without_support(
dist_init, default_vllm_config, monkeypatch
):
"""A monolithic kernel that does not implement hash routing is rejected."""
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
monkeypatch.setattr(
AiterW4A16ExpertsMonolithic,
"_supports_hash_routing",
staticmethod(lambda: False),
)
_, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
)
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
+18 -2
View File
@@ -50,7 +50,7 @@ SKINNY_GEMM_PASS_RATES = {
}
PRESHUFFLED_SHAPES = [
(64, 4096, 8192),
(32, 8192, 16384),
(32, 16384, 16384),
]
@@ -421,8 +421,12 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
M, K, N = shape
assert M <= 64
# The predicate takes K as packed bytes, matching `weight.shape[1]`.
# The predicate takes K as packed bytes, matching `weight.shape[1]`, and
# must agree with the config the kernel itself resolves.
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
assert get_gemm_config("GEMM-AFP4WFP4_PRESHUFFLED", M, N, K)[1]
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
@@ -464,6 +468,18 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
_assert_deterministic(run_preshuffled, n_runs=3)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_preshuffled_tuned_gate_uses_logical_k():
"""AITER ships N=16384-K=16384 but not N=16384-K=8192, so a K=8192 layer
has no tuned preshuffled config even though twice its K does."""
_assert_aiter_supported()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned.cache_clear()
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 16384 // 2)
assert not rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 8192 // 2)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_gemm_a4w4_determinism():
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
+528 -21
View File
@@ -14,6 +14,7 @@ These tests cover:
import math
from types import SimpleNamespace
import numpy as np
import pytest
import torch
@@ -157,6 +158,104 @@ def test_dspark_context_kv_matches_query_insert(
torch.testing.assert_close(kv, before, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("kv_mxfp8", [False, True])
def test_v41_pcp_swa_cache_matches_non_pcp_writer(kv_mxfp8, monkeypatch):
"""PCP SWA writes must preserve the established CUDA cache bytes."""
from vllm.models.deepseek_v41 import attention as attention_module
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
torch.manual_seed(43)
block_size = 64
num_local_tokens = 5
num_tokens = 2 * num_local_tokens
row_size = 528 if kv_mxfp8 else 584
num_blocks = 2
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
backing = torch.full(
(num_blocks, page_stride), 165, dtype=torch.uint8, device="cuda"
)
cache = backing.as_strided(
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
)
reference_backing = backing.clone()
reference = reference_backing.as_strided(cache.shape, cache.stride())
local_kv = torch.randn(num_local_tokens, 512, dtype=torch.bfloat16, device="cuda")
remote_kv = torch.randn_like(local_kv)
gathered_kv = torch.cat((local_kv, remote_kv))
local_positions = torch.arange(num_local_tokens, device="cuda") + 3
cache_positions = torch.cat((local_positions, local_positions + 11))
cache_slots = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device="cuda"
)[:num_tokens]
cache_slots[2] = -1
angles = torch.randn(32, 32, device="cuda")
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
num_heads = 8
q = torch.randn(
num_local_tokens,
num_heads,
512,
dtype=torch.bfloat16,
device="cuda",
)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch.zeros(num_tokens, num_heads, 512, dtype=torch.bfloat16, device="cuda"),
gathered_kv,
reference.view(num_blocks, -1),
cache_slots,
cache_positions,
cos_sin,
num_heads,
1e-20,
block_size,
False,
kv_mxfp8,
)
monkeypatch.setattr(
attention_module,
"get_pcp_group",
lambda: SimpleNamespace(
all_gather=lambda tensor, dim: torch.cat((tensor, remote_kv), dim=dim)
),
)
metadata = SimpleNamespace(
slot_mapping=torch.full(
(num_local_tokens,), -1, dtype=torch.int64, device="cuda"
),
cache_slot_mapping=cache_slots,
cache_positions=cache_positions,
block_size=block_size,
)
attention = SimpleNamespace(
use_pcp=True,
swa_cache_layer=SimpleNamespace(
prefix="swa", kv_cache=cache, block_size=block_size
),
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
_flashinfer_fp8_kv_scale=None,
kv_mxfp8=kv_mxfp8,
accepts_unnormed_unroped_query=False,
n_local_heads=num_heads,
padded_heads=num_heads,
eps=1e-20,
head_dim=512,
)
DeepseekV4Attention._fused_qnorm_rope_kv_insert(
attention, q, local_kv, local_positions, {"swa": metadata}
)
torch.testing.assert_close(
backing.view(torch.uint8),
reference_backing.view(torch.uint8),
rtol=0,
atol=0,
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="graph capture coverage"
)
@@ -360,7 +459,7 @@ def _mxfp8_record_reference(
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
"""The 528-byte V4.1 record quantizes the RoPE dims instead of keeping them
in bf16, so a page is [block_size x 512 fp8][block_size x 16 UE8M0].
in bf16, so a token is [512 fp8][16 UE8M0].
Byte-exact against the reference quantizer, including the rows and page
padding the kernel must leave alone.
@@ -398,9 +497,8 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
values, scales = _mxfp8_record_reference(
latent[t], cos_sin[pos // compress_ratio * compress_ratio]
)
expected[page, row * 512 : (row + 1) * 512] = values
scale_offset = cache_block * 512 + row * 16
expected[page, scale_offset : scale_offset + 16] = scales
expected[page, row * 528 : row * 528 + 512] = values
expected[page, row * 528 + 512 : (row + 1) * 528] = scales
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
@@ -510,14 +608,13 @@ def test_v41_mxfp8_cache_round_trip():
values, scales = _ue8m0_reference(k[t].float(), 32, 448.0)
page, row = divmod(t, block_size)
torch.testing.assert_close(
cache[page, row * 512 : (row + 1) * 512],
cache[page, row * 528 : row * 528 + 512],
values.view(torch.uint8),
rtol=0,
atol=0,
)
scale_offset = block_size * 512 + row * 16
torch.testing.assert_close(
cache[page, scale_offset : scale_offset + 16],
cache[page, row * 528 + 512 : (row + 1) * 528],
(scales.log2() + 127).to(torch.uint8),
rtol=0,
atol=0,
@@ -564,7 +661,8 @@ def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tenso
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales
per token.
Scales are byte-exact against the reference (``amax / 6`` clamped to the
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
@@ -604,17 +702,13 @@ def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
amax = rotated.view(32, 16).abs().amax(-1)
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
record = cache_backing[page, row * 288 : (row + 1) * 288]
got_scales = record[256:]
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
decoded = _decode_nvfp4_row(
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
)
decoded = _decode_nvfp4_row(record[:256], got_scales)
step = scale.float().repeat_interleave(16)
assert ((decoded - rotated).abs() <= step).all()
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
page, row * 256 : (row + 1) * 256
]
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
untouched[page, row * 288 : (row + 1) * 288] = record
# Rows the kernel must not have touched, page padding included.
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
@@ -670,10 +764,8 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
offset=offset,
)
expected = _decode_nvfp4_row(
backing[:, : block_size * 256].reshape(-1, 256),
backing[:, block_size * 256 : block_size * 288].reshape(-1, 32),
)
records = backing[:, : block_size * 288].reshape(-1, 288)
expected = _decode_nvfp4_row(records[:, :256], records[:, 256:])
def check(shorter_by):
assert (storage[:, num_tokens + offset :] == -123).all()
@@ -702,6 +794,72 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
check(1)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="The V4.1 records are SM100-only",
)
@pytest.mark.parametrize("record", ["swa_fp8", "compressed_fp8", "compressed_fp4"])
def test_v41_cache_writers_match_flashmla_decode(record: str):
"""FlashMLA and vLLM's gathers read back the token a V4.1 writer put mid
page in a padded block, so all agree on the interleaved record layout."""
import vllm.v1.attention.ops.flashmla as fm
from vllm.models.deepseek_v41.common.ops.cache_utils import (
dequantize_and_gather_k_cache,
dequantize_and_gather_k_cache_triton,
)
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
device, block_size, topk = "cuda", 64, 64
slot = block_size + 37
bytes_per_token = 288 if record == "compressed_fp4" else 528
torch.manual_seed(0)
kv = torch.randn(1, 512, dtype=torch.bfloat16, device=device)
positions = torch.tensor([slot], device=device)
# cos = 1, sin = 0: RoPE is the identity, so the cache should hold `kv`.
cos_sin = torch.zeros(slot + 1, 64, device=device)
cos_sin[:, :32] = 1
# Padded like a packed block: a multiple of the record and of 512 B.
cache = torch.zeros(2, block_size + 32, bytes_per_token, dtype=torch.uint8)
cache = cache.to(device)[:, :block_size]
if record == "swa_fp8":
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
kv, cache, positions, positions, cos_sin, block_size, None, True
)
else:
rope_quant_insert(kv, positions, cos_sin, cache, positions, 1)
indices = torch.full((1, 1, topk), -1, dtype=torch.int32, device=device)
indices[..., 0] = slot
extra = None
if record == "compressed_fp4":
cache, extra = torch.zeros(2, block_size, 528).to(cache), cache
out, _ = fm.flash_mla_with_kvcache(
q=torch.randn(1, 1, 64, 512, dtype=torch.bfloat16, device=device),
k_cache=cache.unsqueeze(2),
block_table=None,
cache_seqlens=None,
head_dim_v=512,
tile_scheduler_metadata=fm.get_mla_metadata()[0],
is_fp8_kvcache=True,
indices=indices if extra is None else torch.full_like(indices, -1),
extra_k_cache=None if extra is None else extra.unsqueeze(2),
extra_indices_in_kvcache=None if extra is None else indices,
)
for gather in (dequantize_and_gather_k_cache, dequantize_and_gather_k_cache_triton):
rows = torch.zeros(1, slot + 1, 512, dtype=torch.bfloat16, device=device)
block_table = torch.arange(2, dtype=torch.int32, device=device)[None]
seq_lens = torch.tensor([slot + 1], dtype=torch.int32, device=device)
k_cache = cache if extra is None else extra
gather(rows, k_cache, seq_lens, None, block_table, block_size, offset=0)
torch.testing.assert_close(
out[0, 0], rows[0, slot].expand(64, -1), rtol=0, atol=0
)
cos = torch.nn.functional.cosine_similarity(rows[0, slot].float(), kv[0].float(), 0)
assert cos > (0.98 if extra is not None else 0.999)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
@pytest.mark.parametrize("store_fp8", [False, True])
@@ -796,6 +954,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
capacity = 8
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_batched_tokens = 16
vllm_config.parallel_config.prefill_context_parallel_size = 1
spec = CircularBufferSpec(
block_size=capacity,
num_kv_heads=1,
@@ -832,6 +991,67 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
def test_v41_pcp_compressor_latent_is_gathered_for_cache_writes(monkeypatch):
"""PCP compresses locally and publishes the gathered latent to both caches."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41 import attention as attention_module
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
local_latent = torch.arange(8, dtype=torch.float32).view(2, 4)
cache_positions = torch.tensor([0, 1, 10, 11])
observed = {}
class Compressor:
def __call__(self, kv_score, positions):
return local_latent
def insert_cache(self, latent, positions, rotary_emb):
observed["insert"] = (latent, positions)
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
observed["indexer"] = (latent, kwargs)
return None, None, None
monkeypatch.setattr(
attention_module,
"get_pcp_group",
lambda: SimpleNamespace(
all_gather=lambda tensor, dim: torch.cat((tensor, tensor + 10), dim=dim)
),
)
metadata = {"swa": SimpleNamespace(cache_positions=cache_positions)}
attention = SimpleNamespace(
use_pcp=True,
compressor=Compressor(),
indexer=prepare_indexer,
aux_stream_list=None,
ln_events=[None, None],
swa_cache_layer=SimpleNamespace(prefix="swa"),
rotary_emb=SimpleNamespace(),
indexer_rotary_emb=SimpleNamespace(),
n_local_heads=1,
head_dim=4,
_wq_b_proj=lambda qr, scale: qr,
_fused_qnorm_rope_kv_insert=lambda q, kv, pos, meta: q,
_sparse_indexer_and_attn=lambda *args: None,
)
rows = torch.zeros(2, 4)
positions = torch.arange(2)
with override_forward_context(ForwardContext({}, metadata, {})):
DeepseekV4Attention._prepare_and_attn(
attention, rows, rows, rows, None, rows, rows, positions, rows
)
expected_global = torch.cat((local_latent, local_latent + 10))
insert_latent, insert_positions = observed["insert"]
torch.testing.assert_close(insert_latent, expected_global)
torch.testing.assert_close(insert_positions, cache_positions)
indexer_latent, indexer_kwargs = observed["indexer"]
torch.testing.assert_close(indexer_latent, local_latent)
torch.testing.assert_close(indexer_kwargs["cache_latent"], expected_global)
torch.testing.assert_close(indexer_kwargs["cache_positions"], cache_positions)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
@pytest.mark.parametrize(
"use_aux,use_graph", [(False, False), (True, False), (True, True)]
@@ -871,6 +1091,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
context = ForwardContext({}, metadata, {})
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
torch.nn.Module.__init__(compressor)
compressor.pcp_size = 1
compressor.head_dim, compressor.rope_head_dim, compressor.compress_ratio = (
512,
64,
@@ -896,7 +1117,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
use_fp4_kv=False,
)
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale):
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
DeepseekV4Indexer._produce_k(indexer, latent, positions, rotary)
return None, None, None
@@ -906,6 +1127,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
auxiliary = [torch.cuda.Stream()] if use_aux else None
attention = SimpleNamespace(
use_pcp=False,
compressor=compressor,
indexer=prepare_indexer,
aux_stream_list=auxiliary,
@@ -963,6 +1185,291 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
previous = expected
@pytest.mark.parametrize("pcp_size", [2, 4])
@pytest.mark.parametrize(
"lengths,prefilling,starts",
[
([9, 1], [True, False], [1, 4]),
([1, 1], [False, False], [16, 3]),
([3, 1], [True, False], [1, 0]),
],
)
def test_v41_pcp_ring_links_match_request_positions(
pcp_size, lengths, prefilling, starts, monkeypatch
):
"""Cross-chunk predecessors and open ring writes must belong to the request."""
from vllm.models.deepseek_v41 import compressor as compressor_module
from vllm.models.deepseek_v41.compressor import (
CompressorMetadata,
CompressorMetadataBuilder,
)
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
from vllm.v1.worker.gpu.pcp_manager import PCPManager
monkeypatch.setattr(
pcp_manager_module,
"async_tensor_h2d",
lambda array, **kwargs: torch.from_numpy(array),
)
manager = PCPManager(pcp_size, 0, torch.device("cpu"), shard_decode_requests=True)
query_start = np.array([0, *np.cumsum(lengths)], dtype=np.int32)
segments, counts = manager._build_batch_layout(
np.array(lengths), np.array(starts), np.array(prefilling), query_start
)
width = max(counts)
capacity = 8
blocks = [1, (1 << 20) + 2]
num_rows = min(width, 2 * len(lengths))
metadata_by_rank = []
positions_by_rank = []
gathered_keys = []
for local_segments in segments:
positions = torch.zeros(width, dtype=torch.int64)
slots = torch.full((width,), -1, dtype=torch.int64)
local_query_start = [0]
keys = [-1] * num_rows
for row, segment in enumerate(local_segments):
req = segment.global_batch_req_idx
begin = starts[req] + segment.global_batch_slice.start - query_start[req]
local = segment.rank_local_batch_slice
positions[local] = torch.arange(begin, begin + segment.num_tokens)
slots[local] = blocks[req] * capacity + positions[local] % capacity
local_query_start.append(local.stop)
keys[row] = (blocks[req] << 32) | int(positions[local.stop - 1])
if not local_segments:
local_query_start.append(0)
metadata_by_rank.append(
CompressorMetadata(
slot_mapping=slots,
query_start_loc=torch.tensor(local_query_start, dtype=torch.int32),
token_to_req_indices=torch.empty(0, dtype=torch.int32),
)
)
positions_by_rank.append(positions)
gathered_keys.extend(keys)
keys = torch.tensor(gathered_keys, dtype=torch.int64)
contributed_keys = []
def all_gather(tensor, dim):
contributed_keys.append(tensor.clone())
return keys
monkeypatch.setattr(
compressor_module,
"get_pcp_group",
lambda: SimpleNamespace(all_gather=all_gather),
)
builder = SimpleNamespace(
capacity=capacity,
device=torch.device("cpu"),
vllm_config=SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_seqs=len(lengths))
),
)
expected_rows = {
block: max(key for key in gathered_keys if key >> 32 == block)
for block in blocks
}
expected_slots = [
block * capacity + (key & 0xFFFFFFFF) % capacity
for block, key in expected_rows.items()
if key % 2 == 0
]
for metadata, positions in zip(metadata_by_rank, positions_by_rank):
num_reqs = metadata.query_start_loc.numel() - 1
CompressorMetadataBuilder._link_pcp_chunks(
builder, metadata, num_reqs, positions
)
assert metadata.prev_row_indices is not None
assert metadata.gather_tokens is not None
assert metadata.ring_write_slots is not None
for row, token in enumerate(metadata.gather_tokens.tolist()):
key = int(contributed_keys[-1][row])
if key >= 0:
assert int(metadata.slot_mapping[token]) // capacity == key >> 32
assert int(positions[token]) == key & 0xFFFFFFFF
for start, prev in zip(
metadata.query_start_loc[:-1].tolist(), metadata.prev_row_indices.tolist()
):
position = int(positions[start])
block = int(metadata.slot_mapping[start]) // capacity
wanted = (block << 32) | (position - 1)
if position % 2 == 1 and wanted in gathered_keys:
assert prev >= 0 and gathered_keys[prev] == wanted
else:
assert prev == -1
writes = metadata.ring_write_slots.tolist()
assert sorted(slot for slot in writes if slot >= 0) == sorted(expected_slots)
for row, slot in enumerate(writes):
if slot >= 0:
assert gathered_keys[row] == expected_rows[slot // capacity]
assert (gathered_keys[row] & 0xFFFFFFFF) % capacity == slot % capacity
torch.testing.assert_close(torch.cat(contributed_keys), keys)
def _build_pcp_ring_metadata(ranks, all_gather, spec, config, device, monkeypatch):
from vllm.models.deepseek_v41 import compressor as compressor_module
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
monkeypatch.setattr(
compressor_module,
"get_pcp_group",
lambda: SimpleNamespace(all_gather=all_gather),
)
for local in ranks:
builder = CompressorMetadataBuilder(spec, ["ring"], config, device)
local.metadata = builder.build(0, local.common)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Triton kernel")
@pytest.mark.parametrize("pcp_size", [2, 4])
def test_v41_pcp_compressor_preserves_pairs_and_decode_state(pcp_size, monkeypatch):
"""Pairs split across PCP chunks and decode owner changes keep exact latents."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41 import compressor as compressor_module
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
fused_save_compress_norm,
)
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import CircularBufferSpec
from vllm.v1.worker.gpu.pcp_manager import PCPManager
torch.manual_seed(42)
device = torch.device("cuda")
norm = torch.randn(512, dtype=torch.bfloat16, device=device)
# Block 0 is reserved as the null block; requests use physical blocks 1/2.
reference_state = torch.zeros(3, 8, 1024, device=device)
rank_states = [reference_state.clone() for _ in range(pcp_size)]
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=4),
parallel_config=SimpleNamespace(prefill_context_parallel_size=pcp_size),
speculative_config=None,
)
spec = CircularBufferSpec(
block_size=8, num_kv_heads=1, head_size=1024, head_size_v=0, dtype=torch.float32
)
starts = np.zeros(2, dtype=np.int32)
for lengths, prefilling in [
([7, 3], [True, True]),
([9, 1], [True, False]),
([1, 1], [False, False]),
([1, 1], [False, False]),
]:
lengths = np.array(lengths, dtype=np.int32)
query_start = np.concatenate(([0], np.cumsum(lengths))).astype(np.int32)
qsl = torch.from_numpy(query_start).to(device)
positions = torch.cat(
[torch.arange(s, s + n, device=device) for s, n in zip(starts, lengths)]
)
req_ids = torch.repeat_interleave(
torch.arange(2, device=device, dtype=torch.int32),
torch.from_numpy(lengths).to(device),
)
slots = (req_ids.long() + 1) * 8 + positions % 8
raw = torch.randn(sum(lengths), 1024, device=device)
expected = torch.empty(sum(lengths), 512, dtype=torch.bfloat16, device=device)
fused_save_compress_norm(
raw,
positions,
reference_state,
slots,
qsl,
req_ids,
norm,
1e-20,
2,
expected,
)
ranks = []
for rank in range(pcp_size):
manager = PCPManager(pcp_size, rank, device, shard_decode_requests=True)
segments, counts = manager._build_batch_layout(
lengths, starts, np.array(prefilling), query_start
)
width = max(counts)
local_indices = manager._padded_gather_idx[
rank * width : (rank + 1) * width
]
local_lengths = [s.num_tokens for s in segments[rank]] or [0]
local_qsl = torch.tensor([0, *np.cumsum(local_lengths)], dtype=torch.int32)
local_reqs = [s.global_batch_req_idx for s in segments[rank]] or [0]
common = CommonAttentionMetadata(
query_start_loc=local_qsl.to(device),
query_start_loc_cpu=local_qsl,
seq_lens=torch.zeros(len(local_reqs), dtype=torch.int32, device=device),
num_reqs=len(local_reqs),
num_actual_tokens=counts[rank],
max_query_len=max(local_lengths),
max_seq_len=int(max(starts + lengths)),
block_table_tensor=torch.tensor(
np.array(local_reqs) + 1, dtype=torch.int32, device=device
).unsqueeze(1),
slot_mapping=torch.full((width * pcp_size,), -1, device=device),
positions=positions[local_indices],
)
ranks.append(
SimpleNamespace(
indices=local_indices[: counts[rank]],
raw=raw[local_indices],
positions=common.positions,
common=common,
)
)
# Each rank's builder contributes its last-row candidates to a gather.
candidates: list[torch.Tensor] = []
_build_pcp_ring_metadata(
ranks,
lambda t, dim, out=candidates: out.append(t.clone()) or t.repeat(pcp_size),
spec,
config,
device,
monkeypatch,
)
_build_pcp_ring_metadata(
ranks,
lambda t, dim, out=candidates: torch.cat(out),
spec,
config,
device,
monkeypatch,
)
gathered_rows = torch.cat([r.raw[r.metadata.gather_tokens] for r in ranks])
monkeypatch.setattr(
compressor_module,
"get_pcp_group",
lambda rows=gathered_rows: SimpleNamespace(all_gather=lambda t, dim: rows),
)
for rank, local in enumerate(ranks):
compressor = SimpleNamespace(
pcp_size=pcp_size,
head_dim=512,
compress_ratio=2,
rms_norm_eps=1e-20,
norm=SimpleNamespace(weight=norm),
state_cache=SimpleNamespace(prefix="ring", kv_cache=rank_states[rank]),
)
context = ForwardContext({}, {"ring": local.metadata}, {})
with override_forward_context(context):
actual = DeepseekCompressor.forward(
compressor, local.raw, local.positions
)
assert isinstance(actual, torch.Tensor)
valid = (positions[local.indices] + 1) % 2 == 0
torch.testing.assert_close(
actual[: local.indices.numel()][valid],
expected[local.indices][valid],
rtol=0,
atol=0,
)
for state in rank_states[1:]:
torch.testing.assert_close(state, rank_states[0], rtol=0, atol=0)
starts += lengths
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
+14 -5
View File
@@ -502,11 +502,12 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
def test_v2_model_state_gathers_lookback_window():
@pytest.mark.parametrize("pcp", [False, True])
def test_v2_model_state_gathers_lookback_window(pcp):
"""The window is gathered on device from the runner's token history."""
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
depth, max_num_reqs, max_model_len = 3, 4, 16
depth, max_num_reqs, max_model_len = 3, 6, 16
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
state.rope_state = None
state.supports_mm_inputs = False
@@ -527,15 +528,23 @@ def test_v2_model_state_gathers_lookback_window():
# Batch rows -> request state rows: a request two decode steps in, a fresh
# request, and one at the end of a 5-token prompt.
input_batch = SimpleNamespace(
idx_mapping=torch.tensor([2, 0, 3], dtype=torch.int64, device="cuda")
idx_mapping=torch.tensor(
[2, 0, 3, 2] if pcp else [2, 0, 3], dtype=torch.int64, device="cuda"
),
query_start_loc=torch.arange(5 if pcp else 4, dtype=torch.int32, device="cuda"),
positions=torch.tensor([8, 0, 6, 5] if pcp else [8, 0, 6], device="cuda"),
)
window = state.prepare_inputs(input_batch, req_states)["lookback_token_ids"]
assert window.cpu().tolist() == [
expected = [
[22, 21, 16],
[-1, -1, -1],
[0, 0, 0],
[-1, -1, -1],
]
if pcp:
# The second PCP chunk of request 2 starts at 5, not at its global 8.
expected.append([15, 14, 13])
expected.extend([[-1] * depth] * (max_num_reqs - len(expected)))
assert window.cpu().tolist() == expected
# Graph capture runs on the dummy inputs and replays read the same buffer.
dummy = state.prepare_dummy_inputs(num_reqs=3, num_tokens=3)["lookback_token_ids"]
assert dummy is window and torch.all(dummy == -1)
@@ -525,9 +525,8 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
step = 32.0 * tile_scale.repeat_interleave(32, dim=-1)
assert ((rec_fused - rec_ref).abs() <= step).all()
# The NoPE dims see no rotation at all, so they must be byte-identical.
nope_bytes = slice(0, block_size * HEAD_DIM)
ref_rows = k_cache_ref[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
fused_rows = k_cache_fused[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
ref_rows = k_cache_ref.view(num_blocks, block_size, V41_HEAD_BYTES)
fused_rows = k_cache_fused.view(num_blocks, block_size, V41_HEAD_BYTES)
torch.testing.assert_close(
fused_rows[..., :NOPE_DIM], ref_rows[..., :NOPE_DIM], rtol=0, atol=0
)
@@ -109,7 +109,7 @@ def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
@pytest.mark.parametrize(
("input_size", "output_size"),
[(3072, 256), (4096, 8), (4096, 192), (6144, 128), (6144, 256)],
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
)
def test_rocm_gfx950_enables_fp32_router_gemm(
monkeypatch, input_size: int, output_size: int
@@ -147,7 +147,6 @@ def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
[
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
pytest.param(2048, 64, torch.float32, False, True, id="shape"),
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
],
+171 -5
View File
@@ -5,6 +5,9 @@
import pytest
import torch
from vllm.model_executor.layers.fused_moe.router import (
rocm_fp32_router_gemm as rocm_router,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import (
fp32_router_gemm_dispatch_impl,
)
@@ -19,6 +22,11 @@ SHAPES = [
(4096, 192),
(6144, 128),
(6144, 256),
(2560, 512),
(2880, 128),
(5120, 16),
(7168, 384),
(1000, 100),
]
MAX_TOKENS = 32
ATOL = 5e-4
@@ -80,7 +88,7 @@ def test_rocm_fp32_router_gemm_preserves_topk(
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device)
weight = torch.randn(num_experts, hidden_size, dtype=torch.float32, device=device)
top_k = 2 if (hidden_size, num_experts) == (4096, 8) else 8
top_k = 2 if num_experts <= 16 else 8
actual = rocm_fp32_router_gemm(x, weight).topk(top_k, dim=-1).indices
reference = _reference(x, weight)
reference_values, expected = reference.topk(top_k, dim=-1)
@@ -113,15 +121,22 @@ def test_rocm_fp32_router_gemm_rejects_invalid_inputs() -> None:
weight,
)
with pytest.raises(ValueError, match="shape"):
rocm_fp32_router_gemm(x[:, :2048].contiguous(), weight[:, :2048].contiguous())
with pytest.raises(ValueError, match="hidden sizes"):
rocm_fp32_router_gemm(x, weight[:, :2048].contiguous())
with pytest.raises(ValueError, match="num_experts"):
big_weight = torch.empty(1024, 7168, dtype=torch.float32, device=device)
rocm_fp32_router_gemm(
torch.empty(MAX_TOKENS, 7168, dtype=torch.bfloat16, device=device),
big_weight,
)
with pytest.raises(ValueError, match="contiguous"):
wide_x = torch.randn(4, hidden_size * 2, dtype=torch.bfloat16, device=device)
rocm_fp32_router_gemm(wide_x[:, ::2], weight)
with pytest.raises(ValueError, match="float32"):
rocm_fp32_router_gemm(x, weight.to(torch.bfloat16))
with pytest.raises(ValueError, match="float32 or bfloat16"):
rocm_fp32_router_gemm(x, weight.to(torch.float16))
with pytest.raises(ValueError, match="bfloat16 or float32"):
rocm_fp32_router_gemm(x.to(torch.float16), weight)
@@ -226,3 +241,154 @@ def test_rocm_fp32_router_gemm_dynamic_compile_dispatch() -> None:
)
output = compiled_dispatch(x)
torch.testing.assert_close(output, _reference(x, weight), atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize("hidden_size", [64, 100, 513, 1000, 2049, 3000])
@pytest.mark.parametrize("num_tokens", [1, 5, 32])
@torch.inference_mode()
def test_rocm_fp32_router_gemm_any_hidden_size(num_tokens: int, hidden_size: int):
"""Hidden sizes that do not divide the K tile exercise the masked tail."""
torch.manual_seed(6100 + num_tokens + hidden_size)
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(64, hidden_size, dtype=torch.float32, device="cuda")
torch.testing.assert_close(
rocm_fp32_router_gemm(x, weight), _reference(x, weight), atol=ATOL, rtol=RTOL
)
@pytest.mark.parametrize("case", ["large", "near_zero", "zero_input", "ones"])
@torch.inference_mode()
def test_rocm_fp32_router_gemm_special_values(case: str) -> None:
torch.manual_seed(7100)
x = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
if case in ("large", "near_zero"):
scale = 100.0 if case == "large" else 1e-4
x = (x * scale).to(torch.bfloat16)
weight *= scale
elif case == "zero_input":
x.zero_()
else:
x.fill_(1.0)
output = rocm_fp32_router_gemm(x, weight)
expected = _reference(x, weight)
assert torch.isfinite(output).all()
if case == "zero_input":
assert torch.equal(output, torch.zeros_like(output))
else:
assert ((output - expected).norm() / expected.norm()).item() < 1e-5
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
@torch.inference_mode()
def test_rocm_fp32_router_gemm_deterministic(num_tokens: int) -> None:
torch.manual_seed(8100 + num_tokens)
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(384, 7168, dtype=torch.float32, device="cuda")
first = rocm_fp32_router_gemm(x, weight)
assert torch.equal(first, rocm_fp32_router_gemm(x, weight))
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
@pytest.mark.parametrize(("hidden_size", "num_experts"), SHAPES)
@torch.inference_mode()
def test_rocm_fp32_router_gemm_bf16_weight(
num_tokens: int, hidden_size: int, num_experts: int
) -> None:
torch.manual_seed(9100 + num_tokens + hidden_size + num_experts)
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(num_experts, hidden_size, dtype=torch.bfloat16, device="cuda")
# Past this size hipBLASLt beats the kernel for BF16 weights on MI355.
eligible = num_tokens * num_experts <= 8192
assert rocm_router.can_use_rocm_fp32_router_gemm(x, weight) == eligible
if eligible:
torch.testing.assert_close(
rocm_fp32_router_gemm(x, weight),
_reference(x.float(), weight.float()),
atol=ATOL,
rtol=RTOL,
)
def _make_gate_linear(
monkeypatch: pytest.MonkeyPatch,
input_size,
output_size,
params_dtype=torch.float32,
):
for module in ("layers.linear", "parameter"):
monkeypatch.setattr(
f"vllm.model_executor.{module}.get_tensor_model_parallel_rank", lambda: 0
)
monkeypatch.setattr(
f"vllm.model_executor.{module}.get_tensor_model_parallel_world_size",
lambda: 1,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
gate = GateLinear(
input_size=input_size,
output_size=output_size,
bias=False,
out_dtype=torch.float32,
params_dtype=params_dtype,
).cuda()
with torch.no_grad():
gate.weight.normal_()
gate.quant_method.process_weights_after_loading(gate)
return gate
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("num_tokens", [1, 32, 33])
@torch.inference_mode()
def test_gate_linear_uses_rocm_fp32_router_gemm(
monkeypatch: pytest.MonkeyPatch, num_tokens: int, x_dtype: torch.dtype
) -> None:
"""A router shape outside the old tuned list takes the fast path for M<=32."""
gate = _make_gate_linear(monkeypatch, 2560, 512)
assert gate.allow_fp32_router_gemm
calls = []
kernel = rocm_router.rocm_fp32_router_gemm
def spy(hidden_states, router_weight):
calls.append(hidden_states.shape[0])
return kernel(hidden_states, router_weight)
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
x = torch.randn(num_tokens, 2560, dtype=x_dtype, device="cuda")
output, bias = gate(x)
assert bias is None
assert calls == ([num_tokens] if num_tokens <= MAX_TOKENS else [])
torch.testing.assert_close(output, _reference(x, gate.weight), atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize(
("num_tokens", "num_experts", "uses_kernel"),
[(1, 256, True), (32, 256, True), (33, 256, False), (32, 512, False)],
)
@torch.inference_mode()
def test_gate_linear_bf16_weight_uses_rocm_router_gemm(
monkeypatch: pytest.MonkeyPatch, num_tokens, num_experts, uses_kernel
) -> None:
"""BF16-weight gates with FP32 logits use the kernel only where it beats
hipBLASLt; everything else keeps the torch.mm path."""
gate = _make_gate_linear(monkeypatch, 7168, num_experts, torch.bfloat16)
assert gate.allow_cublas_router_gemm
calls = []
kernel = rocm_router.rocm_fp32_router_gemm
def spy(hidden_states, router_weight):
calls.append(hidden_states.shape[0])
return kernel(hidden_states, router_weight)
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
output, _ = gate(x)
assert output.dtype == torch.float32
assert calls == ([num_tokens] if uses_kernel else [])
torch.testing.assert_close(
output, _reference(x.float(), gate.weight.float()), atol=ATOL, rtol=RTOL
)
@@ -830,15 +830,19 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
_assert_same_selection(dense, out, ref)
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
def _build_fp4_paged_cache(
req_lens: torch.Tensor, page_kv: int, head_dim: int, page_pad: int = 0
):
"""MXFP4 paged cache in DeepGEMM's fused layout: per page, all tokens'
fp4 data first, then their packed UE8M0 scales."""
fp4 data first, then their packed UE8M0 scales; ``page_pad`` bytes pad
each page like the other layers' pages in a packed block."""
device = req_lens.device
pages_per = (req_lens + page_kv - 1) // page_kv
num_pages, max_pages = int(pages_per.sum()), int(pages_per.max())
head_bytes = head_dim // 2 + 4
kv_cache = torch.zeros(
num_pages, page_kv, 1, head_dim // 2 + 4, dtype=torch.uint8, device=device
)
num_pages, page_kv * head_bytes + page_pad, dtype=torch.uint8, device=device
)[:, : page_kv * head_bytes].view(num_pages, page_kv, 1, head_bytes)
block_table = torch.zeros(
len(req_lens), max_pages, dtype=torch.int32, device=device
)
@@ -864,7 +868,10 @@ def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
@pytest.mark.parametrize("next_n", [1, 2])
@pytest.mark.parametrize("capture", [False, True])
def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool):
@pytest.mark.parametrize("page_pad", [0, 256])
def test_sparse_mqa_logits_paged_matches_dense_masked(
next_n: int, capture: bool, page_pad: int
):
"""Paged sparse-MQA decode selects exactly the values ``torch.topk`` picks
from the dense bf16 masked logits, incl. MTP rows (next_n > 1 flattened to
one query row each), CUDA graph capture/replay and schedule reuse by a
@@ -886,8 +893,10 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
.clamp(min=0)
.to(torch.int32)
)
kv_cache, block_table = _build_fp4_paged_cache(req_lens, page_kv, head_dim)
assert kv_cache.stride(0) % 512 == 0 # the paged sparse kernel requires it
kv_cache, block_table = _build_fp4_paged_cache(
req_lens, page_kv, head_dim, page_pad
)
assert kv_cache.stride(0) % 512 == page_pad # padded pages are not 512B
q_fp, q_sf = _quant_fp4(
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
@@ -17,6 +17,15 @@ if current_platform.is_cuda():
from vllm.model_executor.layers import utils
def _on_gfx11() -> bool:
"""gfx11 check that is safe to evaluate at collection time on any platform."""
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx11
return on_gfx11()
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
x = torch.randn(1, 64, dtype=torch.float16)
@@ -145,6 +154,101 @@ def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def _row_stride_padded(weight: torch.Tensor, pad: int) -> torch.Tensor:
"""A weight whose row stride is offset by ``pad`` elements, as produced by
``maybe_pad_weight_avoid_cache_aliasing``."""
padded = torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
assert padded.shape == weight.shape
assert padded.stride() == (weight.shape[1] + pad, 1)
return padded
def test_rocm_unquantized_gemm_wvsplitk_accepts_row_stride_padded_weight(monkeypatch):
# wvSplitK reads the weight row stride, so a row-stride-padded weight must
# still reach it instead of falling back to torch.nn.functional.linear.
x = torch.randn(4, 1024, dtype=torch.float16)
weight = _row_stride_padded(torch.randn(128, 1024, dtype=torch.float16), 64)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
assert wvsplitk_mock.call_args.args[0].stride() == weight.stride()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.parametrize(
"k,element_size,expected",
[
(4096, 2, 64), # 8192 B row stride, aliases
(12288, 2, 64), # 24576 B row stride, aliases
(1024, 2, 64), # smallest aliasing bf16/fp16 K
(2560, 2, 0), # 5120 B, no aliasing
(11008, 2, 0), # 22016 B, no aliasing
(2048, 1, 128), # fp8: 2048 B, aliases, twice the pad
(512, 4, 32), # fp32: 2048 B, aliases, half the pad
],
)
def test_cache_aliasing_pad_elems(k, element_size, expected):
assert utils.cache_aliasing_pad_elems(k, element_size) == expected
def test_maybe_pad_weight_avoid_cache_aliasing_skips_non_gfx11(monkeypatch):
# Padding is gfx11-only, even for a K whose row stride would alias there.
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: False)
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
assert utils.maybe_pad_weight_avoid_cache_aliasing(weight) is weight
def test_maybe_pad_weight_avoid_cache_aliasing(monkeypatch):
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
padded = utils.maybe_pad_weight_avoid_cache_aliasing(weight)
assert padded.shape == weight.shape
assert padded.stride() == (4096 + 64, 1)
assert torch.equal(padded, weight)
# Idempotent: the padded stride no longer aliases, so a second call is a no-op
# and preserves the pointer captured by any graph.
repadded = utils.maybe_pad_weight_avoid_cache_aliasing(padded)
assert repadded.data_ptr() == padded.data_ptr()
@pytest.mark.parametrize(
"weight",
[
torch.randn(128, 2560, dtype=torch.bfloat16), # no aliasing
torch.randn(2560, 128, dtype=torch.bfloat16).t(), # not contiguous
torch.randn(2, 128, 4096, dtype=torch.bfloat16), # not 2D
],
)
def test_maybe_pad_weight_avoid_cache_aliasing_skips(monkeypatch, weight):
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
assert (
utils.maybe_pad_weight_avoid_cache_aliasing(weight).data_ptr()
== weight.data_ptr()
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):
@@ -164,6 +268,26 @@ def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch,
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _on_gfx11(), reason="cache-aliasing padding only applies on gfx11"
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_unquantized_gemm_padded_weight_real_kernel(monkeypatch, dtype):
x = torch.randn(4, 4096, device="cuda", dtype=dtype)
weight = _row_stride_padded(torch.randn(128, 4096, device="cuda", dtype=dtype), 64)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
original_wvsplitk = utils.ops.wvSplitK
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
@@ -123,19 +123,12 @@ def test_mega_moe_m_values_reach_mega_gate(monkeypatch) -> None:
threshold (1 for 128 experts), so the forward actually reaches the kernel."""
dg = SimpleNamespace(
get_bf16_mega_gate_config=lambda m, *_: {"num_sms": -(-m // 16)},
get_block_m_for_mega_moe=lambda *_: 16,
)
monkeypatch.setattr(deep_gemm_warmup, "_import_deep_gemm", lambda: dg)
monkeypatch.setattr(deep_gemm_warmup, "_mega_moe_uses_mega_gate", lambda _: True)
monkeypatch.setattr(
deep_gemm_warmup, "get_ep_group", lambda: SimpleNamespace(world_size=8)
)
buffer = SimpleNamespace(num_max_tokens_per_rank=64)
module = SimpleNamespace(
gate=SimpleNamespace(input_size=4096, output_size=128),
experts=SimpleNamespace(
top_k=6, num_experts=128, get_symm_buffer=lambda: buffer
),
experts=SimpleNamespace(top_k=6, get_block_m=lambda num_tokens: 16),
use_sequence_parallel=False,
)
assert deep_gemm_warmup._get_fp8_fp4_mega_moe_m_values(module, 40) == [16, 32, 40]
@@ -139,6 +139,122 @@ def test_topk_softmax_dispatch(use_rocm_aiter: bool):
assert topk_func == vllm_topk_softmax
def _topk_gating_launch(
num_tokens: int = 4,
num_experts: int = 512,
topk: int = 10,
num_shared_experts: int = 0,
weights_width: int | None = None,
ids_width: int | None = None,
contiguous: bool = True,
) -> dict:
"""Tensors for one softmax top-k launch, as dispatch_topk_softmax_func sees them."""
width = topk + num_shared_experts
weights = torch.empty(num_tokens, weights_width or width)
ids = torch.empty(num_tokens, ids_width or width, dtype=torch.int32)
ids = ids[:, :topk]
if contiguous:
gating = torch.empty(num_tokens, num_experts + num_shared_experts)
else:
gating = torch.empty(num_experts + num_shared_experts + 3, num_tokens).t()
return dict(topk_weights=weights, topk_indices=ids, gating_output=gating)
@pytest.mark.parametrize(
"launch, num_shared_experts, scoring_func, expect_gating",
[
pytest.param({}, 0, "", True, id="plain-softmax"),
pytest.param({"num_tokens": 4096}, 0, "", True, id="max-tokens"),
pytest.param({"num_tokens": 4097}, 0, "", False, id="too-many-tokens"),
pytest.param({"contiguous": False}, 0, "", False, id="non-contiguous"),
pytest.param({}, 0, "sigmoid", False, id="scoring-without-shared"),
pytest.param({"num_shared_experts": 1}, 1, "sigmoid", True, id="one-shared"),
pytest.param({"num_shared_experts": 8}, 8, "sigmoid", True, id="eight-shared"),
pytest.param(
{"num_shared_experts": 3}, 3, "sigmoid", False, id="unsupported-count"
),
pytest.param({"num_shared_experts": 1}, 1, "", False, id="no-shared-scoring"),
pytest.param(
{"num_shared_experts": 1}, 1, "softmax", False, id="wrong-shared-scoring"
),
pytest.param(
{"num_shared_experts": 1, "weights_width": 10},
1,
"sigmoid",
False,
id="weights-too-narrow",
),
pytest.param(
{"num_shared_experts": 1, "ids_width": 12},
1,
"sigmoid",
False,
id="row-stride-mismatch",
),
pytest.param(
{"num_tokens": 4097, "num_shared_experts": 1},
1,
"sigmoid",
False,
id="shared-too-many-tokens",
),
],
)
@pytest.mark.parametrize("gating_enabled", [True, False])
def test_topk_softmax_dispatch_aiter_topk_gating(
monkeypatch: pytest.MonkeyPatch,
launch: dict,
num_shared_experts: int,
scoring_func: str,
expect_gating: bool,
gating_enabled: bool,
):
"""AITER launches use topk_gating only where it is enabled, and only when it
supports them.
Every other AITER launch must reach the legacy topk_softmax, so this is the
one place the choice is made.
"""
monkeypatch.setattr(
rocm_aiter_ops, "is_topk_gating_enabled", lambda: gating_enabled
)
topk_func = dispatch_topk_softmax_func(
True,
**_topk_gating_launch(**launch),
num_shared_experts=num_shared_experts,
shared_expert_scoring_func=scoring_func,
)
if expect_gating and gating_enabled:
assert topk_func == rocm_aiter_ops.topk_gating
else:
assert topk_func == rocm_aiter_ops.topk_softmax
@pytest.mark.parametrize(
"cdna, fused_moe_enabled, expected",
[
pytest.param(3, True, True, id="cdna3"),
pytest.param(4, True, True, id="cdna4"),
pytest.param(2, True, False, id="cdna2"),
pytest.param(3, False, False, id="cdna3-aiter-moe-off"),
pytest.param(4, False, False, id="cdna4-aiter-moe-off"),
],
)
def test_topk_gating_enabled_by_cdna(
monkeypatch: pytest.MonkeyPatch,
cdna: int,
fused_moe_enabled: bool,
expected: bool,
):
"""topk_gating is on for CDNA3 and CDNA4 with AITER MoE enabled, nowhere else."""
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
monkeypatch.setattr(
rocm_aiter_ops, "is_fused_moe_enabled", lambda: fused_moe_enabled
)
monkeypatch.setattr("vllm.platforms.rocm.get_cdna_version", lambda: cdna)
assert bool(rocm_aiter_ops.is_topk_gating_enabled()) is expected
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
@@ -7,9 +7,16 @@ from unittest.mock import Mock, patch
import pytest
import torch
from vllm.config import VllmConfig
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import (
BatchDescriptor,
DPMetadata,
ForwardContext,
get_forward_context,
set_forward_context,
)
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
from vllm.model_executor.layers.fused_moe.modular_kernel import (
@@ -36,6 +43,12 @@ pytestmark = pytest.mark.cpu_test
_REC_MODULE = "vllm.model_executor.layers.fused_moe.routed_experts_capturer"
def _forward_context(**kwargs):
return ForwardContext(
no_compile_layers={}, attn_metadata={}, slot_mapping={}, **kwargs
)
def _capturer_with_buffer(
*,
max_tokens: int = 8,
@@ -319,13 +332,120 @@ def test_routed_experts_capturer_single_dp_no_metadata():
"""dp_metadata is None: capture writes the full topk_ids rows."""
capturer = _capturer_with_buffer(dp_rank=0)
topk = torch.tensor([[1, 2], [3, 4], [5, 6]], dtype=torch.int32)
ctx = SimpleNamespace(dp_metadata=None)
ctx = _forward_context()
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
capturer.capture(layer_id=0, topk_ids=topk)
assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
assert capturer.device_buffer[3, 0, 0].item() == -1
@pytest.mark.parametrize("tp_size, num_tokens", [(2, 6), (2, 3), (2, 1), (4, 2)])
@pytest.mark.parametrize(
"cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL, CUDAGraphMode.PIECEWISE]
)
def test_single_dp_ep_capture_restores_tp_shards(tp_size, num_tokens, cg_mode):
"""Capture all current rows after model-level SP, including buffer reuse."""
capturer = _capturer_with_buffer(tp_size=tp_size)
config = SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_size=1,
tensor_parallel_size=tp_size,
enable_expert_parallel=True,
use_sequence_parallel_moe=False,
is_moe_model=True,
),
compilation_config=SimpleNamespace(
fast_moe_cold_start=False, static_forward_context={}
),
)
tp_group = Mock()
shard_n = (num_tokens + tp_size - 1) // tp_size
with (
set_forward_context(
None,
config,
num_tokens=num_tokens,
cudagraph_runtime_mode=cg_mode,
),
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
):
assert get_forward_context().dp_metadata is None
for offset in (20, 1):
expected = torch.arange(num_tokens * 2, dtype=torch.int32).reshape(-1, 2)
expected += offset
padded = torch.full((shard_n * tp_size, 2), 99, dtype=torch.int32)
padded[:num_tokens] = expected
tp_group.all_gather.return_value = padded
capturer.capture(0, padded[:shard_n])
assert torch.equal(
capturer.snapshot_routing_data(num_tokens)[:, 0].int(), expected
)
assert (capturer.device_buffer[num_tokens:, 0] == -1).all()
if num_tokens == 1:
tp_group.all_gather.assert_not_called()
else:
assert tp_group.all_gather.call_count == 2
@pytest.mark.parametrize("num_tokens, num_rows", [(6, 6), (3, 4), (0, 0)])
def test_single_dp_tp_capture_does_not_regather_full_batch(num_tokens, num_rows):
capturer = _capturer_with_buffer(tp_size=2)
topk = torch.arange(num_rows * 2, dtype=torch.int32).reshape(-1, 2)
ctx = _forward_context(batch_descriptor=BatchDescriptor(num_tokens=num_tokens))
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
patch(f"{_REC_MODULE}.get_tp_group") as get_tp_group,
):
capturer.capture(0, topk)
get_tp_group.assert_not_called()
assert torch.equal(capturer.device_buffer[:num_tokens, 0], topk[:num_tokens])
assert (capturer.device_buffer[num_tokens:, 0] == -1).all()
@pytest.mark.parametrize("num_tokens, num_rows", [(6, 2), (0, 1)])
def test_single_dp_tp_capture_rejects_unexpected_batch_without_writing(
num_tokens, num_rows
):
capturer = _capturer_with_buffer(tp_size=2)
topk = torch.ones(num_rows, 2, dtype=torch.int32)
ctx = _forward_context(batch_descriptor=BatchDescriptor(num_tokens=num_tokens))
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
pytest.raises(AssertionError, match="unexpected topk_ids batch dim"),
):
capturer.capture(0, topk)
assert (capturer.device_buffer == -1).all()
@pytest.mark.parametrize("tp_rank", [0, 1])
def test_capture_uses_pre_sp_size_after_model_shards_padding_mask(tp_rank):
"""Model-level SP shortens the mask; capture still needs all three rows."""
from vllm.models.common.ops import sequence_parallel as sp
capturer = _capturer_with_buffer(tp_size=2)
expected = torch.arange(8, dtype=torch.int32).reshape(4, 2)
ctx = _forward_context(
is_padding=torch.zeros(3, dtype=torch.bool),
batch_descriptor=BatchDescriptor(num_tokens=3),
)
with (
patch.object(sp, "get_tensor_model_parallel_world_size", return_value=2),
patch.object(sp, "get_tensor_model_parallel_rank", return_value=tp_rank),
):
ctx.is_padding = sp.sp_padding_mask(ctx.is_padding, torch.empty(3, 1))
assert len(ctx.is_padding) == 2
tp_group = Mock()
tp_group.all_gather.return_value = expected
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
patch(f"{_REC_MODULE}.get_tp_group", return_value=tp_group),
):
capturer.capture(0, expected[2 * tp_rank : 2 * (tp_rank + 1)])
tp_group.all_gather.assert_called_once()
torch.testing.assert_close(capturer.device_buffer[:3, 0], expected[:3])
assert (capturer.device_buffer[3:, 0] == -1).all()
@pytest.mark.parametrize(
("num_experts", "dtype_name", "torch_dtype"),
[(256, "uint8", torch.uint8), (257, "uint16", torch.uint16)],
@@ -359,7 +479,7 @@ def test_routed_experts_capturer_narrows_snapshot(output_dtype):
capturer = _capturer_with_buffer(dtype=torch.int32)
capturer.output_dtype = output_dtype
topk = torch.tensor([[1, 2], [254, 255]], dtype=torch.int64)
ctx = SimpleNamespace(dp_metadata=None)
ctx = _forward_context()
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
capturer.capture(layer_id=0, topk_ids=topk)
@@ -375,8 +495,8 @@ def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
"""N == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
capturer = _capturer_with_buffer(dp_rank=1)
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
ctx = _forward_context(
dp_metadata=DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
)
# Concatenated order: rank0 rows then rank1 rows.
topk = torch.tensor(
@@ -392,8 +512,8 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
"""N == token_num_per_dp: topk is already local to this DP rank."""
capturer = _capturer_with_buffer(dp_rank=1)
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
ctx = _forward_context(
dp_metadata=DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
)
topk = torch.tensor([[10, 11], [12, 13], [14, 15]], dtype=torch.int32)
with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
@@ -404,8 +524,8 @@ def test_routed_experts_capturer_dp_modular_local_tokens():
def test_routed_experts_capturer_dp_ep_gathered_shards():
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
num_tokens_dp = torch.tensor([3, 2], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(
ctx = _forward_context(
dp_metadata=DPMetadata(
num_tokens_across_dp_cpu=num_tokens_dp,
local_sizes=[2, 2, 1, 1],
)
@@ -421,13 +541,17 @@ def test_routed_experts_capturer_dp_ep_gathered_shards():
assert torch.equal(capturer.device_buffer[:2, 0], topk[4:])
def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
@pytest.mark.parametrize("single_dp", [False, True])
def test_routed_experts_capturer_sp_modular_gathers_tp_shards(single_dp):
capturer = _capturer_with_buffer(dp_rank=0 if single_dp else 1, tp_size=2)
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
ctx = _forward_context(
dp_metadata=(
None if single_dp else DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
),
batch_descriptor=BatchDescriptor(num_tokens=3),
)
local_shard = torch.tensor([[10, 11], [12, 13]], dtype=torch.int32)
local_shard = torch.tensor([[10, 12], [11, 13]], dtype=torch.int32).t()
gathered = torch.tensor([[10, 11], [12, 13], [14, 15], [-1, -1]], dtype=torch.int32)
tp_group = Mock()
tp_group.all_gather.return_value = gathered
@@ -438,16 +562,83 @@ def test_routed_experts_capturer_sp_modular_gathers_tp_shards():
):
capturer.capture(layer_id=0, topk_ids=local_shard)
tp_group.all_gather.assert_called_once_with(local_shard, dim=0)
tp_group.all_gather.assert_called_once()
gather_input = tp_group.all_gather.call_args.args[0]
assert gather_input.is_contiguous()
torch.testing.assert_close(gather_input, local_shard)
assert tp_group.all_gather.call_args.kwargs == {"dim": 0}
assert torch.equal(capturer.device_buffer[:3, 0], gathered[:3])
def test_routed_experts_capturer_rejects_incomplete_dp_shard_groups():
"""Invalid gather metadata must not silently select another rank's rows."""
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
ctx = _forward_context(
dp_metadata=DPMetadata(
num_tokens_across_dp_cpu=torch.tensor([3, 2]),
local_sizes=[2, 2, 2],
)
)
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
pytest.raises(ValueError, match="not divisible by DP rank count"),
):
capturer.capture(0, torch.ones(6, 2, dtype=torch.int32))
assert (capturer.device_buffer == -1).all()
@pytest.mark.parametrize("layer_id", [-1, 4])
def test_routed_experts_capturer_rejects_layer_before_restoring_rows(layer_id):
capturer = _capturer_with_buffer(tp_size=2)
ctx = _forward_context(batch_descriptor=BatchDescriptor(num_tokens=3))
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
patch(f"{_REC_MODULE}.get_tp_group") as get_tp_group,
pytest.raises(IndexError, match="exceeds capture buffer"),
):
capturer.capture(layer_id, torch.ones(2, 2, dtype=torch.int32))
get_tp_group.assert_not_called()
assert (capturer.device_buffer == -1).all()
@pytest.mark.parametrize(
("counts", "local_sizes", "start", "stop"),
[
([3, 0], [2, 2, 0, 0], 3, 3),
([0, 3], None, 0, 3),
([0, 0], [0, 0, 0, 0], 0, 0),
([2, 2], [1, 1, 1, 1], 2, 4),
],
ids=["idle-rank", "only-active-rank", "all-idle", "unpadded-gather-tie"],
)
def test_routed_experts_capturer_dp_preserves_empty_and_unpadded_spans(
counts, local_sizes, start, stop
):
capturer = _capturer_with_buffer(dp_rank=1, tp_size=2)
topk = torch.arange(sum(counts) * 2, dtype=torch.int32).reshape(-1, 2)
ctx = _forward_context(
dp_metadata=DPMetadata(
num_tokens_across_dp_cpu=torch.tensor(counts), local_sizes=local_sizes
)
)
with (
patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
patch(f"{_REC_MODULE}.get_tp_group") as get_tp_group,
):
capturer.capture(0, topk)
get_tp_group.assert_not_called()
torch.testing.assert_close(
capturer.device_buffer[: stop - start, 0], topk[start:stop]
)
assert (capturer.device_buffer[stop - start :, 0] == -1).all()
def test_routed_experts_capturer_dp_unexpected_batch_raises():
"""Mismatch between topk batch dim and DP layout: fail fast."""
capturer = _capturer_with_buffer(dp_rank=0)
num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
ctx = SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
ctx = _forward_context(
dp_metadata=DPMetadata(num_tokens_across_dp_cpu=num_tokens_dp)
)
# total=5, local=2: n=1 matches neither naive (5) nor modular (2).
topk = torch.tensor([[1, 2]], dtype=torch.int32)
+325 -13
View File
@@ -1,11 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm fused KDA decode kernel must match the Triton chain it replaces.
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
The fused kernel folds the packed causal conv1d update, the gated delta-rule
recurrence and the gated output RMSNorm into one launch, and updates both the
conv state and the recurrent state in place. Every one of those outputs is
compared against the three-kernel path the AMD layer falls back to.
Two fused paths fold the packed causal conv1d update, the gated delta-rule
recurrence and the gated output RMSNorm into one launch, updating the conv
state and the recurrent state in place:
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
Every output of each is compared against the unfused path the AMD layer falls
back to, with the argument layout the layer itself uses.
"""
import pytest
@@ -14,7 +20,7 @@ import torch
from vllm.platforms import current_platform
def _on_supported_arch() -> bool:
def _hip_kernel_arch() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx942, on_gfx950
@@ -22,10 +28,26 @@ def _on_supported_arch() -> bool:
return on_gfx950() or on_gfx942()
def _aiter_gluon_kda_available() -> bool:
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
pytestmark = pytest.mark.skipif(
not _on_supported_arch(),
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
)
requires_hip_kernel_arch = pytest.mark.skipif(
not _hip_kernel_arch(),
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
)
requires_aiter_gluon_kda = pytest.mark.skipif(
not _aiter_gluon_kda_available(),
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
)
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
HEAD_DIM = 128
@@ -41,7 +63,11 @@ def _requires_kernel() -> None:
class KdaDecodeInputs:
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout."""
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
``num_spec`` > 0 sizes the conv state for speculative decoding
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
"""
def __init__(
self,
@@ -49,6 +75,7 @@ class KdaDecodeInputs:
num_heads: int,
num_slots: int,
seed: int = 0,
num_spec: int = 0,
) -> None:
torch.manual_seed(seed)
device = "cuda"
@@ -70,9 +97,15 @@ class KdaDecodeInputs:
for i in range(3)
]
)
# SD cache layout: [slots, width - 1, 3 * dim]; the layer transposes it.
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
self.conv_state = (
torch.randn(num_slots, CONV_WIDTH - 1, 3 * dim, device=device, dtype=DTYPE)
torch.randn(
num_slots,
CONV_WIDTH - 1 + num_spec,
3 * dim,
device=device,
dtype=DTYPE,
)
* 0.5
)
self.recurrent_state = (
@@ -112,6 +145,12 @@ class KdaDecodeInputs:
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
return state.transpose(-1, -2)
def empty_out(self) -> torch.Tensor:
num_tokens = self.mixed_qkv.shape[0]
return torch.empty(
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
)
def _gated_rmsnorm(
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
@@ -196,6 +235,7 @@ def _run_fused(
return out, conv_state, recurrent_state
@requires_hip_kernel_arch
@pytest.mark.parametrize("num_heads", [12, 24, 96])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
@torch.inference_mode()
@@ -214,6 +254,7 @@ def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
"""Only the slots named by state_indices may be written."""
@@ -234,6 +275,7 @@ def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
)
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_without_output_norm() -> None:
"""Omitting the gate/norm pair returns the raw recurrent output."""
@@ -265,6 +307,7 @@ def test_fused_kda_decode_without_output_norm() -> None:
assert torch.isfinite(out).all()
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_skips_null_block_padding() -> None:
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
@@ -291,6 +334,7 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_hip_kernel_arch
def test_decode_weights_staged_after_loading(dist_init) -> None:
"""The fused-decode weight copies must not depend on the weight loaders.
@@ -323,11 +367,15 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
config, vllm_config, prefix="model.layers.0.self_attn"
)
assert layer.decode_conv1d_weight is not None
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
# kernel gets an fp32 norm copy.
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
with torch.no_grad():
for param in layer.parameters():
param.normal_()
layer.decode_conv1d_weight.fill_(float("nan"))
layer.decode_norm_weight.fill_(float("nan"))
if layer.decode_norm_weight is not None:
layer.decode_norm_weight.fill_(float("nan"))
process_weights_after_loading(
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
@@ -341,7 +389,271 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
torch.testing.assert_close(
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
)
if layer.decode_norm_weight is not None:
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
assert not layer.decode_conv1d_weight.requires_grad
# --------------------------------------------------------------------------
# non-spec decode: fused_recurrent_kda_packed_decode
# --------------------------------------------------------------------------
def _run_aiter_packed_decode(
inp: KdaDecodeInputs,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
from vllm._aiter_ops import rocm_aiter_ops
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
num_tokens = inp.mixed_qkv.shape[0]
core_attn_out = inp.empty_out()
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
mixed_qkv=inp.mixed_qkv,
g=inp.g1[0],
beta=inp.beta[0],
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
ssm_state_indices=inp.state_indices[:num_tokens],
conv_state=inp.conv_state_view(conv_state),
conv_weight=inp.decode_conv1d_weight,
out_gate=inp.g2[:num_tokens],
norm_weight=inp.norm_weight_bf16,
norm_eps=NORM_EPS,
out=core_attn_out[0, :num_tokens],
)
return core_attn_out, conv_state, recurrent_state
@requires_aiter_gluon_kda
@pytest.mark.parametrize("num_heads", [12, 24])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
@torch.inference_mode()
def test_aiter_packed_decode_matches_triton_chain(
num_heads: int, num_tokens: int
) -> None:
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
# The fused kernel keeps the recurrent output in fp32 through the norm,
# while the Triton chain rounds it to BF16 in between.
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
# The conv state is a pure shift-and-append: it must be bit-exact.
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_aiter_gluon_kda
@torch.inference_mode()
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
"""Only the slots named by the state indices may be written."""
num_tokens, num_heads, num_slots = 4, 12, 9
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
touched = set(inp.state_indices.tolist())
untouched = [slot for slot in range(num_slots) if slot not in touched]
assert untouched, "test needs at least one unused slot"
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
)
@requires_aiter_gluon_kda
@torch.inference_mode()
def test_aiter_packed_decode_skips_null_block_padding() -> None:
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
must do the same rather than read-modify-write slot 0 once per padded row.
"""
num_real, num_padded, num_heads = 3, 5, 12
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
inp.state_indices[num_real:] = 0
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
torch.testing.assert_close(
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
)
assert torch.isfinite(actual_out).all()
# Slot 0 must be untouched by both paths.
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
# --------------------------------------------------------------------------
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
# --------------------------------------------------------------------------
class SpecBatch:
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
per-token slot table the GDN metadata hands the layer.
"""
def __init__(
self,
seq_lens: list[int],
num_accepted: list[int],
num_heads: int,
num_spec: int,
seed: int = 0,
) -> None:
assert len(seq_lens) == len(num_accepted)
assert max(seq_lens) <= num_spec + 1
n_seqs = len(seq_lens)
slots_per_seq = num_spec + 1
num_slots = n_seqs * slots_per_seq + 2
self.inp = KdaDecodeInputs(
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
)
# Distinct slots per (request, token), shuffled, none at slot 0.
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
self.query_start_loc = torch.tensor(
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
dtype=torch.int32,
device="cuda",
)
self.num_accepted_tokens = torch.tensor(
num_accepted, dtype=torch.int32, device="cuda"
)
self.max_query_len = slots_per_seq
def _run_spec_triton_chain(
sb: SpecBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
inp = sb.inp
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
conv_out = causal_conv1d_update(
inp.mixed_qkv,
inp.conv_state_view(conv_state),
inp.conv_weights,
None,
activation="silu",
conv_state_indices=sb.state_indices[:, 0],
num_accepted_tokens=sb.num_accepted_tokens,
query_start_loc=sb.query_start_loc,
max_query_len=sb.max_query_len,
validate_data=False,
out=torch.empty_like(inp.mixed_qkv),
)
q, k, v = (
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
)
core_attn_out, _ = fused_recurrent_kda(
q=q,
k=k,
v=v,
raw_g=inp.g1,
raw_beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
cu_seqlens=sb.query_start_loc,
ssm_state_indices=sb.state_indices,
num_accepted_tokens=sb.num_accepted_tokens,
)
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
return out, conv_state, recurrent_state
def _run_aiter_spec(
sb: SpecBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
from vllm._aiter_ops import rocm_aiter_ops
inp = sb.inp
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
q, k, v = (
x.view(1, -1, inp.num_heads, HEAD_DIM)
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
)
n_spec = q.shape[1]
core_attn_out = inp.empty_out()
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
q=q,
k=k,
v=v,
g=inp.g1,
beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
cu_seqlens=sb.query_start_loc,
ssm_state_indices=sb.state_indices,
num_accepted_tokens=sb.num_accepted_tokens,
conv_state=inp.conv_state_view(conv_state),
conv_weight=inp.decode_conv1d_weight,
out_gate=inp.g2[:n_spec].unsqueeze(0),
norm_weight=inp.norm_weight_bf16,
norm_eps=NORM_EPS,
out=core_attn_out[:, :n_spec],
)
return core_attn_out, conv_state, recurrent_state
@requires_aiter_gluon_kda
@pytest.mark.parametrize("num_heads", [12, 24])
@pytest.mark.parametrize(
"num_spec, seq_lens, num_accepted",
[
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
(6, [7, 3], [1, 4]), # longer window, two requests
],
)
@torch.inference_mode()
def test_aiter_spec_decode_matches_triton_chain(
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
) -> None:
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
# Nothing outside the slot table may change.
touched = set(sb.state_indices.flatten().tolist())
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
assert untouched
torch.testing.assert_close(
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
@@ -75,6 +75,9 @@ AITER_MODEL_LIST = [
"facebook/opt-125m", # opt
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
pytest.param(
"facebook/opt-350m", # opt with projected token embeddings
),
pytest.param(
"microsoft/phi-2", # phi
marks=[pytest.mark.core_model, pytest.mark.slow_test],
@@ -70,6 +70,10 @@ def gemma3_chat_template(content: str) -> str:
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
def deepseek_vl2_chat_template(content: str) -> str:
return f"<|User|>: {content}\n\n<|Assistant|>:"
def ernie45_vl_chat_template(content: str) -> str:
return (
f"<|begin_of_sentence|>User: {content}"
@@ -255,7 +259,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
vllm_runner_kwargs={
"load_format": "dummy",
"attention_backend": (
"ROCM_FLASHMLA_SPARSE_DSV4"
"ROCM_FLASHMLA_SPARSE_DSV41"
if current_platform.is_rocm()
else "FLASHMLA_SPARSE_DSV41"
),
@@ -356,6 +360,15 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
},
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
),
"deepseek_vl2": VitCudagraphTestConfig(
model="deepseek-ai/deepseek-vl2-tiny",
modalities=["image"],
image_prompt=deepseek_vl2_chat_template("<image>\nWhat is in this image?"),
vllm_runner_kwargs={
"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]},
},
marks=[pytest.mark.core_model],
),
"gemma4": VitCudagraphTestConfig(
model="google/gemma-4-E2B-it",
image_prompt=(
+5 -3
View File
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
The runtime reads a question's options from the 255 code logits of its readout
at the last prompt token. vLLM's classify head must give the same code logits
for the same prompt tokens, and for an image prompt that vLLM processes itself.
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
weight file without an index, next to the readout file.
"""
import pytest
@@ -14,7 +16,7 @@ from transformers import AutoModel
from vllm.assets.image import ImageAsset
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "vllm-sr/d3-edge"
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
STATE = (
"The order arrived damaged yesterday. The customer has a receipt and asks "
"for a replacement today."
@@ -47,7 +49,7 @@ def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
return torch.softmax(logits[:count].float(), dim=-1)
@pytest.mark.parametrize("model", [MODEL])
@pytest.mark.parametrize("model", MODELS)
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
runtime = reference.runtime
+14 -11
View File
@@ -70,20 +70,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
),
)
with (
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
pytest.raises(ValueError, match="MoE SP requires PP=1"),
):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
@spawn_new_process_for_each_test
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
@pytest.mark.parametrize("use_sp", [False, True])
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
backend: str, use_sp: bool
) -> None:
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
model.use_hc_sequence_parallel = use_sp
model.use_sequence_parallel = False
torch.nn.Module.__init__(model)
@@ -98,7 +99,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
def decoder(**kwargs):
"""Check that the embedding residual shares the hidden state's token slice."""
hidden = kwargs["hidden_states"]
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
if backend == "nvidia":
prev_block_output = kwargs["prev_block_output"]
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
model.layers = [decoder]
@@ -120,9 +123,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
with (
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
+53 -8
View File
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
if not current_platform.is_cuda_alike():
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
from tests.utils import multi_gpu_test
from vllm.config import (
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
from vllm.utils.network_utils import get_open_port
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
from .test_ple import _ConvBatchCase, _make_conv_metadata
if current_platform.is_rocm():
from vllm.models.qwen4_exp.amd import model as qwen4_model
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
else:
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
class _TPAttentionProjection(nn.Module):
"""Keep real TP communication without setting up an attention KV cache."""
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
set_current_vllm_config(config),
set_forward_context({layer.ple.prefix: metadata}, config),
):
layer.ple.start_prefetch(ids, qsl, context)
if hasattr(layer.ple, "start_prefetch"):
layer.ple.start_prefetch(ids, qsl, context)
output = layer(
*inputs,
positions,
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
ngram_context=context,
)
if config.parallel_config.enable_hc_sp:
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
# ROCm all-gather requires contiguous inputs.
output = tuple(
sp_all_gather(x.contiguous())[:num_tokens] for x in output
)
# A later MoE call may reuse workspace-backed output storage.
outputs.append(tuple(x.clone() for x in output))
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
query_start_loc=None,
ngram_context=None,
)
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
def _check_compiled_sp_ops() -> None:
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
sp_reduce_scatter,
sp_shard,
)
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
return gathered, reduced
# vLLM drops Dynamo guards, so one trace must serve every token count.
compiled = torch.compile(
collectives,
fullgraph=True,
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
)
traced = torch.randn(8, 16)
torch._dynamo.mark_dynamic(traced, 0)
compiled(traced)
for num_tokens in (7, 3, 1):
torch.manual_seed(num_tokens)
x = torch.randn(num_tokens, 16)
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
"""Initialize rank-local runtime state and run one equivalence check."""
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
torch.set_default_dtype(torch.bfloat16)
if moe_sp:
_check_moe_sp(rank, config)
elif compiled_ops:
_check_compiled_sp_ops()
else:
_check_decoder_ple(rank)
finally:
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
@multi_gpu_test(num_gpus=2)
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
"""Padding in compiled SP collectives must not specialize on the traced size."""
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
+9 -2
View File
@@ -462,7 +462,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"),
"OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
"OPTForCausalLM": _HfExamplesInfo(
"facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
"facebook/opt-125m",
{"350m": "facebook/opt-350m", "1b": "facebook/opt-iml-max-1.3b"},
),
"OrionForCausalLM": _HfExamplesInfo(
"OrionStarAI/Orion-14B-Chat", trust_remote_code=True
@@ -731,7 +732,13 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
),
"D3ForSequenceClassification": _HfExamplesInfo(
"vllm-sr/d3-edge",
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
extras={
"d3": "vllm-sr/d3",
"d3-flash": "vllm-sr/d3-flash",
"d3-mini": "vllm-sr/d3-mini",
"d3-nano": "vllm-sr/d3-nano",
"d3-lite": "vllm-sr/d3-lite",
},
max_model_len=4096,
),
"GPT2ForSequenceClassification": _HfExamplesInfo(
+66
View File
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.models.deepseek_v41.nvidia import model as model_module
@pytest.mark.parametrize(
"pcp_size,use_ep", [(1, False), (2, True), (2, False), (4, False)]
)
@pytest.mark.parametrize("use_sp", [False, True])
def test_pcp_gathers_routing_ids_once_after_local_input_preparation(
pcp_size, use_ep, use_sp, monkeypatch
):
"""Embedding keeps local IDs; all decoder layers receive aligned routing IDs."""
gather = Mock(side_effect=lambda ids, dim: torch.cat([ids] * pcp_size, dim=dim))
monkeypatch.setattr(
model_module, "get_pcp_group", lambda: SimpleNamespace(all_gather=gather)
)
monkeypatch.setattr(
model_module,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(model_module, "is_forward_context_available", lambda: False)
monkeypatch.setattr(model_module, "sp_shard", lambda tensor: tensor[::2])
monkeypatch.setattr(
model_module, "sp_all_gather", lambda tensor: tensor.repeat_interleave(2, dim=0)
)
monkeypatch.setattr(model_module, "hc_collapse_triton", lambda hidden, mix: hidden)
embed = Mock(side_effect=lambda ids: ids[:, None].float())
run_layers = Mock(
side_effect=lambda layer_ids, hidden, *args: (hidden, None, None, None, hidden)
)
model = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=pcp_size, enable_expert_parallel=use_ep
),
use_native_mega_moe=False,
use_sequence_parallel=use_sp,
engram_hash=None,
embed_input_ids=embed,
start_layer=0,
decoder_replay_start=2,
decoder_replay_layers=None,
aux_hidden_state_layers=(),
_mtp_hidden_buffer=None,
_run_layers=run_layers,
_collapse=lambda hidden, *args: hidden,
norm=lambda hidden: hidden,
)
for offset in (0, 10):
local_ids = torch.tensor([3, 7, 11, 15]) + offset
model_module.DeepseekV4Model.forward(model, local_ids, torch.arange(4), None)
assert embed.call_args.args[0] is local_ids
routing_ids = local_ids[::2] if use_sp else local_ids
needs_gather = pcp_size > 1 and not use_ep
if needs_gather:
torch.testing.assert_close(gather.call_args.args[0], routing_ids)
routing_ids = routing_ids.repeat(pcp_size)
torch.testing.assert_close(run_layers.call_args.args[3], routing_ids)
assert gather.call_count == 2 * int(needs_gather)
+51 -1
View File
@@ -4,7 +4,7 @@
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
@@ -16,6 +16,9 @@ from vllm.forward_context import (
ForwardContext,
override_forward_context,
)
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
@@ -63,6 +66,7 @@ def state(monkeypatch):
cfg.scheduler_config.max_num_seqs = 16
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.aux_output_config.enable_return_routed_experts = False
cfg.compilation_config.fast_moe_cold_start = False
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = DecoderReplayLayers(WINDOW, MagicMock(), MagicMock(), set(), "swa_first")
@@ -172,6 +176,31 @@ def test_replay_batch_keeps_each_request_window(state):
assert context.dp_metadata is None and context.is_padding is None
def test_replay_context_restores_tp_routing_rows(state):
"""Capture uses the compact replay batch's size, not the outer batch's."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
replay, _ = _prepare(state, batch, CUDAGraphMode.NONE)
context = replay.forward_context
assert context.is_padding is None and context.dp_metadata is None
count = len(REPLAY_ROWS)
assert context.batch_descriptor.num_tokens == count
capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
capturer.dp_rank, capturer.tp_size = 0, 2
capturer.device_buffer = torch.full((401, 1, 2), -1, device=DEVICE)
expected = torch.tensor(REPLAY_ROWS, device=DEVICE)[:, None].expand(-1, 2)
gathered = torch.cat((expected, torch.full((1, 2), -1, device=DEVICE)))
with (
override_forward_context(context),
patch(
"vllm.model_executor.layers.fused_moe.routed_experts_capturer.get_tp_group"
) as group,
):
group.return_value.all_gather.return_value = gathered
capturer.capture(0, gathered[: (count + 1) // 2])
torch.testing.assert_close(capturer.device_buffer[:count, 0], expected)
assert (capturer.device_buffer[count:] == -1).all()
def test_replay_batch_keeps_device_decode_boundaries(state):
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
@@ -195,6 +224,27 @@ def test_prompt_logprobs_requests_keep_their_rows(state):
assert build.batch.max_query_len == 300
@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE])
def test_aux_output_requests_keep_every_routing_row(state, cg_mode):
"""R3 needs every prefill row even when final logits only need the window."""
state.vllm_config.aux_output_config.enable_return_routed_experts = True
for index in range(3):
state.add_request(
index,
SimpleNamespace(
replay_start=0,
sampling_params=SimpleNamespace(prompt_logprobs=None),
),
)
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
state.decoder_replay_layers.trim_threshold = WINDOW + 1
replay, _ = _prepare(state, batch, cg_mode)
if cg_mode == CUDAGraphMode.NONE:
assert replay is None
else:
torch.testing.assert_close(replay.rows, torch.arange(401, device=DEVICE))
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
"""Adaptive verification bounds the decodes' device-side query lengths
above their CPU lengths; the replay batch keeps that bound."""
+12 -726
View File
@@ -8,10 +8,8 @@ import torch
from transformers import DeepseekV4Config
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
DeepGemmSm100MegaMoEBackend,
)
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
bind_routed_experts_capturer,
DeepGemmMegaMoEExperts,
MegaMoEActivation,
)
from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
@@ -20,17 +18,14 @@ from vllm.model_executor.models.interfaces import (
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4ForCausalLM,
DeepseekV4MegaMoEExperts,
DeepseekV4MoE,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.torch_utils import set_default_torch_dtype
pytestmark = pytest.mark.skipif(
@@ -40,11 +35,12 @@ pytestmark = pytest.mark.skipif(
@pytest.fixture(autouse=True)
def sm100_mega_moe_backend(monkeypatch):
def sm100_mega_moe_layout(monkeypatch):
"""Build the SM100 kernel layout on any GPU; kernels are gated per test."""
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.get_deep_gemm_mega_moe_backend",
lambda *args: DeepGemmSm100MegaMoEBackend(),
"vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe."
"check_deep_gemm_mega_moe_supported",
lambda *args: None,
)
@@ -125,7 +121,8 @@ def test_deepseek_v41_moe_routes_without_hash_table(
)
assert moe.gate.tid2eid is None
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
assert isinstance(moe.experts, DeepGemmMegaMoEExperts)
assert moe.experts.activation == MegaMoEActivation("swiglu", clamp=10.0)
assert moe.experts.w13_weight.shape == (num_experts, 256, 128)
assert moe.experts.top_k == top_k
assert (config.n_routed_experts, config.num_experts_per_tok) == (384, 6)
@@ -151,10 +148,9 @@ def test_deepseek_v41_moe_routes_without_hash_table(
routed = {}
def check_routing(x, weights, ids, *, activation_clamp):
def check_routing(x, weights, ids, shared_experts):
routed["ids"], routed["weights"] = ids, weights
assert activation_clamp == config.swiglu_limit
return x.clone()
return x + shared_experts(x)
monkeypatch.setattr(moe.experts, "forward", check_routing)
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
@@ -280,10 +276,10 @@ def test_deepseek_v4_mega_gate_hash_routing_correctness(
dim=-1, keepdim=True
)
def check_routing(x, weights, ids, *, activation_clamp):
def check_routing(x, weights, ids, shared_experts):
torch.testing.assert_close(ids, expected_ids)
torch.testing.assert_close(weights, expected_weights, rtol=1e-3, atol=1e-4)
return x.clone()
return x + shared_experts(x)
monkeypatch.setattr(moe.experts, "forward", check_routing)
monkeypatch.setattr(moe.shared_experts, "forward", torch.zeros_like)
@@ -308,634 +304,6 @@ def test_deepseek_v4_mega_moe_expert_mapping():
]
@pytest.mark.parametrize("use_kimi", [False, True])
def test_deep_gemm_mega_moe_capture_precedes_eplb(monkeypatch, use_kimi):
experts_cls = DeepseekV4MegaMoEExperts
if use_kimi:
from vllm.models.kimi_k3.nvidia.model import KimiK3MegaMoEExperts
experts_cls = KimiK3MegaMoEExperts
experts = experts_cls.__new__(experts_cls)
torch.nn.Module.__init__(experts)
if use_kimi:
experts.synchronize_first_launch = lambda: None
experts.prefix = "model.layers.3.ffn.experts"
experts.max_num_tokens = 4
experts.capture_fn = None
experts.get_symm_buffer = lambda: object()
experts.eplb_state = SimpleNamespace(
logical_to_physical_map=torch.empty(1),
expert_load_view=torch.empty(1),
logical_replica_count=torch.empty(1),
should_record_tensor=torch.empty(1),
num_unpadded_tokens_tensors=None,
)
topk_ids = torch.tensor([[1, 2], [3, 4]])
captured: list[tuple[int, torch.Tensor]] = []
bind_routed_experts_capturer(
SimpleNamespace(modules=lambda: [experts]),
SimpleNamespace(capture=lambda layer_id, ids: captured.append((layer_id, ids))),
)
class MappingReached(Exception):
pass
def map_ids(**kwargs):
assert captured == [(3, topk_ids)]
raise MappingReached
monkeypatch.setattr(
f"{experts_cls.__module__}.eplb_map_to_physical_and_record",
map_ids,
)
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm", lambda: SimpleNamespace()
)
with pytest.raises(MappingReached):
experts(
torch.empty(2, 8),
torch.empty(2, 2),
topk_ids,
activation_clamp=None,
)
def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
)
experts = DeepseekV4MegaMoEExperts(
vllm_config,
num_experts=4,
num_local_experts=2,
experts_start_idx=2,
top_k=2,
hidden_size=128,
intermediate_size=128,
)
nonlocal_weight = torch.ones(128, 64, dtype=torch.uint8)
assert (
experts.weight_loader(
experts.w13_weight,
nonlocal_weight,
"experts.w13_weight",
shard_id="w1",
expert_id=1,
return_success=True,
)
is False
)
w1 = torch.full((128, 64), 3, dtype=torch.uint8)
w3 = torch.full((128, 64), 7, dtype=torch.uint8)
w2 = torch.full((128, 64), 11, dtype=torch.uint8)
assert experts.weight_loader(
experts.w13_weight,
w1,
"experts.w13_weight",
shard_id="w1",
expert_id=2,
return_success=True,
)
assert experts.weight_loader(
experts.w13_weight,
w3,
"experts.w13_weight",
shard_id="w3",
expert_id=2,
return_success=True,
)
assert experts.weight_loader(
experts.w2_weight,
w2,
"experts.w2_weight",
shard_id="w2",
expert_id=2,
return_success=True,
)
interleaved = experts.w13_weight[0].view(torch.uint8).view(16, 2, 8, 64)
assert torch.equal(interleaved[:, 0], w1.view(16, 8, 64))
assert torch.equal(interleaved[:, 1], w3.view(16, 8, 64))
assert torch.equal(experts.w2_weight[0].view(torch.uint8), w2)
assert torch.count_nonzero(experts.w13_weight[1]) == 0
def _reference_mega_moe_layout(w13, s13, w2, s2):
"""CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales."""
def interleave(t):
g, n, k = t.shape
halves = t.view(g, 2, n // 16, 8, k)
return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k)
def utccp(sf):
g, mn, k = sf.shape
return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k)
return (
interleave(w13),
utccp(interleave(s13.view(torch.int32))),
w2,
utccp(s2.view(torch.int32)),
)
def _make_mega_moe_experts(num_local_experts, num_logical_experts):
return DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=num_local_experts,
num_local_experts=num_local_experts,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=384,
num_logical_experts=num_logical_experts,
)
def _load_mega_moe_checkpoint_shuffled(experts, num_logical_experts, seed):
hidden, inter = experts.hidden_size, experts.intermediate_size
shapes = {
"w13_weight": (num_logical_experts, 2 * inter, hidden // 2),
"w13_weight_scale": (num_logical_experts, 2 * inter, hidden // 32),
"w2_weight": (num_logical_experts, hidden, inter // 2),
"w2_weight_scale": (num_logical_experts, hidden, inter // 32),
}
generator = torch.Generator().manual_seed(seed)
ckpt = {
name: torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
for name, shape in shapes.items()
}
shards = []
for expert_id in range(num_logical_experts):
for name, tensor in ckpt.items():
if name.startswith("w2_"):
shards.append((name, tensor[expert_id], "w2", expert_id))
else:
shards.append((name, tensor[expert_id, :inter], "w1", expert_id))
shards.append((name, tensor[expert_id, inter:], "w3", expert_id))
for i in torch.randperm(len(shards), generator=generator).tolist():
name, loaded_weight, shard_id, expert_id = shards[i]
if name.endswith("scale"):
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
assert experts.weight_loader(
getattr(experts, name),
loaded_weight,
f"experts.{name}",
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
return ckpt
def _expected_mega_moe_layout(ckpt, num_local_experts, num_logical_experts):
physical = torch.arange(num_local_experts) % num_logical_experts
ckpt = {name: tensor[physical] for name, tensor in ckpt.items()}
expected = _reference_mega_moe_layout(*ckpt.values())
if is_deep_gemm_supported():
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
w13, s13, w2, s2 = (t.cuda() for t in ckpt.values())
hidden, inter = w2.shape[1], w2.shape[2] * 2
scales = [
deep_gemm.transform_sf_into_required_layout(
(sf.to(torch.int32) << 23).view(torch.float32),
mn,
k,
(1, 32),
num_local_experts,
)
for sf, mn, k in ((s13, 2 * inter, hidden), (s2, hidden, inter))
]
(l1, l1_sf), (l2, l2_sf) = deep_gemm.transform_weights_for_mega_moe(
(w13.view(torch.int8), scales[0]), (w2.view(torch.int8), scales[1])
)
for actual, reference in zip((l1, l1_sf, l2, l2_sf), expected):
assert torch.equal(actual.cpu().view(reference.dtype), reference)
return expected
def _assert_mega_moe_layout(experts, expected):
actual = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
for tensor, reference in zip(actual, expected):
assert torch.equal(tensor.view(reference.dtype), reference)
def test_deepseek_v4_mega_moe_loads_shards_into_kernel_layout():
"""Shards in any order, including EPLB replicas, land in DeepGEMM's layout."""
experts = _make_mega_moe_experts(num_local_experts=4, num_logical_experts=3)
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 3, seed=0)
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 4, 3))
# DeepGEMM's MN-major scale strides.
assert experts.w13_weight_scale.stride() == (768 * 2, 1, 768)
assert experts.w2_weight_scale.stride() == (256 * 3, 1, 256)
def test_deepseek_v4_mega_moe_reload_overwrites_kernel_layout_in_place():
"""A second load must update the live kernel tensors without reallocating."""
experts = _make_mega_moe_experts(num_local_experts=2, num_logical_experts=2)
_load_mega_moe_checkpoint_shuffled(experts, 2, seed=0)
data_ptrs = [param.data_ptr() for param in experts.parameters()]
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 2, seed=1)
assert [param.data_ptr() for param in experts.parameters()] == data_ptrs
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 2, 2))
@pytest.mark.parametrize("intermediate_size", [512, 2304])
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(intermediate_size):
"""Keep native widths so V4.1 routed and shared experts can fuse."""
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=128,
intermediate_size=intermediate_size,
)
assert experts.intermediate_size == intermediate_size
assert experts.w13_weight.shape == (2, 2 * intermediate_size, 64)
assert experts.w13_weight_scale.shape == (2, 2 * intermediate_size, 1)
assert experts.w2_weight.shape == (2, 128, intermediate_size // 2)
assert experts.w2_weight_scale.shape == (2, 128, intermediate_size // 128)
transformed = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
for actual, param in zip(transformed, experts.parameters()):
assert actual.data_ptr() == param.data_ptr()
def test_deepseek_v4_mega_moe_loads_shared_experts_into_kernel_layout(monkeypatch):
"""Shared expert shards land directly in the MegaMoE kernel layout."""
monkeypatch.setattr(
DeepGemmSm100MegaMoEBackend, "supports_shared_experts", lambda *args: True
)
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=384,
num_shared_experts=1,
)
assert experts.has_fused_shared_experts
hidden, inter = 256, 384
generator = torch.Generator().manual_seed(42)
def rand(*shape):
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
w1, w3, w2 = rand(inter, hidden), rand(inter, hidden), rand(hidden, inter)
s1, s3, s2 = (
rand(inter, hidden // 32),
rand(inter, hidden // 32),
rand(hidden, inter // 32),
)
fp8, e8m0 = torch.float8_e4m3fn, torch.float8_e8m0fnu
# The down shard arrives as `.w2.` (MTP, DSpark) or `.down_proj.` (mapped).
for name, data in [
("shared_experts.w1.weight", w1.view(fp8)),
("shared_experts.w3.weight", w3.view(fp8)),
("shared_experts.down_proj.weight", w2.view(fp8)),
("shared_experts.w1.weight_scale", s1.view(e8m0)),
("shared_experts.w3.weight_scale", s3.view(e8m0)),
("shared_experts.w2.weight_scale", s2.view(e8m0)),
]:
loaded = experts.load_shared_expert_weight(f"ffn.{name}", data)
# Weight tracking checks this name against the registered parameters.
assert loaded in {f"ffn.experts.{n}" for n, _ in experts.named_parameters()}
def l1_halves():
return experts.shared_l1_weight.view(torch.uint8).view(-1, 2, 8, hidden)
# Every tensor matches DeepGEMM's batched transform of the checkpoint.
expected = _reference_mega_moe_layout(
*(t.unsqueeze(0) for t in (torch.cat([w1, w3]), torch.cat([s1, s3]), w2, s2))
)
actual = (
experts.shared_l1_weight,
experts.shared_l1_weight_scale,
experts.shared_l2_weight,
experts.shared_l2_weight_scale,
)
for tensor, reference in zip(actual, expected):
assert torch.equal(tensor.view(reference.dtype), reference[0])
# A reload of one shard overwrites in place and leaves the other half alone.
ptr = experts.shared_l1_weight.data_ptr()
w1b = rand(inter, hidden)
experts.load_shared_expert_weight("shared_experts.w1.weight", w1b.view(fp8))
assert experts.shared_l1_weight.data_ptr() == ptr
assert torch.equal(l1_halves()[:, 0], w1b.view(-1, 8, hidden))
assert torch.equal(l1_halves()[:, 1], w3.view(-1, 8, hidden))
with pytest.raises(ValueError):
experts.load_shared_expert_weight("shared_experts.gate_up_proj.weight", w1)
def _fake_deep_gemm(with_shared: bool):
def old_buffer(group): ...
def old_kernel(y): ...
def new_buffer(group, num_shared_experts=0): ...
def new_kernel(y, shared_l1_weights=None, shared_l2_weights=None): ...
return SimpleNamespace(
get_symm_buffer_for_mega_moe=new_buffer if with_shared else old_buffer,
fp8_fp4_mega_moe=new_kernel if with_shared else old_kernel,
get_block_m_for_mega_moe=lambda *args: 0,
)
@pytest.mark.parametrize(
"weight_dtype,scale_dtype,scale_shape,with_shared,fused",
[
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), True, True),
(torch.float8_e4m3fn, torch.uint8, (256, 8), True, True),
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), False, False),
(torch.bfloat16, None, None, True, False),
(torch.float8_e4m3fn, torch.float32, (2, 2), True, False),
(torch.float8_e4m3fn, torch.uint8, (256, 16), True, False),
],
)
def test_deepseek_v4_mega_moe_decides_shared_fusion_at_construction(
monkeypatch, weight_dtype, scale_dtype, scale_shape, with_shared, fused
):
"""Fuse only shared experts the installed kernel and the 1x32 layout fit."""
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm", lambda: _fake_deep_gemm(with_shared)
)
def linear():
scale = (
None if scale_dtype is None else torch.empty(scale_shape).to(scale_dtype)
)
return SimpleNamespace(
weight=torch.empty(256, 256, dtype=weight_dtype), weight_scale_inv=scale
)
shared = SimpleNamespace(gate_up_proj=linear(), down_proj=linear())
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=128,
num_shared_experts=1,
shared_experts=shared,
)
assert experts.has_fused_shared_experts == fused
assert hasattr(experts, "shared_l1_weight") == fused
@pytest.mark.parametrize("fused", [False, True])
def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
monkeypatch, fused
):
class FakeGate(torch.nn.Module):
weight = torch.empty(2, 128)
tid2eid = None
e_score_correction_bias = None
def forward(self, hidden_states):
return torch.empty(hidden_states.shape[0], 2), None
class FakeExperts(torch.nn.Module):
has_fused_shared_experts = fused
backend = DeepGemmSm100MegaMoEBackend()
def forward(self, hidden_states, *args, **kwargs):
return torch.ones_like(hidden_states)
class FakeSharedExperts(torch.nn.Module):
calls = 0
def forward(self, hidden_states):
self.calls += 1
return torch.full_like(hidden_states, 2)
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.use_native_mega_moe = True
moe.gate = FakeGate()
moe.experts = FakeExperts()
moe.shared_experts = None if fused else FakeSharedExperts()
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = 1
moe.renormalize = True
moe.hash_indices_dtype = torch.int64
moe.routed_scaling_factor = 1.0
moe.swiglu_limit = 10.0
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.fused_topk_bias",
lambda **kwargs: (
torch.ones(kwargs["hidden_states"].shape[0], 1),
torch.zeros(kwargs["hidden_states"].shape[0], 1, dtype=torch.int64),
),
)
output = moe(torch.zeros(2, 128))
expected = 1 if fused else 3
assert torch.all(output == expected)
if not fused:
assert moe.shared_experts.calls == 1
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
)
@pytest.mark.parametrize("num_tokens", [7, 63, 64, 65, 127, 128, 135, 434, 16384])
@pytest.mark.parametrize("hidden_size", [256, 5120])
def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact(
num_tokens, hidden_size
):
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(0)
hidden_states = (
torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.float32,
generator=generator,
)
* 17.0
).to(torch.bfloat16)
hidden_states[0, :32] = 0
hidden_states[1, 32:64] = 1.0e-6
hidden_states[2, 64:96] = -1.0e-6
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
ref_topk_idx = topk_ids.to(torch.int64)
ref_topk_weights = topk_weights.clone()
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_topk_idx = torch.empty_like(ref_topk_idx)
fused_topk_weights = torch.empty_like(ref_topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
)
torch.accelerator.synchronize()
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_topk_idx, ref_topk_idx)
assert torch.equal(
fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8),
)
@pytest.mark.parametrize("shared_block_m", [8, 32, 96, 128, 192])
def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
num_tokens = shared_block_m + 7
hidden_size = 256
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(shared_block_m)
hidden_states = torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.bfloat16,
generator=generator,
)
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
aligned_block_m = ((shared_block_m + 127) // 128) * 128
num_shared_rows = ((num_tokens + shared_block_m - 1) // shared_block_m) * (
aligned_block_m
)
ref_shared_x_sf = torch.zeros(
num_shared_rows,
hidden_size // 128,
dtype=torch.int32,
device=device,
)
for token_id in range(num_tokens):
m_in_block = token_id % shared_block_m
transposed_m = (
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
)
shared_row = token_id // shared_block_m * aligned_block_m + transposed_m
ref_shared_x_sf[shared_row].copy_(ref_x_sf[token_id])
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_shared_storage = torch.full(
(hidden_size // 128, num_shared_rows),
-1,
dtype=torch.int32,
device=device,
)
fused_shared_x_sf = fused_shared_storage.t()
fused_topk_idx = torch.empty_like(topk_ids, dtype=torch.int64)
fused_topk_weights = torch.empty_like(topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
shared_x_sf=fused_shared_x_sf,
shared_block_m=shared_block_m,
)
torch.accelerator.synchronize()
populated = ref_shared_x_sf != 0
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_shared_x_sf[populated], ref_shared_x_sf[populated])
assert torch.all(fused_shared_x_sf[~populated] == -1)
def test_deepseek_v4_pwal_hook_finalizes_mhc_broadcast():
"""The loader invokes the model-level PWAL hook for every load format,
so it must finalize mhc broadcast weights to cover dummy load, which
@@ -1040,85 +408,3 @@ def test_dspark_draft_registers_mixture_of_experts(
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
assert draft.num_routed_experts == hf_config.n_routed_experts
assert draft.num_redundant_experts == 4
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
)
@pytest.mark.parametrize("nonfinite_padding", [False, True])
@pytest.mark.parametrize("num_tokens", [7, 65, 135])
def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(
nonfinite_padding, num_tokens
):
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
hidden_size = 256
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(1)
hidden_states = torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.bfloat16,
generator=generator,
)
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
is_padding = torch.zeros(num_tokens, device=device, dtype=torch.bool)
is_padding[1::3] = True
is_padding[-1] = True
if nonfinite_padding:
topk_weights[is_padding] = float("nan")
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
ref_topk_idx = topk_ids.to(torch.int64)
ref_topk_idx[is_padding] = -1
ref_topk_weights = topk_weights.clone()
ref_topk_weights[is_padding] = 0.0
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_topk_idx = torch.empty_like(ref_topk_idx)
fused_topk_weights = torch.empty_like(ref_topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
is_padding=is_padding,
)
torch.accelerator.synchronize()
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_topk_idx, ref_topk_idx)
assert torch.equal(
fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8),
)
+3 -3
View File
@@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp):
@pytest.mark.parametrize(
"model_arch,supported",
[
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
("Qwen4ExpMTP", not current_platform.is_rocm()),
("Qwen4ExpForCausalLM", True),
("Qwen4ExpForConditionalGeneration", True),
("Qwen4ExpMTP", True),
("LlamaForCausalLM", False),
],
)
+4 -1
View File
@@ -161,12 +161,15 @@ class TestReasoningEndTokenIds:
)
engine = _make_engine(cfg)
assert engine.reasoning_end_token_ids == {201, 202}
assert engine.implicit_reasoning_end_token_ids == {202}
assert engine.find_reasoning_end([5, 201]) == ReasoningEnd(1, False)
assert engine.find_reasoning_end([5, 202]) == ReasoningEnd(1, True)
def test_transition_staying_in_reasoning_is_ignored(self):
cfg = _with_reasoning_exits(("THINK_START", ParserState.REASONING, ()))
assert _make_engine(cfg).reasoning_end_token_ids == {201}
engine = _make_engine(cfg)
assert engine.reasoning_end_token_ids == {201}
assert engine.implicit_reasoning_end_token_ids == frozenset()
def test_unreported_exit_fails_closed(self):
cfg = _with_reasoning_exits(
@@ -6,11 +6,16 @@ from vllm import ModelRegistry
def register():
# Test directly passing the model
from .my_opt import MyOPTForCausalLM
from .my_opt import MyOPTForCausalLM, MyOPTRawInputTokensForCausalLM
if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
if "MyOPTRawInputTokensForCausalLM" not in ModelRegistry.get_supported_archs():
ModelRegistry.register_model(
"MyOPTRawInputTokensForCausalLM", MyOPTRawInputTokensForCausalLM
)
# Test passing lazy model
if "MyGemma2Embedding" not in ModelRegistry.get_supported_archs():
ModelRegistry.register_model(
@@ -5,6 +5,7 @@
import torch
from vllm.model_executor.models.opt import OPTForCausalLM
from vllm.sequence import IntermediateTensors
class MyOPTForCausalLM(OPTForCausalLM):
@@ -15,3 +16,21 @@ class MyOPTForCausalLM(OPTForCausalLM):
logits.zero_()
logits[:, 0] += 1.0
return logits
class MyOPTRawInputTokensForCausalLM(MyOPTForCausalLM):
# Like models whose decoder layers read token ids, this model needs
# input_ids on every pipeline-parallel stage, not only the first.
requires_raw_input_tokens = True
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
assert input_ids is not None, "requires_raw_input_tokens model got no input_ids"
return super().forward(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -7,7 +7,7 @@ chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
assert chatml_jinja_path.exists()
def run_and_test_dummy_opt_api_server(model, tp=1):
def run_and_test_dummy_opt_api_server(model, tp=1, extra_args=(), env_dict=None):
# the model is registered through the plugin
server_args = [
"--gpu-memory-utilization",
@@ -20,8 +20,9 @@ def run_and_test_dummy_opt_api_server(model, tp=1):
"dummy",
"-tp",
f"{tp}",
*extra_args,
]
with RemoteOpenAIServer(model, server_args) as server:
with RemoteOpenAIServer(model, server_args, env_dict=env_dict) as server:
client = server.get_client()
completion = client.chat.completions.create(
model=model,
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
pytestmark = pytest.mark.skipif(
not (has_humming() and current_platform.is_cuda()),
reason="requires humming and CUDA",
)
def test_humming_input_quant_compiles_fullgraph():
"""Humming's forward helpers look up ``DeviceInfo`` on every call; vLLM
traces them with ``fullgraph=True`` (vllm-project/humming#147)."""
import vllm.utils.humming as hm
weight_schema = hm.HummingWeightSchema(b_dtype="int4", weight_scale_group_size=128)
input_schema = hm.HummingInputSchema(a_dtype="int8")
from humming.transform import prepare_layer_config
config = prepare_layer_config(
shape_n=256,
shape_k=256,
weight_schema=weight_schema,
input_schema=input_schema,
torch_dtype=torch.bfloat16,
device=torch.device("cuda"),
)
x = torch.randn(16, 256, dtype=torch.bfloat16, device="cuda")
def quant(t: torch.Tensor):
return hm.may_process_input(config, t)
expected = quant(x)
actual = torch.compile(quant, fullgraph=True)(x)
for a, e in zip(actual, expected):
if e is None:
assert a is None
else:
torch.testing.assert_close(a, e)
@@ -105,6 +105,7 @@ def _make_mock_model_config(*, enable_prompt_embeds: bool = True) -> mock.MagicM
# `safe_load_prompt_embeds` pins each tensor to the model's hidden_size
# and dtype, so the mock must return concrete values.
mc.get_hidden_size.return_value = _MOCK_HIDDEN_SIZE
mc.hf_text_config = object()
mc.dtype = _MOCK_DTYPE
return mc
+1
View File
@@ -34,6 +34,7 @@ class MockModelConfig:
tokenizer_revision = None
tokenizer_mode = "auto"
hf_config = MockHFConfig()
hf_text_config = hf_config
encoder_config: dict[str, Any] | None = None
enable_prompt_embeds: bool = True
skip_tokenizer_init: bool = False
@@ -65,6 +65,40 @@ def _saved_tensor(tensor: torch.Tensor) -> bytes:
return buffer.getvalue()
@pytest.mark.parametrize(
"model, hidden_size, input_size, buffer_size",
[
("facebook/opt-350m", 1024, 512, 512),
("facebook/opt-125m", 768, 768, 768),
("openai/clip-vit-base-patch32", 512, 512, 1024),
("google/siglip-base-patch16-224", 768, 768, 1024),
],
)
def test_prompt_embeds_input_width(model, hidden_size, input_size, buffer_size):
"""OPT uses its token width; dual encoders retain their text input width."""
from vllm.config import ModelConfig
config = ModelConfig(
model=model,
dtype="float32",
enable_prompt_embeds=True,
)
if "opt-" not in model:
config.hf_text_config.projection_dim = buffer_size
assert config.get_hidden_size() == hidden_size
assert config.get_inputs_embeds_size() == buffer_size
tensor = torch.randn(4, input_size)
loaded = safe_load_prompt_embeds(config, _encode(_saved_tensor(tensor)))
torch.testing.assert_close(loaded, tensor)
wrong_size = hidden_size if hidden_size != input_size else input_size + 1
with pytest.raises(VLLMValidationError, match="does not match"):
safe_load_prompt_embeds(
config, _encode(_saved_tensor(torch.randn(4, wrong_size)))
)
UNPARSABLE_PAYLOADS = [
pytest.param(b"", id="empty"),
pytest.param(b"\x80", id="single-byte"),
+30 -13
View File
@@ -2132,8 +2132,8 @@ def test_draft_parallel_config_preserves_ep_without_model(
[
pytest.param("eagle3", False, 0, id="eagle3"),
pytest.param("eagle3", True, 7, id="p-eagle"),
pytest.param("dflash", True, 8, id="dflash"),
pytest.param("dspark", True, 7, id="dspark"),
pytest.param("dflash", True, 0, id="dflash"),
pytest.param("dspark", True, 0, id="dspark"),
pytest.param("mtp", False, 0, id="mtp"),
pytest.param("ngram", False, 0, id="ngram"),
pytest.param("draft_model", False, 1, id="draft-model"),
@@ -3769,34 +3769,48 @@ def test_dual_key_gumbel_requires_probabilistic_drafting():
config._check_supports_watermarking()
@pytest.mark.parametrize("method", ["eagle", "eagle3", "mtp"])
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(method):
@pytest.mark.parametrize(
("method", "parallel_drafting"),
[
("eagle", False),
("eagle3", False),
("mtp", False),
("dspark", True),
("dflash", True),
],
)
def test_dual_key_gumbel_supports_probabilistic_speculative_decoding(
method, parallel_drafting
):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
method=method,
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=False,
parallel_drafting=parallel_drafting,
)
config._check_supports_watermarking()
def test_dual_key_gumbel_supports_dspark():
@pytest.mark.parametrize("architecture", ["DFlash2DraftModel", "LiLiCorrDraftModel"])
def test_dual_key_gumbel_rejects_dflash_candidate_drafts(architecture):
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
method="dspark",
method="dflash",
draft_sample_method="probabilistic",
rejection_sample_method="standard",
parallel_drafting=True,
draft_model_config=SimpleNamespace(architectures=[architecture]),
)
config._check_supports_watermarking()
with pytest.raises(ValueError, match="candidate-head drafters"):
config._check_supports_watermarking()
def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
def test_dual_key_gumbel_rejects_unsupported_speculative_method():
config = _watermarked_vllm_config()
config.watermark_config = WatermarkConfig(algorithm="dual_key_gumbel", key=42)
config.speculative_config = SimpleNamespace(
@@ -3806,7 +3820,7 @@ def test_dual_key_gumbel_rejects_non_autoregressive_speculation():
parallel_drafting=False,
)
with pytest.raises(ValueError, match="autoregressive model-based"):
with pytest.raises(ValueError, match="Watermarking supports only"):
config._check_supports_watermarking()
@@ -4167,7 +4181,7 @@ def test_revision_resolved_for_model(mock_resolve):
@pytest.mark.parametrize(
"enabled,supported,is_cuda,tp_size,pp_size,error",
"enabled,supported,cuda_alike,tp_size,pp_size,error",
[
(True, True, True, 2, 1, None),
(True, False, True, 2, 1, "not supported for this model"),
@@ -4178,7 +4192,7 @@ def test_revision_resolved_for_model(mock_resolve):
(False, False, False, 2, 1, None),
],
)
def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, error):
def test_hc_sp_model_support(enabled, supported, cuda_alike, tp_size, pp_size, error):
"""Validate active HC SP configurations and ignore the flag with TP=1."""
from vllm.model_executor.models.registry import ModelRegistry
@@ -4196,7 +4210,10 @@ def test_hc_sp_model_support(enabled, supported, is_cuda, tp_size, pp_size, erro
distributed_executor_backend="mp",
)
with (
patch("vllm.config.model.current_platform.is_cuda", return_value=is_cuda),
patch(
"vllm.config.model.current_platform.is_cuda_alike",
return_value=cuda_alike,
),
patch.object(ModelRegistry, "is_hc_sp_supported_model", return_value=supported),
patch.object(ModelRegistry, "is_pp_supported_model", return_value=True),
):
+146 -2
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
from types import SimpleNamespace
from typing import _get_protocol_attrs # type: ignore
from unittest.mock import patch
@@ -15,7 +16,11 @@ from transformers import (
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.hf import HfTokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import resolve_tokenizer_args
from vllm.tokenizers.registry import (
cached_tokenizer_from_config,
resolve_tokenizer_args,
tokenizer_args_from_config,
)
def _get_missing_attrs(obj: object, target: type):
@@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str):
@pytest.mark.parametrize(
"code_revision, error_match",
[
pytest.param("never", "not a valid git identifier", id="unknown-revision"),
pytest.param(
"never",
"not a valid git identifier|couldn't find them in the cached files",
id="unknown-revision",
),
pytest.param(
"73293ac504e43f4a895688b57c65cebe24f01a38",
r"tokenization_fast\.py",
@@ -143,6 +152,141 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
_assert_tokenizer_like(tokenizer)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"code_revision",
[
None,
"c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2",
"2efddce40dddaccd37bae208c3c7ca66dbedf68a",
],
)
def test_tokenizer_config_code_revision(code_revision):
"""Load real remote config code at its pin, not the data revision."""
revision = "c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2"
with patch(
"vllm.tokenizers.registry.CachedHfTokenizer.from_pretrained",
return_value=SimpleNamespace(is_fast=True),
) as load_tokenizer:
get_tokenizer(
"hf-internal-testing/test_dynamic_model",
tokenizer_mode="hf",
trust_remote_code=True,
revision=revision,
code_revision=code_revision,
)
config = load_tokenizer.call_args.kwargs["config"]
assert (code_revision or revision) in type(config).__module__.split(".")
assert config.new_hidden_size == 12
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
)
@pytest.mark.parametrize(
"model, code_revision, kwargs, error_match",
[
pytest.param(
None,
"never",
{},
"not a valid git identifier|couldn't find them in the cached files",
id="same-repo",
),
pytest.param(
None,
"73293ac504e43f4a895688b57c65cebe24f01a38",
{},
r"tokenization_fast\.py",
id="same-repo-missing-code",
),
pytest.param(
"facebook/opt-125m",
"27dcfa74d334bc871f3234de431e71c6eeba5dd6",
{},
None,
id="separate-repo",
),
pytest.param(
None,
"never",
{"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"},
None,
id="explicit-revision",
),
pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"),
],
)
def test_config_tokenizer_code_revision(
loader, model, code_revision, kwargs, error_match
):
"""Only same-repo tokenizers inherit model code revisions; overrides win."""
revision = "08a35780b7a78794436d8654aafa26ee0165c149"
repo = "hf-internal-testing/test_dynamic_tokenizer"
config = SimpleNamespace(
model=model or repo,
tokenizer=repo,
runner_type="generate",
tokenizer_mode="hf",
tokenizer_revision=revision,
code_revision=code_revision,
trust_remote_code=True,
skip_tokenizer_init=False,
)
def load():
result = loader(config, **kwargs)
if loader is tokenizer_args_from_config:
mode, name, args, resolved_kwargs = result
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
return result
if error_match is not None:
with pytest.raises(OSError, match=error_match):
load()
else:
tokenizer = load()
reference = AutoTokenizer.from_pretrained(
repo,
revision=revision,
code_revision=kwargs.get("code_revision"),
trust_remote_code=True,
)
assert tokenizer.encode("hello world") == reference.encode("hello world")
_assert_tokenizer_like(tokenizer)
@pytest.mark.parametrize(
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
)
@pytest.mark.parametrize("tokenizer_mode", ["mistral", "auto"])
@pytest.mark.parametrize("code_revision", [None, "never"])
def test_config_mistral_code_revision(loader, tokenizer_mode, code_revision):
"""Config loading must not pass HF-only revisions to the Mistral backend."""
repo = "mistralai/Mistral-7B-Instruct-v0.3"
config = SimpleNamespace(
model=repo,
tokenizer=repo,
runner_type="generate",
tokenizer_mode=tokenizer_mode,
tokenizer_revision=None,
code_revision=code_revision,
trust_remote_code=False,
skip_tokenizer_init=False,
)
tokenizer = loader(config)
if loader is tokenizer_args_from_config:
mode, name, args, kwargs = tokenizer
tokenizer = get_tokenizer(name, *args, tokenizer_mode=mode, **kwargs)
reference = get_tokenizer(repo, tokenizer_mode="mistral")
assert isinstance(tokenizer, MistralTokenizer)
assert tokenizer.encode("hello world") == reference.encode("hello world")
def test_hub_tokenizer_loads_from_local_snapshot():
"""Transformers makes a Hub request per file when given a repo ID."""
with patch.object(
+8 -4
View File
@@ -15,6 +15,7 @@ from vllm.tokenizers.detokenizer_utils import (
)
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.protocol import TokenizerLike
from vllm.transformers_utils.repo_utils import with_retry
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.detokenizer import (
FastIncrementalDetokenizer,
@@ -95,10 +96,13 @@ def _run_incremental_decode(
@pytest.fixture
def tokenizer(tokenizer_name):
return (
MistralTokenizer.from_pretrained(tokenizer_name)
if "mistral" in tokenizer_name
else AutoTokenizer.from_pretrained(tokenizer_name)
return with_retry(
lambda: (
MistralTokenizer.from_pretrained(tokenizer_name)
if "mistral" in tokenizer_name
else AutoTokenizer.from_pretrained(tokenizer_name)
),
log_msg=f"Error loading tokenizer {tokenizer_name}",
)
+118 -1
View File
@@ -9,7 +9,7 @@ import pytest
from transformers import AutoConfig, Qwen3_5MoeConfig
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers import TokenizerLike, registry
from vllm.tokenizers.registry import (
TokenizerRegistry,
cached_get_tokenizer,
@@ -17,6 +17,7 @@ from vllm.tokenizers.registry import (
cached_tokenizer_from_config,
get_tokenizer,
resolve_tokenizer_args,
tokenizer_args_from_config,
)
@@ -116,11 +117,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
)
model_config = SimpleNamespace(
model=str(tmp_path),
skip_tokenizer_init=False,
tokenizer=str(tmp_path),
runner_type="generate",
tokenizer_mode="hf",
tokenizer_revision=None,
code_revision=None,
trust_remote_code=True,
hf_config=Qwen3_5MoeConfig(),
)
@@ -164,3 +167,117 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
if registered_config is not None:
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config
@pytest.fixture
def config_tokenizer_backend(monkeypatch):
"""Keep config helpers and dispatch real; replace Hub/backend loading."""
calls = []
class Backend:
@classmethod
def from_pretrained(cls, name, *args, **kwargs):
calls.append(kwargs)
return SimpleNamespace(is_fast=True)
monkeypatch.setattr(registry, "get_config", lambda *a, **kw: None)
monkeypatch.setattr(registry, "_maybe_register_hf_config", lambda config: None)
monkeypatch.setattr(registry, "is_mistral_model_repo", lambda **kw: True)
monkeypatch.setattr(registry, "any_pattern_in_repo_files", lambda **kw: True)
monkeypatch.setattr(
registry, "get_non_weight_snapshot_path", lambda name, **kw: name
)
monkeypatch.setattr(TokenizerRegistry, "load_tokenizer_cls", lambda mode: Backend)
cached_get_tokenizer.cache_clear()
cached_resolve_tokenizer_args.cache_clear()
yield calls
cached_get_tokenizer.cache_clear()
cached_resolve_tokenizer_args.cache_clear()
def _load_config_tokenizer(loader, config, **kwargs):
result = loader(config, **kwargs)
if loader is tokenizer_args_from_config:
mode, name, args, resolved_kwargs = result
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
return result
@pytest.mark.parametrize(
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
)
@pytest.mark.parametrize("mode", ["mistral", "auto"])
@pytest.mark.parametrize("code_revision", [None, "model-code"])
@pytest.mark.parametrize("overrides", [{}, {"code_revision": None}])
def test_config_mistral_does_not_forward_code_revision(
loader, mode, code_revision, overrides, config_tokenizer_backend
):
"""HF code revisions must not reach MistralCommonBackend's strict kwargs."""
config = SimpleNamespace(
model="test-model",
tokenizer="test-model",
runner_type="generate",
tokenizer_mode=mode,
tokenizer_revision=None,
code_revision=code_revision,
trust_remote_code=False,
skip_tokenizer_init=False,
)
_load_config_tokenizer(loader, config, **overrides)
assert len(config_tokenizer_backend) == 1
assert "code_revision" not in config_tokenizer_backend[0]
@pytest.mark.parametrize(
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
)
@pytest.mark.parametrize("missing", [False, True])
def test_config_tokenizer_without_code_revision(
loader, missing, config_tokenizer_backend
):
"""Default or minimal configs must not introduce an HF-only plugin kwarg."""
config = SimpleNamespace(
model="test-model",
tokenizer="test-model",
runner_type="generate",
tokenizer_mode="test_tokenizer",
tokenizer_revision=None,
trust_remote_code=False,
skip_tokenizer_init=False,
)
if not missing:
config.code_revision = None
_load_config_tokenizer(loader, config)
assert len(config_tokenizer_backend) == 1
assert "code_revision" not in config_tokenizer_backend[0]
@pytest.mark.parametrize(
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
)
@pytest.mark.parametrize("mode", ["hf", "cohere", "kimi_k3", "inkling", "deepseek_v4"])
@pytest.mark.parametrize(
"overrides, expected",
[
({}, "model-code"),
({"code_revision": None}, None),
({"code_revision": "override"}, "override"),
],
)
def test_config_hf_backed_tokenizer_preserves_code_revision(
loader, mode, overrides, expected, config_tokenizer_backend
):
"""Filtering Mistral kwargs must preserve HF wrappers and caller overrides."""
config = SimpleNamespace(
model="test-model",
tokenizer="test-model",
runner_type="generate",
tokenizer_mode=mode,
tokenizer_revision=None,
code_revision="model-code",
trust_remote_code=True,
skip_tokenizer_init=False,
)
_load_config_tokenizer(loader, config, **overrides)
assert len(config_tokenizer_backend) == 1
assert config_tokenizer_backend[0]["code_revision"] == expected
@@ -244,6 +244,22 @@ class TestStreamingExtraction:
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
@pytest.mark.parametrize(
"arguments",
['{"value": -2e-3}', '{"value": 1.5e3}', '{"values": [1.5e3, -2e-3]}'],
)
def test_streaming_scientific_notation(self, parser, mock_request, arguments):
model_output = (
f'{TOOL_CALLS_PREFIX}[{{"calculate": {arguments}}}]{TOOL_CALLS_SUFFIX}'
)
results = self._simulate_streaming(parser, mock_request, list(model_output))
tool_calls = self._collect_tool_calls(results)
assert len(tool_calls) == 1
assert tool_calls[0]["name"] == "calculate"
assert json.loads(tool_calls[0]["arguments"]) == json.loads(arguments)
def test_streaming_missing_tool_suffix(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
@@ -763,7 +763,7 @@ def test_flashinfer_cross_layer_layout(
_test_backend_correctness(
batch_spec=BATCH_SPECS[batch_spec_name],
model="meta-llama/Meta-Llama-3-8B",
model="Qwen/Qwen3-0.6B",
backend_to_test=[AttentionBackendEnum.FLASHINFER],
mask_mod=causal_mask_mod,
kv_cache_dtype=kv_cache_dtype,
@@ -786,7 +786,7 @@ def test_flashinfer_cross_layer_layout(
"single_prefill",
],
)
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
def test_causal_backend_correctness(
@@ -1582,7 +1582,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
_test_backend_correctness(
BATCH_SPECS["small_prefill"],
"meta-llama/Meta-Llama-3-8B",
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
use_sinks=True,
@@ -1629,7 +1629,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
batch_spec = BATCH_SPECS["small_decode"]
vllm_config = create_vllm_config(
model_name="meta-llama/Meta-Llama-3-8B",
model_name="Qwen/Qwen3-0.6B",
max_model_len=max(batch_spec.seq_lens),
block_size=16,
)
@@ -1691,7 +1691,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
_test_backend_correctness(
batch_spec,
"meta-llama/Meta-Llama-3-8B",
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
)
@@ -1994,7 +1994,7 @@ if current_platform.is_rocm():
"mixed_small",
],
)
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
def test_non_causal_backend_correctness(
default_vllm_config,
batch_spec_name: str,
@@ -67,7 +67,7 @@ def test_deepseek_rocm_adaptive_builders_support_varlen_full_graphs():
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
):
assert (
builder_cls.get_cudagraph_support(adaptive_config, SimpleNamespace())
@@ -126,7 +126,7 @@ def test_deepseek_v41_rocm_fixed_k_path_is_unchanged(
for builder_cls in (
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
):
assert (
builder_cls.get_cudagraph_support(fixed_config, SimpleNamespace())
@@ -529,7 +529,7 @@ def make_builder(
window: int = WINDOW,
max_image_tokens: int = MAX_IMG,
max_num_batched_tokens: int = 64,
model_name: str = "meta-llama/Meta-Llama-3-8B",
model_name: str = "Qwen/Qwen3-0.6B",
builder_cls: type[
DeepseekSparseSWAMetadataBuilder
] = DeepseekSparseSWAMetadataBuilder,
@@ -10,9 +10,11 @@ import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.v1.attention.backends.mla.indexer as indexer
from vllm.config import CUDAGraphMode
from vllm.config import AttentionConfig, CUDAGraphMode
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
MIN_ROWS = AttentionConfig().indexer_tp_shard_min_rows
def _chunk(start: int, end: int) -> SimpleNamespace:
"""Chunk over rows [start, end); bounds derive from the global row."""
@@ -80,11 +82,11 @@ def test_shard_exchange_reassembles_rows_only(
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
@pytest.mark.parametrize("tp_size", [2, 4])
@pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
"""The split covers every row once with >= 1 row per rank and declines
below the activation floor (or on a single rank)."""
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
n = MIN_ROWS
shapes = [
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
# uneven requests at different context depths
@@ -97,12 +99,38 @@ def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
torch.tensor(query_lens, dtype=torch.int32),
ratio,
tp_size,
n,
)
assert sizes is not None and len(sizes) == tp_size
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
lens = torch.tensor([n - 1], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size, n) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1, n) is None
def test_row_shard_threshold_is_total_not_per_rank() -> None:
"""The activation floor counts TOTAL rows regardless of tp_size: a
per-rank floor would need 128K rows at TP8, i.e. an unschedulable
--max-num-batched-tokens of 128K on 80 GB Hopper nodes."""
lens = torch.tensor([MIN_ROWS], dtype=torch.int32)
for tp_size in (2, 4, 8):
sizes = indexer.balanced_prefill_row_shard(lens, lens, 4, tp_size, MIN_ROWS)
assert sizes is not None
def test_tp2_old_per_rank_window_stays_replicated() -> None:
"""32K-64K total rows engaged on TP2 under the old per-rank floor
(16K rows/rank); the total floor now keeps them replicated — the
all_gatherv latency term is largest at small tp."""
window = torch.tensor([3 * MIN_ROWS // 4], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(window, window, 4, 2, MIN_ROWS) is None
def test_row_shard_floor_is_at_least_tp_size() -> None:
"""A floor below tp_size still leaves every rank at least one row."""
lens = torch.tensor([8], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, 4, 8, 1) is not None
assert indexer.balanced_prefill_row_shard(lens - 1, lens - 1, 4, 8, 1) is None
def test_row_sharding_gate_rejects_unsupported_configurations(
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Equivalence tests for the fused mamba "align" block-table gather.
`mamba_get_block_table_tensor`'s align branch used to be five PyTorch
statements that expanded to seven kernel launches over a `(num_reqs,)` tensor.
It is now one Triton kernel. These tests pin the fused kernel to the exact
semantics of the expression it replaced, including cases a single-request
decode never exercises but other callers can hit.
`block_table` is deliberately tested non-contiguous: the original
`torch.gather(block_table, 1, ...)` honours any layout, and this is a shared
utility (`gdn_attn`, `mamba_attn`, `short_conv_attn` and `linear_attn` all call
it), so a column stride of 1 must not be assumed.
"""
import pytest
import torch
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
WIDTH = 64
def _spec(block_size: int, num_speculative_blocks: int) -> MambaSpec:
# Only block_size and num_speculative_blocks are read by the align branch;
# shapes/dtypes just have to be well formed for the frozen dataclass.
return MambaSpec(
block_size=block_size,
shapes=((1,),),
dtypes=(torch.float32,),
num_speculative_blocks=num_speculative_blocks,
)
def _reference(block_table, seq_lens, block_size, num_spec_blocks):
"""The align branch exactly as it was before the fusion."""
start_indices = (seq_lens - 1) // block_size
start_indices = start_indices.clamp(min=0)
offsets = torch.arange(
1 + num_spec_blocks, device=block_table.device, dtype=torch.int32
)
indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64)
return torch.gather(block_table, 1, indices_to_gather)
def _make(nreq, block_size, nspec, device, noncontig=False):
torch.manual_seed(0)
width = WIDTH * 2 if noncontig else WIDTH
bt = torch.randint(0, 5000, (nreq, width), dtype=torch.int32, device=device)
if noncontig:
# Every other column: stride(1) == 2, which a kernel assuming a unit
# column stride would read straight through, silently returning the
# wrong blocks.
bt = bt[:, ::2]
assert not bt.is_contiguous()
# Keep start + nspec inside the row, which the gather already requires.
hi = max(1, (WIDTH - 1 - nspec) * block_size)
sl = torch.randint(0, hi, (nreq,), dtype=torch.int32, device=device)
return bt, sl
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
@pytest.mark.parametrize("nreq", [1, 2, 8, 64])
@pytest.mark.parametrize("block_size", [16, 64, 256])
@pytest.mark.parametrize("nspec", [0, 1, 2, 3, 4, 7])
def test_matches_reference(nreq, block_size, nspec):
bt, sl = _make(nreq, block_size, nspec, "cuda")
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
want = _reference(bt, sl, block_size, nspec)
assert got.shape == want.shape
assert got.dtype == want.dtype
torch.testing.assert_close(got, want, atol=0, rtol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
@pytest.mark.parametrize("nspec", [0, 3, 7])
def test_non_contiguous_block_table(nspec):
"""A column stride != 1 must be honoured, not assumed away."""
bt, sl = _make(8, 64, nspec, "cuda", noncontig=True)
got = mamba_get_block_table_tensor(bt, sl, _spec(64, nspec), "align")
want = _reference(bt, sl, 64, nspec)
torch.testing.assert_close(got, want, atol=0, rtol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def test_zero_seq_len_clamps_to_first_row():
"""CUDA-graph padding rows carry seq_len 0; the old code clamped to row 0."""
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
sl = torch.zeros(4, dtype=torch.int32, device="cuda")
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
torch.testing.assert_close(got, _reference(bt, sl, 64, 3), atol=0, rtol=0)
# start clamps to 0, so this is just the first 1 + nspec columns.
torch.testing.assert_close(got, bt[:, :4], atol=0, rtol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def test_zero_requests_returns_empty():
"""An empty batch must not launch a (0,) grid."""
bt = torch.zeros((0, WIDTH), dtype=torch.int32, device="cuda")
sl = torch.zeros(0, dtype=torch.int32, device="cuda")
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
assert got.shape == (0, 4)
assert got.dtype == bt.dtype
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def test_none_mode_returns_the_input_unchanged():
"""`MambaCacheMode` is Literal["align", "none"]; only "none" passes through.
An earlier version of this test also parametrised a "mode" of "all", which
is not a member of that Literal. Anything that is not "none" takes the
align branch and returns a NEW tensor, so asserting identity for it failed
-- correctly. The passthrough contract applies to "none" alone.
"""
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
sl = torch.randint(0, 1000, (4,), dtype=torch.int32, device="cuda")
assert mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "none") is bt
# ---------------------------------------------------------------------------
# CPU tensors. Every test above runs on "cuda", which is exactly how the
# regression reached review: `mamba_get_block_table_tensor` is a SHARED
# utility and its callers legitimately pass CPU tensors -- the GDN and
# linear-attention metadata builders are unit-tested with
# `DEVICE = torch.device("cpu")`. A Triton kernel cannot run on those at all,
# while the torch expression it replaced could. These run without a GPU.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("nspec", [0, 1, 3])
@pytest.mark.parametrize("block_size", [1, 16, 64])
def test_cpu_tensors_match_the_reference(nspec, block_size):
"""The align branch must work on CPU, not just on the accelerator."""
width = 2 + nspec + 8
bt = torch.arange(5 * width, dtype=torch.int32).reshape(5, width)
sl = torch.tensor([0, 1, block_size, block_size * 3 + 1, 7], dtype=torch.int32)
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
want = _reference(bt, sl, block_size, nspec)
assert torch.equal(got, want), f"cpu mismatch: {got.tolist()} != {want.tolist()}"
assert got.device == bt.device
assert got.dtype == bt.dtype
def test_cpu_none_mode_passes_through():
bt = torch.arange(12, dtype=torch.int32).reshape(3, 4)
sl = torch.tensor([1, 2, 3], dtype=torch.int32)
assert mamba_get_block_table_tensor(bt, sl, _spec(16, 3), "none") is bt
@@ -171,6 +171,11 @@ def test_paged_prefill_indices_match_gathered_prefill(
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm")
builder_cls = (
rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder
if model == "deepseek_v41"
else rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder
)
device = torch.device("cuda")
gen = torch.Generator().manual_seed(compress_ratio)
window, width, block_size, max_model_len = 16, 32, 16, 128
@@ -233,9 +238,7 @@ def test_paged_prefill_indices_match_gathered_prefill(
monkeypatch.setattr(
rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: enabled
)
return rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["swa"], vllm_config, device
).build(0, common)
return builder_cls(swa_spec, ["swa"], vllm_config, device).build(0, common)
assert build(enabled=False).prefill_swa_ragged_indices is None
md = build(enabled=True)
@@ -8,6 +8,8 @@ from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config.attention import AttentionConfig
from vllm.config.parallel import ParallelConfig
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import AttentionSelectorConfig
@@ -44,6 +46,48 @@ def cleared_attention_selector_cache():
_cached_get_attn_backend.cache_clear()
def test_deepseek_rocm_backend_registry_paths_are_model_specific():
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV4.get_path() == (
"vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend"
)
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV41.get_path() == (
"vllm.models.deepseek_v41.amd.rocm.DeepseekV41ROCMAiterMLASparseBackend"
)
@pytest.mark.parametrize(
"backend_name, expect_warning",
[
("ROCM_FLASHMLA_SPARSE_DSV4", True),
("ROCM_FLASHMLA_SPARSE_DSV41", False),
],
ids=["legacy", "canonical"],
)
def test_deepseek_v41_rocm_accepts_legacy_backend_alias(backend_name, expect_warning):
from vllm.models.deepseek_v41.amd import model as dsv41_model
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV41ROCMAiterMLAAttention,
DeepseekV41ROCMAiterMLASparseBackend,
)
config = SimpleNamespace(
attention_config=AttentionConfig(backend=backend_name),
parallel_config=ParallelConfig(),
)
with patch.object(dsv41_model.logger, "warning_once") as warning_once:
selected = dsv41_model._select_dsv4_attn_cls(config)
assert selected is DeepseekV41ROCMAiterMLAAttention
if expect_warning:
warning_once.assert_called_once()
else:
warning_once.assert_not_called()
assert (
DeepseekV41ROCMAiterMLASparseBackend.get_name() == "ROCM_FLASHMLA_SPARSE_DSV41"
)
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
+1 -1
View File
@@ -180,7 +180,7 @@ def create_standard_kv_cache_spec(
def create_vllm_config(
model_name: str = "meta-llama/Meta-Llama-3-8B",
model_name: str = "Qwen/Qwen3-0.6B",
tensor_parallel_size: int = 1,
max_model_len: int = 1024,
dtype: ModelDType | torch.dtype = "auto",
+18
View File
@@ -52,6 +52,7 @@ from vllm.v1.core.kv_cache_utils import (
hash_block_tokens,
init_none_hash,
make_block_hash_with_group_id,
null_head_len,
tensor_data,
to_event_extra_keys,
)
@@ -2156,3 +2157,20 @@ def test_wrapped_mamba_group_requires_block_zeroing():
)
assert config.needs_kv_cache_zeroing
def test_null_head_len():
null = KVCacheBlock(0, is_null=True)
real = [KVCacheBlock(i) for i in range(1, 4)]
tables = [
[],
real,
[null] * 5,
[null] * 5 + real,
[null] * 2 + real[:1] + [null] * 3 + real[1:], # not one contiguous head
]
for table in tables:
expected = next((i for i, b in enumerate(table) if not b.is_null), len(table))
assert null_head_len(table) == expected
blocks = KVCacheBlocks(([null] * 4 + real, [null] + real[:1] + [null]))
assert blocks.get_unhashed_block_ids_all_groups() == [[1, 2, 3], [1]]
+283
View File
@@ -558,6 +558,249 @@ def test_schedule_prefills_gating(has_running: bool):
assert any(r.req_id == "new0" for r in output.scheduled_new_reqs)
def _decode_one(scheduler, output, req_ids: list[str]) -> None:
"""Feed back one sampled token per request, so they keep decoding."""
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=req_ids,
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
sampled_token_ids=[[0] for _ in req_ids],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
def _start_decoding(scheduler, req_id: str = "run0"):
"""Prefill one request and take it into the decode state."""
(request,) = create_requests(
num_requests=1, num_tokens=8, max_tokens=100, req_ids=[req_id]
)
scheduler.add_request(request)
output = scheduler.schedule()
assert req_id in output.num_scheduled_tokens
_decode_one(scheduler, output, [req_id])
assert len(scheduler.running) == 1
return output
@pytest.mark.parametrize("interval", [2, 4])
def test_local_prefill_interval_holds_prefill_between_bursts(interval: int):
"""A single engine applies `prefill_schedule_interval` in the scheduler,
timed from the last prefill-carrying step: a waiting prefill is held for the
remainder of the interval while running requests keep decoding, then
admitted once the interval has elapsed.
"""
scheduler = create_scheduler(prefill_schedule_interval=interval)
# This prefill starts the interval.
_start_decoding(scheduler)
assert scheduler.last_prefill_step == scheduler.num_executed_steps
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
# Steps inside the interval decode alone; the waiting prefill is held.
for _ in range(interval - 1):
output = scheduler.schedule()
assert "new0" not in output.num_scheduled_tokens
assert "run0" in output.num_scheduled_tokens
assert new_req.status == RequestStatus.WAITING
_decode_one(scheduler, output, ["run0"])
# The interval has now elapsed, so prefill is admitted again.
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
assert scheduler.last_prefill_step == scheduler.num_executed_steps
def test_empty_schedule_does_not_advance_the_prefill_cadence():
"""An empty schedule() must not consume interval budget.
`current_step` counts every schedule() call. With async scheduling,
step_with_batch_queue() calls schedule() again whenever the batch queue has
room and has_requests() is true -- so a call lands while the previous batch
is still in flight and nothing new can be scheduled. An async KV connector
makes this common, since requests sit in lookup or WAITING_FOR_REMOTE_KVS.
Timing the cadence on current_step lets those empty calls expire the hold
early. Reported on #54627 against the LMCache MP connector.
"""
scheduler = create_scheduler(prefill_schedule_interval=4)
_start_decoding(scheduler)
# One real decode step, left in flight (no update_from_output yet).
output = scheduler.schedule()
assert output.total_num_scheduled_tokens > 0
executed_before = scheduler.num_executed_steps
step_before = scheduler.current_step
# Re-entering schedule() while that batch is still unaccounted for
# schedules nothing -- this is the async-scheduling shape.
for _ in range(3):
empty = scheduler.schedule()
assert empty.total_num_scheduled_tokens == 0
assert scheduler.current_step == step_before + 3, "current_step should advance"
assert scheduler.num_executed_steps == executed_before, (
"an empty schedule() must not count as an executed step"
)
def test_prefill_cadence_counts_executed_steps_not_schedule_calls():
"""The hold spans `interval` EXECUTED steps, however many empty schedule()
calls are interleaved.
Counting schedule() calls instead lets each empty call eat interval budget,
so with an async KV connector prefill lands on nearly every step and the
feature stops doing anything.
"""
interval = 4
scheduler = create_scheduler(prefill_schedule_interval=interval)
_start_decoding(scheduler)
prefill_at = scheduler.num_executed_steps
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
empty_calls = 0
admitted_at = None
for _ in range(50):
output = scheduler.schedule()
if "new0" in output.num_scheduled_tokens:
admitted_at = scheduler.num_executed_steps
break
if output.total_num_scheduled_tokens == 0:
empty_calls += 1
continue
# A real decode step. Leave it in flight and re-enter schedule() twice,
# as async scheduling does while the batch queue has room.
for _ in range(2):
extra = scheduler.schedule()
if "new0" in extra.num_scheduled_tokens:
admitted_at = scheduler.num_executed_steps
break
empty_calls += 1
if admitted_at is not None:
break
_decode_one(scheduler, output, ["run0"])
assert admitted_at is not None, "prefill was never admitted"
assert empty_calls > 0, "test never exercised an empty schedule() call"
# The hold is measured in executed steps. Counting schedule() calls would
# admit after far fewer, because the empty re-entries above would count.
assert admitted_at - prefill_at == interval, (
f"hold lasted {admitted_at - prefill_at} executed steps, expected "
f"{interval}; empty schedule() calls were counted against the cadence"
)
assert scheduler.current_step > scheduler.num_executed_steps
def test_local_prefill_interval_defaults_to_no_gating():
"""The default interval of 1 leaves admission exactly as it was: a waiting
prefill is admitted on the very next step.
"""
scheduler = create_scheduler()
assert scheduler.local_prefill_interval == 1
_start_decoding(scheduler)
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
def test_local_prefill_interval_left_to_engine_under_dp():
"""Under data parallelism the cadence must stay aligned across ranks, so the
DP engine core drives it and the scheduler must not gate on its own.
"""
scheduler = create_scheduler(prefill_schedule_interval=4, data_parallel_size=2)
assert scheduler.local_prefill_interval == 1
_start_decoding(scheduler)
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
def test_local_prefill_interval_never_idles_the_model():
"""With nothing running to decode in prefill's place, the interval must not
hold prefill back, or the step would be wasted entirely.
"""
scheduler = create_scheduler(prefill_schedule_interval=8)
_start_decoding(scheduler)
# Finish the only running request, leaving nothing to decode.
scheduler.finish_requests("run0", RequestStatus.FINISHED_ABORTED)
assert not scheduler.running
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
@pytest.mark.parametrize("interval", [2, 3])
def test_local_prefill_interval_defers_inflight_prefill_chunk(interval: int):
"""The single-engine interval gates prefill WORK, not just new admissions.
A request that is already partially prefilled is held for the remainder of
the interval too, exactly as `throttle_prefills` does for DP. That is the
point of the cadence -- a held step must carry no prefill at all, or decode
does not get the uninterrupted stretch the interval exists to give it.
"""
scheduler = create_scheduler(
max_num_seqs=16,
max_num_batched_tokens=50,
enable_chunked_prefill=True,
prefill_schedule_interval=interval,
)
# A short request that finishes prefill in one step -> a running decode.
(decode_req,) = create_requests(num_requests=1, num_tokens=4, req_ids=["dec0"])
scheduler.add_request(decode_req)
output = scheduler.schedule()
_decode_one(scheduler, output, ["dec0"])
assert decode_req in scheduler.running and not decode_req.is_prefill_chunk
# A long request (80 tokens, budget 50) -> prefilled in chunks. That first
# prefill closed the interval, so walk to the next step that opens it.
(chunk_req,) = create_requests(num_requests=1, num_tokens=80, req_ids=["chk0"])
scheduler.add_request(chunk_req)
for _ in range(interval - 1):
output = scheduler.schedule()
assert "chk0" not in output.num_scheduled_tokens
_decode_one(scheduler, output, ["dec0"])
output = scheduler.schedule()
assert output.num_scheduled_tokens["chk0"] > 0
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=["dec0", "chk0"],
req_id_to_index={"dec0": 0, "chk0": 1},
sampled_token_ids=[[0], []], # no token sampled for partial prefill
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
assert chunk_req.is_prefill_chunk # still mid-prefill, in running
# Held steps: the in-flight chunk is deferred along with new admissions,
# while the decode keeps running.
for _ in range(interval - 1):
output = scheduler.schedule()
assert "chk0" not in output.num_scheduled_tokens
assert "dec0" in output.num_scheduled_tokens
_decode_one(scheduler, output, ["dec0"])
# Once the interval has elapsed the chunk resumes.
output = scheduler.schedule()
assert "chk0" in output.num_scheduled_tokens
def _setup_remote_kv_resume(num_prompt_tokens: int, matched_tokens: int):
"""Drive a remote-KV request `r2` to the resume point (async load complete)
while another request `r1` is already decoding, so the step is throttle-
@@ -1916,6 +2159,46 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
@pytest.mark.parametrize(("method", "max_num_reqs"), [("dflash", 4), ("dspark", 5)])
def test_block_drafters_cap_requests_instead_of_draft_slots(
tmp_path, method, max_num_reqs
):
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
base = create_scheduler(
model=str(tmp_path),
max_num_batched_tokens=20,
num_speculative_tokens=4,
parallel_drafting=True,
skip_tokenizer_init=True,
)
# DFlash/DSpark need a resolvable draft model, so switch the ngram config in
# place and redo the config step. DSpark then has 4 lookahead tokens, not 5.
vllm_config = base.vllm_config
object.__setattr__(vllm_config.speculative_config, "method", method)
object.__setattr__(
vllm_config.speculative_config,
"draft_model_config",
SimpleNamespace(hf_config=SimpleNamespace()),
)
vllm_config._set_max_num_scheduled_tokens()
scheduler = Scheduler(
vllm_config=vllm_config,
kv_cache_config=base.kv_cache_config,
structured_output_manager=base.structured_output_manager,
block_size=base.block_size,
)
for request in create_requests(num_requests=8, num_tokens=2):
scheduler.add_request(request)
# Admission stops at 20 // num_lookahead_tokens, and no draft slots are
# charged: each admitted prompt is scheduled in full.
assert scheduler.schedule().num_scheduled_tokens == {
str(i): 2 for i in range(max_num_reqs)
}
# Note - these test cases mirror some of those in test_rejection_sampler.py
@pytest.mark.parametrize(
"spec_tokens,output_tokens,expected,expected_per_req",
@@ -1013,6 +1013,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
)
total_computed = num_tokens
def test_mamba_honors_drop_eagle_block():
"""MambaManager must drop the EAGLE lookahead block, like FullAttentionManager.
@@ -1067,7 +1068,7 @@ def test_mamba_honors_drop_eagle_block():
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=block_size,
)[0]
)[0][0]
)
# Control: with no eagle drop, both groups see the whole cached prefix.
@@ -1082,3 +1083,35 @@ def test_mamba_honors_drop_eagle_block():
f"MambaManager ignored drop_eagle_block: full-attention dropped the "
f"lookahead block (hit={full_hit}), mamba kept it (hit={mamba_hit})"
)
def test_pop_blocks_for_free_trims_null_head():
block_size = 2
spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=4,
)
block_pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
)
manager = get_sliding_window_manager(spec, block_pool)
null = block_pool.null_block
real = block_pool.get_new_blocks(2)
manager.req_to_blocks["swa"] = [null] * 3 + real
manager.req_to_blocks["dense"] = dense = block_pool.get_new_blocks(2)
assert manager.pop_blocks_for_free("swa") == real
assert manager.pop_blocks_for_free("dense") is dense # no null head, no copy
assert "swa" not in manager.req_to_blocks
assert all(b.ref_cnt == 1 for b in real + dense)
# The null block is never pooled, so touch / free_blocks leave it alone.
null_ref_cnt = null.ref_cnt
num_free_blocks = block_pool.get_num_free_blocks()
block_pool.touch([null])
block_pool.free_blocks([null])
assert null.ref_cnt == null_ref_cnt
assert block_pool.get_num_free_blocks() == num_free_blocks
+2
View File
@@ -85,6 +85,7 @@ def create_scheduler(
kv_cache_spec: KVCacheSpec | None = None,
runner: RunnerOption = "auto",
per_request_spec_decode_metrics: str = "none",
prefill_schedule_interval: int = 1,
scheduling_policy: SchedulerPolicy = "fcfs",
diffusion_canvas_length: int | None = None,
scheduler_cls: type[Scheduler] | None = None,
@@ -134,6 +135,7 @@ def create_scheduler(
enable_chunked_prefill=enable_chunked_prefill,
async_scheduling=async_scheduling,
is_encoder_decoder=model_config.is_encoder_decoder,
prefill_schedule_interval=prefill_schedule_interval,
# Ensure admission/preemption mechanics are deterministic
watermark=0.0,
policy=scheduling_policy,
@@ -160,6 +160,8 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
Skips the parts of __init__ that require a real VllmConfig / model
by patching the attributes directly after construction.
"""
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
mgr = object.__new__(EncoderCudaGraphManager)
mgr.token_budgets = sorted(budgets)
mgr.path_token_budgets = {"default": mgr.token_budgets}
@@ -1044,6 +1046,278 @@ class TestGetInputModality:
assert model.get_input_modality(mm_kwargs) == "video"
# ---------------------------------------------------------------------------
# GPU tests — DeepSeek-VL2 batched-tile encoder CUDA graph
# ---------------------------------------------------------------------------
# DeepSeek-VL2 mock constants (kept small for fast capture)
_DSV2_H = 2 # projector output side per tile (h = w)
_DSV2_DIM = 16 # hidden size
_DSV2_IMG_SIZE = 4 # each tile is 4×4 pixels
_DSV2_BUDGETS = [16, 64]
_DSV2_MAX_BATCH = 4
def _dsv2_output_tokens(tw: int, th: int) -> int:
"""Expected output tokens for an image with (tw, th) local tiles."""
h = w = _DSV2_H
return h * (w + 1) + th * h * (tw * w + 1) + 1
def _make_deepseek_vl2_mm_kwargs(
spatial_crops: list[list[int]],
device: torch.device,
dtype: torch.dtype,
) -> dict[str, Any]:
"""spatial_crops: [[tw, th], ...] per image."""
total_tiles = sum(1 + int(tw) * int(th) for tw, th in spatial_crops)
pixel_values = torch.randn(
total_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
)
return {
"pixel_values": pixel_values,
"images_spatial_crop": torch.tensor(
spatial_crops, dtype=torch.long, device=device
),
}
class MockDeepseekVL2Model(torch.nn.Module, SupportsEncoderCudaGraph):
"""Mock of DeepseekVLV2ForCausalLM for CUDA graph tests.
Simulates the batched-tile ViT pattern: pixel_values is
[N_tiles, 3, H, W], output is assembled with newlines and
view-separator in postprocess_encoder_output.
"""
def __init__(self):
super().__init__()
hw = _DSV2_H * _DSV2_H
in_features = 3 * _DSV2_IMG_SIZE * _DSV2_IMG_SIZE
self.proj = torch.nn.Linear(in_features, hw * _DSV2_DIM)
self.image_newline = torch.nn.Parameter(torch.randn(_DSV2_DIM))
self.view_seperator = torch.nn.Parameter(torch.randn(_DSV2_DIM))
self.global_view_pos = "head"
def get_max_frames_per_video(self) -> int:
return 1
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=["pixel_values"],
out_hidden_size=_DSV2_DIM,
)
def get_encoder_cudagraph_budget_range(self, vllm_config) -> tuple[int, int]:
return (_dsv2_output_tokens(1, 1), 128)
def get_encoder_cudagraph_item_specs(
self, mm_kwargs: dict[str, Any]
) -> list[EncoderItemSpec]:
h = w = _DSV2_H
specs = []
for row in mm_kwargs["images_spatial_crop"].tolist():
tw, th = int(row[0]), int(row[1])
if tw == 0 or th == 0:
break
specs.append(
EncoderItemSpec(
input_size=1 + tw * th,
output_tokens=h * (w + 1) + th * h * (tw * w + 1) + 1,
)
)
return specs
def select_encoder_cudagraph_items(
self, mm_kwargs: dict[str, Any], indices: list[int]
) -> dict[str, Any]:
pixel_values = mm_kwargs["pixel_values"]
images_spatial_crop = mm_kwargs["images_spatial_crop"]
if not indices:
return {
"pixel_values": pixel_values[:0],
"images_spatial_crop": images_spatial_crop[:0],
}
tiles_per_image = [
1 + int(row[0]) * int(row[1]) for row in images_spatial_crop.tolist()
]
cum = [0]
for t in tiles_per_image:
cum.append(cum[-1] + t)
selected_pv = torch.cat([pixel_values[cum[i] : cum[i + 1]] for i in indices])
return {
"pixel_values": selected_pv,
"images_spatial_crop": images_spatial_crop[list(indices)],
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
) -> EncoderCudaGraphCaptureInputs:
hw = _DSV2_H * _DSV2_H
max_tiles = (token_budget + hw - 1) // hw
dummy = torch.randn(
max_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
)
return EncoderCudaGraphCaptureInputs(values={"pixel_values": dummy})
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> EncoderCudaGraphReplayBuffers:
return EncoderCudaGraphReplayBuffers(
values={"pixel_values": mm_kwargs["pixel_values"]}
)
def _tile_forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
"""Project tiles: [N, 3, H, W] → [N, hw, DIM]."""
n = pixel_values.shape[0]
flat = pixel_values.view(n, -1)
hw = _DSV2_H * _DSV2_H
return self.proj(flat).view(n, hw, _DSV2_DIM)
def encoder_cudagraph_forward(
self, values: dict[str, torch.Tensor], path: str = "default"
) -> torch.Tensor:
return self._tile_forward(values["pixel_values"])
def encoder_eager_forward(
self, mm_kwargs: dict[str, Any], path: str = "default"
) -> torch.Tensor:
# Manager always runs `postprocess_encoder_output` after this, whether
# the batch went through eager or CUDA graph replay, so this must
# return the same raw per-tile shape as `encoder_cudagraph_forward`.
return self._tile_forward(mm_kwargs["pixel_values"])
def postprocess_encoder_output(
self,
outputs: dict[str, torch.Tensor],
indices: list[int],
per_item_out_tokens: list[int],
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
assert batch_mm_kwargs is not None
output = outputs["default"]
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
_, hw, n_dim = output.shape
h = w = int(hw**0.5)
tile_offset = 0
for rank, img_idx in enumerate(indices):
tw = int(images_spatial_crop[rank][0])
th = int(images_spatial_crop[rank][1])
n_tiles = 1 + tw * th
tiles = output[tile_offset : tile_offset + n_tiles]
tile_offset += n_tiles
global_f = tiles[0].view(h, w, n_dim)
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
local_f = tiles[1:].view(th, tw, h, w, n_dim)
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
if self.global_view_pos == "head":
emb = torch.cat([global_f, self.view_seperator[None], local_f])
else:
emb = torch.cat([local_f, self.view_seperator[None], global_f])
if isinstance(dest, dict):
dest[img_idx] = emb.clone() if clone else emb
else:
dest[rank] = emb.clone() if clone else emb
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
class TestDeepseekVL2CudaGraph:
"""CUDA graph tests for the batched-tile encoder pattern (DeepSeek-VL2)."""
def setup_method(self):
self.device = torch.device("cuda:0")
self.dtype = torch.float16
self.model = MockDeepseekVL2Model().to(self.device).half()
self.mgr = _make_manager_for_gpu(
self.model, _DSV2_BUDGETS, _DSV2_MAX_BATCH, self.device, self.dtype
)
self.graph_pool = current_platform.graph_pool_handle()
self.mgr.capture(graph_pool=self.graph_pool)
def test_capture_creates_one_graph_per_budget(self):
assert set(self.mgr.budget_graphs["default"].keys()) == set(_DSV2_BUDGETS)
def test_execute_returns_one_tensor_per_image(self):
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
[[1, 1], [1, 1]], self.device, self.dtype
)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert len(result) == 2
def test_output_tokens_1x1_tile(self):
# global=h*(w+1)=2*3=6, local=1*2*(1*2+1)=6, sep=1 → 13
expected = _dsv2_output_tokens(1, 1)
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[1, 1]], self.device, self.dtype)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert result[0].shape == (expected, _DSV2_DIM)
def test_output_tokens_2x1_tile(self):
# global=6, local=1*2*(2*2+1)=10, sep=1 → 17
expected = _dsv2_output_tokens(2, 1)
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[2, 1]], self.device, self.dtype)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert result[0].shape == (expected, _DSV2_DIM)
def test_output_tokens_multi_image(self):
# Image 0: tw=1,th=1 → 13; Image 1: tw=2,th=1 → 17
e0 = _dsv2_output_tokens(1, 1)
e1 = _dsv2_output_tokens(2, 1)
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
[[1, 1], [2, 1]], self.device, self.dtype
)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert result[0].shape == (e0, _DSV2_DIM)
assert result[1].shape == (e1, _DSV2_DIM)
def test_eager_fallback_when_tokens_exceed_all_budgets(self):
# tw=4,th=4: global=6, local=4*2*(4*2+1)=72, sep=1 → 79 > max budget 64
tw, th = 4, 4
expected = _dsv2_output_tokens(tw, th)
assert expected > _DSV2_BUDGETS[-1], "test precondition: must exceed max budget"
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[tw, th]], self.device, self.dtype)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert len(result) == 1
assert result[0].shape == (expected, _DSV2_DIM)
assert self.mgr.graph_misses >= 1
def test_graph_hit_counter(self):
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
[[1, 1], [1, 1]], self.device, self.dtype
)
self.mgr.execute(mm_kwargs)
assert self.mgr.graph_hits == 2
# ---------------------------------------------------------------------------
# GPU tests — video capture, replay, fallback, and mixed image+video
# ---------------------------------------------------------------------------
+3 -1
View File
@@ -368,9 +368,11 @@ async def test_dp_pause_late_request_does_not_block_drain():
# down first. The front-end sets its own copy optimistically when it
# forwards the wake, so sample only after several coordinator
# publishes (every 100ms while stats change) have overwritten it.
# Rank 0 reports the wave only after its first step, which can outlast
# the 2s on a cold device, so wait for the report rather than sample it.
await asyncio.sleep(2)
assert not long_request.done(), "the warmup request was too short to sample"
assert engine.engine_core.dp_engines_running(), (
assert await _poll_flag(engine, True, timeout=30), (
"the coordinator does not report the engines as running while they are"
)
+270 -46
View File
@@ -4,17 +4,28 @@ import asyncio
import os
import threading
import time
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AsyncExitStack
from types import SimpleNamespace
from typing import TYPE_CHECKING, cast
import msgspec
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import requests
from tests.utils import RemoteOpenAIServer
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
# Number of data parallel ranks for external LB testing
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class DictStore:
def __init__(self, values: dict[str, bytes]):
self.values = values
def check(self, keys: list[str]) -> bool:
return all(key in self.values for key in keys)
def get(self, key: str) -> bytes:
return self.values[key]
def set(self, key: str, value: bytes) -> None:
self.values[key] = value
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
client = SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_master_port=12345,
_data_parallel_master_port_list=[12346, 12347],
)
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
control_store, reconfig_store = DictStore({}), DictStore({})
coordinator.reconfig_store_ref = reconfig_store
monkeypatch.setattr(
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
)
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
for store in (control_store, reconfig_store):
# DictStore preserves insertion order: metadata must precede the epoch.
assert list(store.values)[-1] == coordinator.key("current_epoch")
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
bootstrap
)
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
monkeypatch.setattr(
coordinator, "_get_reconfig_store", lambda store=store: store
)
assert coordinator.get_status()["requested_data_parallel_size"] == 3
def test_external_elastic_ep_calculates_target_expert_redundancy():
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
eplb_config=SimpleNamespace(num_redundant_experts=0)
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
@pytest.mark.asyncio
@pytest.mark.parametrize("dp_rank", range(4))
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
"""Validate expert capacity on every rank before waiting for an epoch."""
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
data_parallel_rank=dp_rank,
elastic_ep_max_dp_size=4,
data_parallel_master_ip="127.0.0.1",
_coord_store_port=12345,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
store = DictStore({})
monkeypatch.setattr(
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
)
with pytest.raises(ValueError, match="cannot retain all model experts"):
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
assert not store.values
assert coordinator.prepared_scale is None
@pytest.mark.asyncio
async def test_external_elastic_ep_late_rank_observes_epoch_error():
coordinator = ExternalElasticEPScaleCoordinator(
cast("DPAsyncMPClient", SimpleNamespace())
)
epoch = "failed-epoch"
store = DictStore(
{
coordinator.key("current_epoch"): epoch.encode(),
coordinator.key(epoch, "error"): b"scale failed",
}
)
with pytest.raises(RuntimeError, match="scale failed"):
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing."""
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
self.server_threads: list[threading.Thread] = []
def start_rank(
self,
rank: int,
dp_size: int | None = None,
elastic_ep_scale_up_launch: bool = False,
) -> tuple[RemoteOpenAIServer, list[str]]:
"""Start one externally managed DP rank."""
dp_size = self.dp_size if dp_size is None else dp_size
server_args = self.base_server_args.copy()
server_args.extend(
[
"--data-parallel-size",
str(dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank),
"--api-server-count",
str(self.api_server_count),
]
)
env_dict = {
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
),
}
if elastic_ep_scale_up_launch:
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
server = RemoteOpenAIServer(
self.model_name,
server_args,
auto_port=False,
env_dict=env_dict,
)
print(
f"Server rank {rank} started successfully with "
f"{self.api_server_count} API servers"
)
server_and_args = (server, server_args)
self.servers.append(server_and_args)
return server_and_args
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
"""Start all server instances for external LB mode."""
"""Start all initial server instances for external LB mode."""
start_errors: list[Exception] = []
for rank in range(self.dp_size):
# Create server args for this specific rank
server_args = self.base_server_args.copy()
# Add external LB specific arguments
server_args.extend(
[
"--data-parallel-size",
str(self.dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank), # Different port for each rank
"--api-server-count",
str(self.api_server_count),
]
)
# Use a thread to start each server to allow parallel initialization
def start_server(r: int, sargs: list[str]):
def start_server(r: int):
try:
# Start the server
server = RemoteOpenAIServer(
self.model_name,
sargs,
auto_port=False,
env_dict={
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
),
},
)
server.__enter__()
print(
f"Server rank {r} started successfully with "
f"{self.api_server_count} API servers"
)
self.servers.append((server, sargs))
self.start_rank(r)
except Exception as e:
print(f"Failed to start server rank {r}: {e}")
raise
start_errors.append(e)
thread = threading.Thread(target=start_server, args=(rank, server_args))
thread = threading.Thread(target=start_server, args=(rank,))
thread.start()
self.server_threads.append(thread)
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
for thread in self.server_threads:
thread.join()
if start_errors:
start_error = start_errors[0]
raise RuntimeError("Failed to start external LB servers") from start_error
# Give servers additional time to fully initialize and coordinate
time.sleep(2)
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
assert parallel_config["enable_expert_parallel"] is expected
def _send_scale_command(
server: RemoteOpenAIServer, new_dp_size: int
) -> requests.Response:
return requests.post(
server.url_for("scale_elastic_ep"),
json={"new_data_parallel_size": new_dp_size},
timeout=600,
)
def _wait_for_scale_requests_to_start(
scale_requests: list[Future[requests.Response]],
timeout: float = 30,
) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for request in scale_requests:
if request.done():
response = request.result()
pytest.fail(
"Scale request finished before the new rank was launched: "
f"{response.status_code} {response.text}"
)
if all(request.running() for request in scale_requests):
return
time.sleep(0.1)
pytest.fail("Timed out waiting for all scale requests to start")
@pytest.mark.distributed(num_gpus=3)
@pytest.mark.skipif(
TP_SIZE != 1 or current_platform.device_count() < 3,
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
)
@pytest.mark.skipif(
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
reason="Elastic EP scaling is not supported by the Rust frontend",
)
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
server_args = [
*default_server_args,
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--attention-backend",
"TRITON_MLA",
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
"3",
"--enable-eplb",
"--eplb-config",
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
]
server_manager = ExternalLBServerManager(
ELASTIC_EP_MODEL_NAME,
dp_size=2,
api_server_count=1,
base_server_args=server_args,
)
with server_manager:
existing_servers = [server for server, _ in server_manager.servers]
with ThreadPoolExecutor(max_workers=2) as executor:
scale_requests = [
executor.submit(_send_scale_command, server, 3)
for server in existing_servers
]
_wait_for_scale_requests_to_start(scale_requests)
server_manager.start_rank(
rank=2,
dp_size=3,
elastic_ep_scale_up_launch=True,
)
for request in scale_requests:
response = request.result(timeout=600)
assert response.status_code == 200, response.text
for server, _ in server_manager.servers:
parallel_config = _get_parallel_config(server)
assert parallel_config["data_parallel_size"] == 3
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
with server.get_client() as client:
completion = client.completions.create(
model=ELASTIC_EP_MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].finish_reason in ("length", "stop")
def test_external_lb_server_info(server_manager):
servers = server_manager.servers
api_server_count = server_manager.api_server_count
_assert_expert_parallel_config(servers)
for i, (server, _) in enumerate(servers):
print(f"Testing {i=}")
@@ -5,7 +5,13 @@ import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from vllm.platforms import current_platform
from ...utils import (
assert_request_outputs_match,
compute_acceptance_len,
get_test_prompts,
)
from .._correctness import check_mtp_correctness
@@ -37,3 +43,107 @@ def test_mtp_correctness(
expected_accuracy_threshold,
vllm_runner,
)
@single_gpu_only
def test_mtp_draft_lm_head_quantization(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
vllm_runner,
):
"""A quantized draft copy of the shared lm_head only changes drafting:
greedy outputs match stock MTP and acceptance stays within 10%."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = get_test_prompts(mm_enabled=False)
results = {}
# NVFP4 needs a CUDA W4A16 kernel.
quants = (None, "fp8", "nvfp4") if current_platform.is_cuda() else (None, "fp8")
for quant in quants:
with vllm_runner(
"Qwen/Qwen3.5-0.8B-Base",
block_size=None,
enable_chunked_prefill=None,
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={
"method": "mtp",
"num_speculative_tokens": 3,
"draft_lm_head_quantization": quant,
},
disable_log_stats=False,
) as runner:
outputs = runner.llm.chat(prompts, sampling_config)
results[quant] = (outputs, compute_acceptance_len(runner.llm.get_metrics()))
ref_outputs, ref_al = results.pop(None)
for quant, (outputs, al) in results.items():
assert_request_outputs_match(
ref_outputs,
outputs,
# Not bit-exact: a different acceptance length changes the target's
# verify batch shapes, which can flip a near-tied greedy token.
required_matches=int(0.8 * len(ref_outputs)) + 1,
context=f"draft_lm_head_quantization={quant}",
)
assert al >= 0.9 * ref_al, (
f"{quant}: acceptance length {al:.3f} vs {ref_al:.3f}"
)
@single_gpu_only
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
)
def test_mtp_draft_lm_head_quantization_survives_sleep_and_reload(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
vllm_runner,
):
"""The quantized draft head is derived from the target head: it must come
back intact after a level 1 sleep and be re-derived after a target weight
reload, so outputs and acceptance match the run before."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = get_test_prompts(mm_enabled=False)
with vllm_runner(
"Qwen/Qwen3.5-0.8B-Base",
block_size=None,
enable_chunked_prefill=None,
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={
"method": "mtp",
"num_speculative_tokens": 3,
"draft_lm_head_quantization": "nvfp4",
},
enable_sleep_mode=True,
disable_log_stats=False,
) as runner:
llm = runner.llm
def run():
before = llm.get_metrics()
outputs = llm.chat(prompts, sampling_config)
return outputs, compute_acceptance_len(llm.get_metrics(), before)
ref_outputs, ref_al = run()
llm.sleep(level=1)
llm.wake_up()
outputs, al = run()
assert_request_outputs_match(
ref_outputs,
outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context="after level 1 sleep",
)
assert al == pytest.approx(ref_al, rel=0.03)
llm.collective_rpc("reload_weights")
outputs, al = run()
assert_request_outputs_match(
ref_outputs,
outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context="after weight reload",
)
assert al == pytest.approx(ref_al, rel=0.03)
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
False,
),
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
],
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
) -> None:
"""The DP master node holds a coordination store for both engine backends,
so engines there pick world-group ports at bind time. Every other
deployment keeps the pre-allocated ports and must not hold one."""
so engines there pick world-group ports at bind time. Elastic EP also
creates its coordination store before startup. The remaining cases
below keep the pre-allocated ports and must not hold a store."""
class FakeManager:
def __init__(self, **kwargs) -> None:
+52 -1
View File
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
def setsockopt(self, *_args, **_kwargs):
pass
def getsockopt_string(self, _option):
return "inproc://dummy"
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
monkeypatch.setattr(
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
@@ -210,6 +213,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
client = object.__new__(DPLBAsyncMPClient)
client.client_count = client_count
client.reqs_in_flight = {}
client.streaming_reqs = set()
client.engine_inflight = Counter()
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
@@ -407,11 +411,47 @@ def test_dplb_finished_requests_release_inflight():
assert req.request_id not in client.reqs_in_flight
@pytest.mark.parametrize("final_sent", [False, True])
def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
"""A streaming-input chunk in flight when its session finished may
re-create it on any engine, so an open session's abort reaches every
engine. Other aborts go only to the routed engine."""
client = _make_dplb_client(num_engines=3)
client.resources = SimpleNamespace(engine_dead=False)
sent: dict[bytes, list[str]] = {}
async def abort_requests(request_ids, engine):
sent[engine] = request_ids
client._abort_requests = abort_requests
chunk = make_request(SamplingParams(max_tokens=1))
chunk.resumable = True
client.get_core_engine_for_request(chunk)
if final_sent:
chunk.resumable = False
client.get_core_engine_for_request(chunk)
# The engine finished the session, dropping its routing entry.
outputs = EngineCoreOutputs(finished_requests={chunk.request_id})
asyncio.run(DPLBAsyncMPClient.process_engine_outputs(client, outputs))
client.reqs_in_flight["live"] = client.core_engines[1]
asyncio.run(client.abort_requests_async([chunk.request_id, "live"]))
if final_sent:
expected = {client.core_engines[1]: ["live"]}
else:
expected = {engine: [chunk.request_id] for engine in client.core_engines}
expected[client.core_engines[1]].append("live")
assert sent == expected
assert not client.streaming_reqs
@pytest.mark.parametrize(
("effective_size", "other_size"),
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
)
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
import msgspec
client = object.__new__(MPClient)
@@ -419,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
client.coordinator_input_address = None
client.coordinator_output_address = None
payload = msgspec.msgpack.encode(
EngineCoreReadyResponse(
@@ -441,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
instance_id="test-instance",
supports_lora=False,
max_loras=0,
coord_store_port=1234,
coordinator_input_address="tcp://127.0.0.1:1235",
coordinator_output_address="tcp://127.0.0.1:1236",
)
)
fields = msgspec.msgpack.decode(payload)
@@ -461,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client._apply_ready_response(b"")
assert cache_config.effective_attention_block_size is None
assert client.vllm_config.parallel_config._coord_store_port == 1234
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
def test_apply_ready_response_syncs_mamba_block_size():
@@ -471,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
@@ -1541,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
+132
View File
@@ -42,6 +42,7 @@ from vllm.v1.engine.output_processor import (
OutputProcessor,
RequestOutputCollector,
RequestState,
StreamingInputOutputCollector,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.loggers import PrometheusStatLogger
@@ -1642,6 +1643,137 @@ async def test_cumulative_output_collector_n():
assert third[0].text == "c"
def _make_boundary_output(
token_id: int, finish_reason: str | None, finished: bool = False
) -> RequestOutput:
return RequestOutput(
request_id="my-request-id",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="a",
token_ids=[token_id],
cumulative_logprob=None,
logprobs=None,
finish_reason=finish_reason,
)
],
finished=finished,
)
@pytest.mark.asyncio
async def test_streaming_input_collector_preserves_chunk_boundary():
"""A pending chunk-boundary marker (finish_reason on a non-finished
output) must not be erased by merging the next chunk's output into it."""
collector = StreamingInputOutputCollector(
RequestOutputKind.DELTA, request_id="my-request-id-int"
)
collector.put(_make_boundary_output(0, "stop"))
collector.put(_make_boundary_output(1, None))
collector.put(_make_boundary_output(2, None))
boundary = await collector.get()
assert boundary.outputs[0].finish_reason == "stop"
assert boundary.outputs[0].token_ids == [0]
following = await collector.get()
assert following.outputs[0].finish_reason is None
assert following.outputs[0].token_ids == [1, 2]
assert collector.output is None
assert not collector._backlog
assert not collector.ready.is_set()
def _make_streaming_input_request(
request_id: str, prompt_token_ids: list[int]
) -> EngineCoreRequest:
"""One chunk of a streaming-input (resumable) request."""
return EngineCoreRequest(
request_id=request_id,
prompt_token_ids=prompt_token_ids,
mm_features=None,
sampling_params=SamplingParams(
detokenize=False, output_kind=RequestOutputKind.DELTA
),
pooling_params=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
resumable=True,
external_req_id=f"{request_id}-ext",
)
def _add_streaming_request_with_queued_chunk(
processor: OutputProcessor, request_id: str
) -> RequestOutputCollector:
queue = RequestOutputCollector(RequestOutputKind.DELTA, request_id=request_id)
processor.add_request(
_make_streaming_input_request(request_id, [1, 2, 3]), None, queue=queue
)
processor.add_request(_make_streaming_input_request(request_id, [7, 8]), None)
return queue
def test_streaming_input_stop_is_chunk_boundary():
"""STOP is a per-chunk boundary: finished is forced False, the queued
chunk is applied, and the request state stays alive."""
processor = OutputProcessor(None, log_stats=False)
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
processed = processor.process_outputs(
[
EngineCoreOutput(
request_id="stream-req",
new_token_ids=[4],
finish_reason=FinishReason.STOP,
)
]
)
assert processed.reqs_to_abort == []
assert processor.get_num_unfinished_requests() == 1
output = queue.get_nowait()
assert output is not None
assert output.finished is False
assert output.outputs[0].finish_reason == "stop"
req_state = processor.request_states["stream-req"]
assert req_state.prompt_token_ids is not None
assert req_state.prompt_token_ids[-2:] == [7, 8]
@pytest.mark.parametrize("finish_reason", [FinishReason.ABORT, FinishReason.ERROR])
def test_streaming_input_abort_error_is_terminal(finish_reason: FinishReason):
"""ABORT/ERROR is terminal, not a chunk boundary: finished stays True,
the state is freed, and an abort is routed back to the engine core."""
processor = OutputProcessor(None, log_stats=False)
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
processed = processor.process_outputs(
[
EngineCoreOutput(
request_id="stream-req",
new_token_ids=[],
finish_reason=finish_reason,
)
]
)
assert processor.get_num_unfinished_requests() == 0
assert processed.reqs_to_abort == ["stream-req"]
output = queue.get_nowait()
assert output is not None
assert output.finished is True
assert output.outputs[0].finish_reason == str(finish_reason)
@pytest.mark.parametrize("runner", ["generate", "pooling"])
@pytest.mark.parametrize("abort_by", ["internal", "external"])
def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
@@ -281,6 +281,7 @@ class _FinishedProcess:
def test_wait_for_engine_startup_reports_watched_process_exit():
"""Update the startup test argument so it reaches frontend failure handling."""
ctx = zmq.Context()
handshake_socket = ctx.socket(zmq.ROUTER)
recv, send = connection.Pipe(duplex=False)
@@ -190,11 +190,13 @@ def _apply_ft(server, instruction: str, params: dict | None = None) -> dict:
def _kill_worker_process(server) -> None:
"""SIGKILL only the worker proc, leaving EngineCore and API server alive."""
workers = [
p
for p in psutil.Process(server.proc.pid).children(recursive=True)
if "Worker" in " ".join(p.cmdline())
]
workers = []
for p in psutil.Process(server.proc.pid).children(recursive=True):
# Skip descendants that already exited: cmdline() raises ZombieProcess
# if unreaped, NoSuchProcess if reaped. A dead worker still fails below.
with contextlib.suppress(psutil.ZombieProcess, psutil.NoSuchProcess):
if "Worker" in " ".join(p.cmdline()):
workers.append(p)
assert len(workers) == 1, f"expected 1 worker proc, found: {workers}"
workers[0].kill()
@@ -73,6 +73,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVQuantMode,
MLAAttentionSpec,
compute_layer_kv_cache_shape_bytes,
)
@@ -1551,6 +1552,95 @@ class TestNixlHandshake:
with pytest.raises(AssertionError):
worker.add_remote_agent(bad_meta, remote_tp_size=2)
@pytest.mark.parametrize(
("kv_quant_mode", "local_tp", "remote_tp", "num_kv_heads", "rejected"),
[
(KVQuantMode.NVFP4, 2, 1, 8, True),
(KVQuantMode.NVFP4, 1, 2, 8, True),
(KVQuantMode.NVFP4, 2, 2, 8, False),
# Both sides hold one replicated head per rank: no head split.
(KVQuantMode.NVFP4, 16, 8, 8, False),
(KVQuantMode.NONE, 2, 1, 8, False),
],
)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_handshake_rejects_nvfp4_kv_head_split(
self,
default_vllm_config,
dist_init,
kv_quant_mode,
local_tp,
remote_tp,
num_kv_heads,
rejected,
):
"""NVFP4 pages keep all heads' data before all heads' scales, so
heterogeneous-TP head splitting would silently corrupt the KV.
"""
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=max(1, num_kv_heads // local_tp),
head_size=16,
dtype=torch.uint8,
kv_quant_mode=kv_quant_mode,
),
)
],
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
return_value=local_tp,
):
worker = FakeNixlConnectorWorker(
vllm_config,
"engine",
hand_shake_latency=0,
kv_cache_config=kv_cache_config,
)
worker.transfer_topo.total_num_kv_heads = num_kv_heads
worker.transfer_topo.local_physical_heads = max(1, num_kv_heads // local_tp)
local_len = 4096 * worker.block_size
remote_len = (
local_len
* max(1, num_kv_heads // remote_tp)
// worker.transfer_topo.local_physical_heads
)
worker.block_len_per_layer = [local_len]
worker.num_blocks = 1
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
meta = NixlAgentMetadata(
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
kv_caches_base_addr=[0],
device_id=0,
num_blocks=1,
block_lens=[remote_len],
block_strides=[remote_len],
kv_cache_layout=worker.kv_cache_layout,
block_size=worker.block_size,
ssm_sizes=(0, 0),
attn_backend_name=worker.backend_name,
physical_blocks_per_logical_kv_block=1,
)
if rejected:
with pytest.raises(RuntimeError, match="NVFP4"):
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
else:
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
# NOTE: resource cleanup in mp backend is a bit finicky, so the order in which
# we put here is important. First run ray, it will clean up the resources, then
@@ -653,6 +653,41 @@ def test_madvise_unexpected_oserror_propagates(iid, monkeypatch):
assert exc_info.value.errno == errno.EIO
def test_segmented_region_maps_files_back_to_back(iid, monkeypatch):
"""Rows split over three shm files mapped back to back: each worker sees
the other's writes in every file, and the creator removes every file."""
monkeypatch.setattr(region_module, "_MIN_SEGMENT_BYTES", PAGE_SIZE)
regions = [
SharedOffloadRegion(
engine_id=iid,
num_chunks=5,
rank=rank,
kv_bytes_per_chunk=2 * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
num_segments=3,
)
for rank in range(2)
]
segments = regions[0]._segments
try:
assert [length // PAGE_SIZE for _, _, length in segments] == [4, 4, 2]
for rank, region in enumerate(regions):
ranges = list(region.iter_worker_slot_ranges())
first_row = rank * 5 // 2
assert ranges[0] == ((first_row * 2 + rank) * PAGE_SIZE, PAGE_SIZE)
assert sorted(ranges) == [
((row * 2 + rank) * PAGE_SIZE, PAGE_SIZE) for row in range(5)
]
region.create_next_worker_view(PAGE_SIZE).fill_(rank + 1)
rows = regions[0].base_tensor.view(5, 2, PAGE_SIZE)
assert (rows[:, 0] == 1).all() and (rows[:, 1] == 2).all()
del rows
finally:
for region in regions:
region.cleanup()
assert not any(os.path.exists(path) for path, _, _ in segments)
# ---------------------------------------------------------------------------
# Multi-worker race — concurrent construction
# ---------------------------------------------------------------------------
@@ -899,6 +934,7 @@ def _unmapped_registration_region(num_chunks, slot_size, num_workers):
region.cpu_page_size = slot_size
region._row_stride = num_workers * slot_size
region.total_size_bytes = region.num_chunks * region._row_stride
region._segments = [("unused", 0, region.total_size_bytes)]
region._base = SimpleNamespace(data_ptr=lambda: 0x100000)
region.pinned_addresses = []
region.is_pinned = False
+42
View File
@@ -15,6 +15,8 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
)
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
@@ -70,6 +72,46 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
torch.testing.assert_close(result, expected)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("use_fp64", [False, True])
def test_dspark_revisited_position_uses_step_seed(use_fp64):
"""A cached draft at a revisited position must use its current step."""
device = torch.device("cuda")
batch = 4096
spec = DSparkSpeculator.__new__(DSparkSpeculator)
spec.acceptance_estimator = None
spec._d2t_scatter_index = None
spec.draft_watermarker = None
spec.use_fp64_gumbel = use_fp64
spec.temperature = torch.ones(batch, device=device)
spec.seeds = torch.arange(batch, dtype=torch.int64, device=device)
spec._step_cols = torch.arange(2, dtype=torch.int32, device=device)
spec.draft_logits = torch.empty(batch, 2, 2, device=device)
logits = torch.tensor([0.7, 0.3], device=device).log().expand(batch, -1)
mapping = torch.arange(batch, device=device)
positions = torch.full((batch,), 22, dtype=torch.int64, device=device)
first = spec._sample_logits(logits, mapping, positions, 1)
repeated = spec._sample_logits(logits, mapping, positions, 1)
next_round = spec._sample_logits(logits, mapping, positions, 0)
torch.testing.assert_close(first, repeated)
expected = gumbel_sample(
logits,
mapping,
spec.temperature,
spec.seeds,
positions - 1,
apply_temperature=True,
is_drafting=True,
logits_cache=spec.draft_logits,
logits_cache_col=spec._step_cols[0],
use_fp64=use_fp64,
)
torch.testing.assert_close(next_round, expected)
mismatch_rate = (first != next_round).float().mean().item()
assert 0.35 < mismatch_rate < 0.5
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
monkeypatch, cuda_alike_platform
+55
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers import logits_processor
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.models.lilicorr import LiLiCorrHead
from vllm.transformers_utils.configs.eagle import EAGLEConfig
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import CandidateSampler
from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import LiLiCorrSpeculator
@@ -500,6 +501,60 @@ def test_candidate_walk_preserves_conditional_scores_and_padding(probabilistic):
assert cache[1].isneginf().all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("use_fp64", [False, True])
def test_candidate_walk_revisited_position_matches_step_seed(use_fp64):
"""The candidate walk must use the same step key as ordinary draft sampling."""
device = torch.device("cuda")
batch, steps, top_k = 4096, 2, 2
sampler = CandidateSampler(batch, steps, top_k, device)
candidates = torch.arange(top_k, device=device).expand(batch, steps, top_k)
logits = torch.tensor([0.7, 0.3], device=device).log()
scores = logits.expand(batch, steps, top_k, top_k)
mapping = torch.arange(batch, device=device)
temperature = torch.ones(batch, device=device)
seeds = mapping.to(torch.int64)
positions = torch.arange(21, 21 + steps, device=device).repeat(batch)
tokens = torch.empty(batch, steps, dtype=torch.int64, device=device)
cache = torch.empty(batch, steps, top_k, device=device)
first_round = None
for start in (21, 22):
positions.view(batch, steps).copy_(
torch.arange(start, start + steps, device=device)
)
sampler.sample(
candidates,
scores,
batch,
positions,
mapping.repeat_interleave(steps),
temperature,
seeds,
tokens,
cache,
use_fp64,
)
for step in range(steps):
expected = gumbel_sample(
logits.expand(batch, -1),
mapping,
temperature,
seeds,
positions.view(batch, steps)[:, step] - 1,
apply_temperature=True,
is_drafting=True,
logits_cache=cache,
logits_cache_col=torch.tensor(step, device=device),
use_fp64=use_fp64,
)
torch.testing.assert_close(tokens[:, step], expected)
if first_round is None:
first_round = tokens.clone()
else:
mismatch_rate = (first_round[:, 1] != tokens[:, 0]).float().mean().item()
assert 0.35 < mismatch_rate < 0.5
@pytest.mark.parametrize(
("convolution", "mismatch", "quantized"),
[
@@ -114,6 +114,10 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
def reasoning_end_token_ids(self):
return self._end_token_ids
@property
def implicit_reasoning_end_token_ids(self):
return frozenset()
def find_reasoning_end(self, token_ids):
self.windows.append(list(token_ids))
for offset, token in enumerate(token_ids):
@@ -557,6 +557,79 @@ def test_gumbel_drafted_rejection_sample_is_unbiased(num_speculative_steps: int)
)
@pytest.mark.parametrize("use_block_verification", [False, True])
@pytest.mark.parametrize("probabilistic_draft", [False, True])
def test_multiround_rejection_sample_is_unbiased(
use_block_verification: bool, probabilistic_draft: bool
):
"""Uncommitted positions must receive fresh noise on later verification rounds."""
torch.manual_seed(42)
num_trials, num_tokens, k = 32768, 6, 2
device = "cuda"
target_probs = torch.tensor(
[0.3, 0.7] if probabilistic_draft else [0.5, 0.5], device=device
)
draft_logits = torch.tensor(
[0.7, 0.3] if probabilistic_draft else [1.0, 0.0], device=device
).log()
inputs = _build_rejection_sample_inputs(
target_probs.log(), draft_logits, k, 1.0, num_trials
)
seeds = inputs["seed"]
output = torch.full((num_trials, num_tokens + k), -1, device=device)
output[:, 0] = torch.multinomial(target_probs, num_trials, replacement=True)
lengths = torch.ones(num_trials, dtype=torch.int64, device=device)
accepted = 0
while (lengths < num_tokens).any():
active = (lengths < num_tokens).nonzero().flatten()
n = active.numel()
starts = lengths[active]
local_positions = torch.arange(k + 1, device=device).repeat(n)
positions = (starts[:, None] + torch.arange(k + 1, device=device)).flatten()
draft_sampled = torch.zeros((n, k + 1), dtype=torch.int64, device=device)
draft_sampled[:, 0] = output[active, starts - 1]
logits_cache = inputs["draft_logits"] if probabilistic_draft else None
if probabilistic_draft:
for step in range(k):
draft_sampled[:, step + 1] = gumbel_sample(
draft_logits.expand(n, -1),
active,
inputs["temperature"],
seeds,
starts + step,
apply_temperature=True,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=torch.tensor(step, device=device),
)
sampled, counts = rejection_sample(
target_logits=target_probs.log().expand(n * (k + 1), -1).contiguous(),
draft_logits=logits_cache,
draft_sampled=draft_sampled.flatten(),
cu_num_logits=torch.arange(n + 1, dtype=torch.int32, device=device)
* (k + 1),
pos=positions,
idx_mapping=active,
expanded_idx_mapping=active.repeat_interleave(k + 1),
expanded_local_pos=local_positions,
temperature=inputs["temperature"],
seed=seeds,
num_speculative_steps=k,
use_block_verification=use_block_verification,
)
for column in range(k + 1):
valid = counts > column
output[active[valid], starts[valid] + column] = sampled[valid, column]
lengths[active] += counts
accepted += (counts - 1).sum().item()
assert accepted > num_trials, "The test must exercise accepted draft tokens."
for position in range(num_tokens):
_assert_distribution_match(
output[:, position], target_probs, device, label=f"position {position}"
)
@pytest.mark.parametrize("num_speculative_steps", [1, 3])
def test_greedy_rejection_sample(num_speculative_steps: int):
"""Verify that greedy (temperature=0) always outputs the target argmax
@@ -3,6 +3,7 @@
import asyncio
from collections.abc import AsyncGenerator
from unittest import mock
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -198,3 +199,52 @@ async def test_empty_input_stream_finishes_without_engine_request():
assert queue.get_nowait() is STREAM_FINISHED
llm._add_request.assert_not_awaited()
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
llm = AsyncLLM.__new__(AsyncLLM)
llm._validate_streaming_input_sampling_params = MagicMock()
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
llm.model_config = MagicMock()
llm.output_processor = MagicMock()
llm.output_processor.has_request.return_value = has_request
# The first processed request is the final (end-of-stream) request.
reqs = [MagicMock(prompt_embeds=None) for _ in range(num_chunks + 1)]
reqs[0].request_id = "r1-int"
llm.input_processor = MagicMock()
llm.input_processor.process_inputs.side_effect = reqs
return llm, reqs
async def _stream_chunks(llm: AsyncLLM, num_chunks: int) -> None:
async def chunks():
for i in range(num_chunks):
yield StreamingInput(prompt=f"frame-{i}")
with mock.patch(
"vllm.v1.engine.async_llm.extract_prompt_components",
return_value=("", None, None),
):
queue = await llm._add_streaming_input_request(
"r1", chunks(), SamplingParams(max_tokens=4)
)
task = queue._input_stream_task
assert task is not None
await task
@pytest.mark.asyncio
@pytest.mark.parametrize("session_live", [True, False])
async def test_handle_inputs_stops_after_session_finished(session_live: bool):
"""Once the session's request state is gone (aborted or errored), no
further chunks or final request are sent: they would recreate a request
state that is never cleaned up."""
llm, reqs = _make_streaming_llm(num_chunks=2, has_request=session_live)
await _stream_chunks(llm, num_chunks=2)
sent = [call.args[0] for call in llm._add_request.await_args_list]
# Both chunks then the final request, or only the first chunk.
assert sent == ([reqs[1], reqs[2], reqs[0]] if session_live else [reqs[1]])
@@ -5,9 +5,11 @@ import unittest
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
from vllm.config import DeviceConfig, VllmConfig
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -28,7 +30,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
)
from vllm.v1.outputs import LogprobsLists, ModelRunnerOutput
from vllm.v1.outputs import DraftTokenIds, LogprobsLists, ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm.v1.structured_output import StructuredOutputManager
@@ -676,3 +678,41 @@ class TestStreamingScheduler(unittest.TestCase):
cached_state_cycle1["prompt_token_ids"]
is not cached_state_cycle3["prompt_token_ids"]
), "Cached states from different cycles should be independent objects."
def _model_output(request_id: str, token_ids: list[int]) -> ModelRunnerOutput:
return ModelRunnerOutput(
req_ids=[request_id],
req_id_to_index={request_id: 0},
sampled_token_ids=[token_ids],
logprobs=None,
prompt_logprobs_dict={request_id: None},
pooler_output=[],
)
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling):
"""A stop on an accepted draft trims the step's later tokens. The next
chunk must still be computed in full rather than landing on positions
whose KV belongs to the trimmed drafts."""
scheduler = create_spec_scheduler(
num_speculative_tokens=2,
async_scheduling=async_scheduling,
speculative_method="ngram_gpu" if async_scheduling else None,
)
session = DummyRequest("session", prompt_token_ids=[1, 2, 3], max_tokens=2)
scheduler.add_request(session)
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
scheduler.update_draft_token_ids(DraftTokenIds(["session"], [[11, 12]]))
# Both drafts are accepted, but max_tokens stops at 11.
scheduler.update_from_output(
scheduler.schedule(), _model_output("session", [11, 12, 13])
)
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
scheduler.add_request(DummyRequest("session", prompt_token_ids=[7, 8]))
output = scheduler.schedule()
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
assert output.num_scheduled_tokens["session"] == 2
@@ -6,6 +6,7 @@ from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from threading import Event
import numpy as np
import pytest
from transformers import AutoTokenizer
@@ -72,20 +73,26 @@ class MockReasoner:
class MockEngineReasoner(ParserEngineReasoningAdapter):
"""Ends reasoning implicitly on ``marker``, which is also content."""
"""Ends reasoning implicitly on ``marker``, which is also content, and
explicitly on ``end_marker`` when one is given."""
def __init__(self, tokenizer, marker: int):
def __init__(self, tokenizer, marker: int, end_marker: int | None = None):
self.marker = marker
self.end_marker = end_marker
@property
def reasoning_end_token_ids(self):
return frozenset({self.marker, self.end_marker} - {None})
@property
def implicit_reasoning_end_token_ids(self):
return frozenset({self.marker})
def find_reasoning_end(self, token_ids):
ids = list(token_ids)
if self.marker in ids:
return ReasoningEnd(ids.index(self.marker), True)
return ReasoningEnd(len(ids), False)
for offset, token in enumerate(token_ids):
if token in self.reasoning_end_token_ids:
return ReasoningEnd(offset, token == self.marker)
return ReasoningEnd(len(token_ids), False)
def _single_token(tokenizer, text: str) -> int:
@@ -122,6 +129,7 @@ def _build_harness(
enable_in_reasoning: bool = False,
reasoning_parser_kwargs: dict | None = None,
structured_outputs: StructuredOutputsParams | None = None,
num_spec_tokens: int = NUM_SPEC_TOKENS,
) -> tuple[StructuredOutputManager, Request]:
vllm_config = VllmConfig(
model_config=ModelConfig(tokenizer=TOKENIZER),
@@ -132,8 +140,10 @@ def _build_harness(
),
speculative_config=SpeculativeConfig(
model="[ngram]",
num_speculative_tokens=NUM_SPEC_TOKENS,
),
num_speculative_tokens=num_spec_tokens,
)
if num_spec_tokens
else None,
)
manager = StructuredOutputManager(vllm_config)
if use_reasoner:
@@ -171,8 +181,20 @@ def _build_harness(
return manager, request
def _row_pattern(bitmask) -> str:
return "".join("U" if (row == -1).all() else "C" for row in bitmask)
def _row_pattern(bitmask, masked_ids=()) -> str:
"""U: unconstrained; R: all but `masked_ids` allowed; C: grammar rows."""
reasoning_row = np.full(bitmask.shape[1], -1, dtype=np.int32)
for token_id in masked_ids:
reasoning_row.view(np.uint32)[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
def kind(row) -> str:
if (row == -1).all():
return "U"
if masked_ids and (row == reasoning_row).all():
return "R"
return "C"
return "".join(kind(row) for row in bitmask)
def _run_real_flow(
@@ -183,6 +205,7 @@ def _run_real_flow(
expected_row_pattern: str,
expected_reasoning: bool | None,
expect_terminated: bool,
masked_ids: list[int] | None = None,
) -> None:
assert len(expected_validated) <= len(raw_drafts)
@@ -197,7 +220,7 @@ def _run_real_flow(
)
assert bitmask is not None
assert bitmask.shape[0] == len(raw_drafts) + 1
assert _row_pattern(bitmask) == expected_row_pattern
assert _row_pattern(bitmask, masked_ids or ()) == expected_row_pattern
# grammar_bitmask() must rollback any speculative state it advanced.
assert manager.validate_tokens(request, list(raw_drafts)) == expected_validated
@@ -375,7 +398,8 @@ FLOW_CASES = [
FlowCase(
raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"),
expected_validated=(" ",),
expected_row_pattern="UUUUU",
# The grammar cannot start with the marker, so reasoning rows mask it.
expected_row_pattern="RRUUU",
expected_reasoning=False,
expect_terminated=False,
engine_reasoner=True,
@@ -421,9 +445,93 @@ def test_real_flow(
expected_row_pattern=case.expected_row_pattern,
expected_reasoning=case.expected_reasoning,
expect_terminated=case.expect_terminated,
masked_ids=[reasoner_kwargs["marker"]] if case.engine_reasoner else None,
)
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize(
("structured_outputs", "raw_drafts", "expected_row_pattern"),
[
# JSON cannot start with the marker: reasoning rows mask it and keep
# every other token, including the explicit end, allowed.
pytest.param(
StructuredOutputsParams(json=JSON_SCHEMA), (" ", " "), "RRR", id="json"
),
# The grammar can start with the marker: it stays allowed and, once
# sampled, is the first token the grammar consumes.
pytest.param(
StructuredOutputsParams(regex="z[0-9]+"),
(" ", IMPLICIT_THINK_END, "1"),
"UUCC",
id="regex_starts_with_marker",
),
],
)
def test_reasoning_rows_mask_grammar_invalid_implicit_end(
tokenizer,
backend: StructuredOutputsBackend,
structured_outputs: StructuredOutputsParams,
raw_drafts: tuple[str, ...],
expected_row_pattern: str,
):
marker = _single_token(tokenizer, IMPLICIT_THINK_END)
manager, request = _build_harness(
tokenizer,
backend,
reasoning_ended=False,
reasoning_parser_kwargs={
"marker": marker,
"end_marker": _single_token(tokenizer, THINK_END),
},
structured_outputs=structured_outputs,
)
manager.reasoner_cls = MockEngineReasoner
drafts = _to_token_ids(tokenizer, raw_drafts)
assert manager.validate_tokens(request, drafts) == drafts
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert _row_pattern(bitmask, [marker]) == expected_row_pattern
@pytest.mark.parametrize("backend", BACKENDS)
def test_parallel_fill_masks_grammar_invalid_implicit_end(
tokenizer, backend: StructuredOutputsBackend
):
marker, end = _to_token_ids(tokenizer, (IMPLICIT_THINK_END, THINK_END))
manager, request = _build_harness(
tokenizer,
backend,
reasoning_ended=False,
reasoning_parser_kwargs={"marker": marker, "end_marker": end},
num_spec_tokens=0,
)
manager.reasoner_cls = MockEngineReasoner
def row_pattern() -> str:
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={},
)
assert bitmask is not None
return _row_pattern(bitmask, [marker])
manager.fill_bitmask_parallel_threshold = 0
manager.fill_bitmask_parallel_batch_size = 16
with ThreadPoolExecutor(max_workers=1) as executor:
manager.executor_for_fillmask = executor
assert row_pattern() == "R"
request.append_output_token_ids([end])
assert manager.accept_tokens(request, [end])
assert row_pattern() == "C"
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize(
("use_reasoner", "reasoning_ended", "enable_in_reasoning"),
+47 -25
View File
@@ -9,12 +9,14 @@ never addressed by the logical view.
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
import vllm.v1.hisparse.binding as attn_utils_module
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
@@ -403,49 +405,63 @@ class _FakeSharedHostRegion:
self.cleanup_calls += 1
def test_profiling_cleanup_releases_tp_shared_region_once(monkeypatch):
"""TP-shared profiling pools must use region-aware chunk cleanup."""
region = _FakeSharedHostRegion()
def test_profiling_cleanup_keeps_host_pool_for_real_kv_cache(monkeypatch):
"""The real KV cache must reuse the profiling host pool, not pin it again."""
pinned = torch.empty(8, dtype=torch.int8)
allocate_pinned = MagicMock(return_value=(pinned, pinned))
monkeypatch.setattr(
hisparse_runtime_module, "allocate_pinned_host_pool", allocate_pinned
)
monkeypatch.setattr(
hisparse_runtime_module, "check_hisparse_host_memory", lambda size: None
)
vllm_config = SimpleNamespace(instance_id="profiling-reuse")
kv_cache_config = SimpleNamespace(hisparse_shared_host_pool=False)
pool = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
backing = pool.allocate(8)
runtime = SimpleNamespace(
_host_cache=object(),
registered_host_pool=region.base_tensor,
hot_backing=object(),
shared_host_region=region,
_host_cache=backing, registered_host_pool=pinned, hot_backing=object()
)
forward_context = {
"layer": SimpleNamespace(
hisparse_cache=SimpleNamespace(runtime=runtime),
)
"layer": SimpleNamespace(hisparse_cache=SimpleNamespace(runtime=runtime))
}
released = []
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
released.append((runtimes, pinned_host_pools, shared_host_region))
monkeypatch.setattr(
attn_utils_module,
"release_pinned_state",
release_pinned_state,
)
attn_utils_module.release_hisparse_profiling_cache(forward_context)
assert released == [([runtime], [], region)]
assert not hasattr(runtime, "_host_cache")
reused = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
assert reused is pool
assert reused.allocate(8) is backing
allocate_pinned.assert_called_once_with(8)
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
@pytest.mark.parametrize("failure_phase", ["allocation", "binding", "buffers"])
def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
"""A failure after mmap allocation must not leak the shared registration."""
@pytest.mark.parametrize("shared", [False, True])
def test_init_hisparse_rolls_back_host_pool(monkeypatch, failure_phase, shared):
"""Failed initialization must release shared or private registrations."""
region = _FakeSharedHostRegion()
registered = torch.empty(4096, dtype=torch.int8)
cudart = MagicMock()
cudart.cudaHostUnregister.return_value.value = 0
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
scheduler_config=SimpleNamespace(max_num_seqs=1),
instance_id="rollback",
)
pools = []
def allocate(*args):
args[-1].shared_region = region
pools.append(args[-1])
if shared:
args[-1].shared_region = region
else:
args[-1].registered = registered
args[-1].backing = registered[:8]
if failure_phase == "allocation":
raise RuntimeError("initialization failed")
return {}
@@ -472,7 +488,13 @@ def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
{},
SimpleNamespace(),
)
assert region.cleanup_calls == 1
assert region.cleanup_calls == int(shared)
if shared:
cudart.cudaHostUnregister.assert_not_called()
else:
cudart.cudaHostUnregister.assert_called_once_with(registered.data_ptr())
assert attn_utils_module.get_hisparse_host_pool(vllm_config, None) is not pools[0]
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
def test_reshape_padded_kv_cache_strides_by_padded_page():
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The CPU worker's automatic KV cache sizing must report the right cause.
A worker whose own footprint exceeds its reservation and a node that is short
of free memory need opposite remedies, so they get separate errors.
"""
from types import SimpleNamespace
import pytest
from vllm.platforms import current_platform
from vllm.utils.cpu_resource_utils import MemoryNodeInfo
# Importing the CPU worker patches torch and the GPU buffer utilities for the
# whole process (vllm/v1/worker/cpu/shm.py), so the import is deferred into
# the tests and they run only on the CPU platform.
pytestmark = pytest.mark.skipif(
not current_platform.is_cpu(), reason="CPU worker test; needs the CPU platform"
)
GiB = 1 << 30
def _worker(monkeypatch, *, reserved, rss, total=8 * GiB, available=4 * GiB):
import vllm.v1.worker.cpu_worker as cpu_worker_mod
from vllm.v1.worker.cpu_worker import CPUWorker
worker = CPUWorker.__new__(CPUWorker)
worker.requested_cpu_memory = reserved
worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=None)
monkeypatch.setattr(CPUWorker, "_should_warm_up_model", lambda self: False)
monkeypatch.setattr(
cpu_worker_mod,
"get_allowed_cpu_list",
lambda: [SimpleNamespace(numa_node=0)],
)
monkeypatch.setattr(
cpu_worker_mod,
"get_memory_node_info",
lambda node: MemoryNodeInfo(total_memory=total, available_memory=available),
)
process = SimpleNamespace(memory_info=lambda: SimpleNamespace(rss=rss))
monkeypatch.setattr(cpu_worker_mod.psutil, "Process", lambda pid: process)
return worker
def test_footprint_above_reservation_asks_for_a_larger_reservation(monkeypatch):
# 0.76 GiB already in use against a 0.16 GiB reservation (2% of 8 GiB).
worker = _worker(monkeypatch, reserved=int(0.16 * GiB), rss=int(0.76 * GiB))
with pytest.raises(ValueError) as exc:
worker.determine_available_memory()
msg = str(exc.value)
assert "increase `--gpu-memory-utilization` to more than 0.10" in msg
assert "--kv-cache-memory-bytes" in msg
# Neither the misleading remedy nor a negative KV size.
assert "other processes" not in msg
assert "-0." not in msg
def test_short_node_memory_still_asks_to_free_memory(monkeypatch):
worker = _worker(monkeypatch, reserved=6 * GiB, rss=1 * GiB, available=2 * GiB)
with pytest.raises(ValueError, match="Reduce CPU memory used by other processes"):
worker.determine_available_memory()
def test_budget_that_fits_sizes_the_kv_cache(monkeypatch):
worker = _worker(monkeypatch, reserved=3 * GiB, rss=1 * GiB)
assert worker.determine_available_memory() == 2 * GiB
+217
View File
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.online.lm_head import (
quantized_lm_head_copy,
refresh_quantized_lm_heads,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
initialize_layerwise_reload,
record_metadata_for_reloading,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.platforms import current_platform
from vllm.v1.worker.gpu_worker import Worker
# Not a multiple of the vocab padding, so the head has padded rows.
VOCAB, HIDDEN = 4000, 512
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="Marlin needs CUDA"
)
# NVFP4 needs a CUDA W4A16 kernel; ROCm only runs the fp8 cases.
requires_nvfp4 = pytest.mark.skipif(
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
)
NVFP4 = pytest.param("nvfp4", marks=requires_nvfp4)
def _target_head(monkeypatch, seed: int = 0) -> ParallelLMHead:
# Online methods read the model dtype from the current config.
model_config = SimpleNamespace(dtype=torch.bfloat16)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.online.fp8.get_current_vllm_config",
lambda: SimpleNamespace(model_config=model_config),
)
head = ParallelLMHead(VOCAB, HIDDEN, disable_tp=True)
gen = torch.Generator().manual_seed(seed)
head.weight.data.copy_(torch.randn(head.weight.shape, generator=gen) * 0.02)
return head.to("cuda", torch.bfloat16)
def _tensors(module: torch.nn.Module) -> dict[str, torch.Tensor]:
return dict(module.named_parameters()) | dict(module.named_buffers())
@requires_cuda
@pytest.mark.parametrize(
("quantization", "max_rel_err", "min_top1"),
[("fp8", 0.05, 0.9), pytest.param("nvfp4", 0.15, 0.75, marks=requires_nvfp4)],
)
def test_tracks_bf16_head_through_logits_processor(
default_vllm_config, dist_init, monkeypatch, quantization, max_rel_err, min_top1
):
head = _target_head(monkeypatch)
target_weight = head.weight.detach().clone()
draft_head = quantized_lm_head_copy(head, quantization)
assert isinstance(draft_head, ParallelLMHead)
# The source head keeps its bf16 weight, and no bf16 copy is kept around.
assert torch.equal(head.weight, target_weight)
assert draft_head.weight.dtype != torch.bfloat16
logits_processor = LogitsProcessor(VOCAB)
hidden = torch.randn(2048, HIDDEN, device="cuda", dtype=torch.bfloat16)
ref = logits_processor(head, hidden).float()
out = logits_processor(draft_head, hidden).float()
assert out.shape == ref.shape == (2048, VOCAB)
rel_err = ((out - ref).norm() / ref.norm()).item()
top1 = (out.argmax(-1) == ref.argmax(-1)).float().mean().item()
assert rel_err < max_rel_err
assert top1 > min_top1
@requires_cuda
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
def test_refresh_follows_target_update(
default_vllm_config, dist_init, monkeypatch, quantization
):
"""After the target's weights change, the copy is re-derived in place: it
equals a fresh quantization of the new target, keeps its storage, and
frees the temporary copy without waiting for the cycle collector."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
drafter.lm_head = quantized_lm_head_copy(head, quantization)
ptrs = {k: t.data_ptr() for k, t in _tensors(drafter.lm_head).items()}
new_weight = _target_head(monkeypatch, seed=1).weight.detach()
head.weight.data.copy_(new_weight)
gc.disable()
try:
allocated = torch.accelerator.memory_allocated()
refresh_quantized_lm_heads(drafter)
assert torch.accelerator.memory_allocated() == allocated
finally:
gc.enable()
expected = _tensors(quantized_lm_head_copy(head, quantization))
for name, tensor in _tensors(drafter.lm_head).items():
assert tensor.data_ptr() == ptrs[name]
if name != "workspace":
assert torch.equal(tensor, expected[name]), name
assert torch.equal(head.weight, new_weight)
def _draft_update_worker(vllm_config, drafter: torch.nn.Module, **engine) -> Worker:
worker = object.__new__(Worker)
worker.vllm_config = vllm_config
worker.weight_transfer_engine = SimpleNamespace(
supports_draft_weight_update=True,
reset_weight_update_target=lambda: None,
**engine,
)
worker.model_runner = SimpleNamespace(get_draft_model=lambda: drafter)
worker._weight_update_active = False
worker._set_draft_weight_update_target = lambda: None
return worker
@requires_cuda
def test_draft_weight_update_refreshes_copy(
default_vllm_config, dist_init, monkeypatch
):
"""A draft weight update can write a target weight the drafter shares, such
as an embedding tied to the lm_head; the copy must follow it."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
drafter.lm_head = quantized_lm_head_copy(head, "fp8")
worker = _draft_update_worker(
default_vllm_config,
drafter,
start_weight_update=lambda: None,
finish_weight_update=lambda: None,
)
worker.start_draft_weight_update()
head.weight.data.copy_(_target_head(monkeypatch, seed=1).weight)
worker.finish_weight_update()
expected = _tensors(quantized_lm_head_copy(head, "fp8"))
for name, tensor in _tensors(drafter.lm_head).items():
assert torch.equal(tensor, expected[name]), name
@requires_cuda
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
def test_draft_weight_update_loads_lm_head(
default_vllm_config, dist_init, monkeypatch, quantization
):
"""A draft update carrying lm_head.weight loads the shared target head, as
without quantization, and the copy is re-derived from it in place."""
head = _target_head(monkeypatch)
record_metadata_for_reloading(head)
drafter = torch.nn.Module()
draft_head = quantized_lm_head_copy(head, quantization)
drafter.lm_head = draft_head
ptrs = {k: t.data_ptr() for k, t in _tensors(draft_head).items()}
new_weight = torch.randn(VOCAB, HIDDEN, device="cuda", dtype=torch.bfloat16)
def load(update_info):
param = drafter.get_parameter("lm_head.weight")
getattr(param, "weight_loader", default_weight_loader)(param, new_weight)
worker = _draft_update_worker(
default_vllm_config,
drafter,
start_weight_update=lambda: initialize_layerwise_reload(drafter),
update_weights=load,
finish_weight_update=lambda: finalize_layerwise_reload(drafter, None),
)
worker.start_draft_weight_update()
worker.update_weights({})
worker.finish_weight_update()
assert drafter.lm_head is draft_head
assert torch.equal(head.weight[:VOCAB], new_weight)
expected = _tensors(quantized_lm_head_copy(head, quantization))
for name, tensor in _tensors(draft_head).items():
assert tensor.data_ptr() == ptrs[name]
if name != "workspace":
assert torch.equal(tensor, expected[name]), name
hidden = torch.randn(64, HIDDEN, device="cuda", dtype=torch.bfloat16)
assert LogitsProcessor(VOCAB)(draft_head, hidden).isfinite().all()
@requires_cuda
@requires_nvfp4
def test_layerwise_reload_of_drafter_keeps_copy(
default_vllm_config, dist_init, monkeypatch
):
"""A drafter weight update reloads the drafter layerwise; the derived copy
must come back unchanged, not be re-quantized."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
record_metadata_for_reloading(drafter)
drafter.lm_head = quantized_lm_head_copy(head, "nvfp4")
before = {k: t.clone() for k, t in _tensors(drafter.lm_head).items()}
def fail(*args, **kwargs):
raise AssertionError("the copy was re-quantized")
monkeypatch.setattr(
drafter.lm_head.quant_method, "process_weights_after_loading", fail
)
initialize_layerwise_reload(drafter)
finalize_layerwise_reload(drafter, None)
after = _tensors(drafter.lm_head)
assert after.keys() == before.keys()
for name, tensor in before.items():
assert torch.equal(after[name], tensor), name
+2 -4
View File
@@ -302,10 +302,8 @@ def _sync_dp(
reduced[4] = int(allow_ubatching)
reduced[5] = 8 # num_reqs
with (
patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)),
patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)),
):
dp_group = SimpleNamespace(cpu_all_reduce_=lambda t: t.copy_(reduced))
with patch.object(dp_utils, "get_dp_group", lambda: dp_group):
return dp_utils.sync_cudagraph_and_dp_padding(
cudagraph_manager=cudagraph_manager,
desired_batch_desc=BatchExecutionDescriptor(
@@ -65,6 +65,9 @@ class _RecordingModelRunner:
def reset_lora_state(self) -> None:
self.reset_lora_calls += 1
def get_draft_model(self) -> nn.Module | None:
return None
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
@@ -62,7 +62,7 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> No
monkeypatch.setattr(
hisparse_binding,
"HiSparseHostPool",
"get_hisparse_host_pool",
lambda vllm_config, kv_cache_config: host_pool,
)
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
+5
View File
@@ -1257,6 +1257,8 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
worker.cache_handles = []
worker.pinned_host_pools = []
worker.shared_host_region = object()
worker.vllm_config = SimpleNamespace(instance_id="shutdown")
pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
released = False
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
@@ -1273,6 +1275,9 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
worker.shutdown()
assert released
new_pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
assert new_pool is not pool
hisparse_runtime_module.drop_hisparse_host_pool(worker.vllm_config)
class _TestReplaySSMMixer(MambaMixer2):
+2
View File
@@ -253,6 +253,8 @@ def test_philox_mixed_draft_sampling_caches_unprocessed_logits():
positions,
apply_temperature=True,
is_drafting=True,
logits_cache=cache,
logits_cache_col=cols,
)
watermarked = philox_gumbel_sample(processed_logits, contexts, 42)
expected = torch.where(skip_mask, ordinary, watermarked)
+157 -7
View File
@@ -36,6 +36,7 @@ from vllm.v1.worker.gpu.sample.watermark import (
philox_gumbel_sample,
repeated_context_mask,
)
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@@ -254,8 +255,8 @@ def test_speculative_decoding_uses_fixed_dual_key_roles():
assert draft is not None
assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a")
assert draft.num_speculative_steps == 3
assert draft.deduplicate_contexts == "all"
assert draft.deduplicate_contexts_max_history is None
assert draft.state.deduplicate_contexts == "all"
assert draft.state.deduplicate_contexts_max_history is None
def test_recovery_key_rejects_the_unsplit_dual_key_watermarker():
@@ -1324,8 +1325,8 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
deduplicate_contexts="none",
deduplicate_contexts_max_history=None,
)
draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
draft_watermarker.enabled = torch.tensor([True, False])
draft_watermarker.state.contexts.copy_(torch.tensor([[1, 2], [3, 4]]))
draft_watermarker.state.enabled = torch.tensor([True, False])
speculator.draft_watermarker = draft_watermarker
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample",
@@ -1343,7 +1344,7 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch):
)
assert torch.equal(sampled, torch.tensor([7, 4]))
assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]]))
assert torch.equal(draft_watermarker.state.contexts, torch.tensor([[2, 7], [4, 4]]))
def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch):
@@ -1602,11 +1603,12 @@ def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
[True, True],
]
assert torch.equal(
draft_watermarker.prior_contexts[:num_reqs].cpu(),
draft_watermarker.state.prior_contexts[:num_reqs].cpu(),
torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]),
)
assert torch.equal(
draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]])
draft_watermarker.state.contexts[:num_reqs].cpu(),
torch.tensor([[6, 6], [6, 6]]),
)
for step, logits in enumerate(step_logits):
# The 0-d draft step selects the logits cache column.
@@ -1661,6 +1663,154 @@ def test_draft_watermarker_cuda_graph_reads_current_prompt_lens():
assert mask.item()
def test_dflash_draft_sampler_watermarks_drafts_in_step_order(monkeypatch):
class NextTokenWatermarker:
context_width = 2
@staticmethod
def _try_sample_block(*args, **kwargs):
return None
@staticmethod
def sample(logits, contexts, random_sampler=None, skip_mask=None):
watermarked = contexts[:, -1] + 1
return WatermarkSample(
torch.where(skip_mask, random_sampler(logits), watermarked), logits
)
def ordinary_sample(logits, idx_mapping, temperature, seed, pos, **kwargs):
assert kwargs["use_fp64"]
cache, col = kwargs["logits_cache"], kwargs["logits_cache_col"]
cache[idx_mapping, col] = kwargs["logits_cache_source"]
return seed[idx_mapping] + pos
monkeypatch.setattr(
"vllm.v1.watermarking.watermarker.gumbel_sample", ordinary_sample
)
num_reqs, num_steps, vocab_size = 3, 2, 4
draft_watermarker = DraftWatermarker(
NextTokenWatermarker(),
max_num_reqs=8,
device=torch.device("cpu"),
num_speculative_steps=num_steps,
deduplicate_contexts="none",
deduplicate_contexts_max_history=None,
)
# Batch rows hold request slots 2, 0, 1. Slot 0 is greedy and row 2 opted out.
draft_watermarker.state.contexts[:num_reqs] = torch.tensor([[1, 2], [3, 4], [5, 6]])
draft_watermarker.state.enabled[:num_reqs] = torch.tensor([True, True, False])
speculator = object.__new__(DFlashSpeculator)
speculator.model = SimpleNamespace(
compute_logits=lambda hidden_states: hidden_states
)
speculator.draft_watermarker = draft_watermarker
speculator.use_fp64_gumbel = True
speculator.acceptance_estimator = None
batch_slots = torch.tensor([2, 0, 1])
logits = torch.arange(num_reqs * num_steps * vocab_size, dtype=torch.float32)
draft_logits = torch.zeros(num_reqs, num_steps, vocab_size)
sampled = speculator.sample_draft(
hidden_states=logits.view(-1, vocab_size),
sample_src_positions=torch.arange(num_reqs * num_steps),
idx_mapping=batch_slots.repeat_interleave(num_steps),
temperature=torch.tensor([0.0, 1.0, 0.5]),
seeds=torch.tensor([100, 200, 300]),
draft_step=torch.arange(num_steps).repeat(num_reqs),
draft_logits=draft_logits,
num_steps=num_steps,
)
assert torch.equal(sampled, torch.tensor([3, 4, 102, 103, 204, 205]))
assert torch.equal(draft_logits[batch_slots], logits.view(num_reqs, num_steps, -1))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("deduplicate_contexts", ["none", "single_turn", "all"])
def test_dflash_draft_sampler_matches_step_loop_on_cuda(deduplicate_contexts, dtype):
"""The fused CUDA `sample_block` is bit-exact with sampling step by step.
Covers repeated contexts, a NaN logit, and greedy, opted-out and padding
rows; padding rows are discarded, so only valid rows are compared.
"""
num_reqs, num_steps, vocab_size, max_num_reqs = 4, 5, 1500, 6
generator = torch.Generator().manual_seed(0)
logits = torch.randn(num_reqs, num_steps, vocab_size, generator=generator) * 3
logits[..., :3] += 6
logits[0, 1, 5] = float("nan")
logits = logits.to(dtype).cuda()
idx_mapping = torch.tensor([4, 2, 0, -1], dtype=torch.int32)
valid = (idx_mapping >= 0).cuda()
idx_mapping = idx_mapping.repeat_interleave(num_steps).cuda()
temperature = torch.tensor([0.7, 1.0, 0.0, 1.3, 0.6, 0.9]).cuda()
seeds = torch.arange(max_num_reqs).cuda() * 1000
pos = (torch.arange(num_steps).repeat(num_reqs) + 20).cuda()
draft_step = torch.arange(num_steps).repeat(num_reqs).cuda()
contexts = torch.randint(0, 3, (num_reqs, 2), generator=generator)
contexts[0, 0] = -1
all_token_ids = torch.randint(0, 3, (max_num_reqs, 32), generator=generator)
def run(fused: bool):
draft_watermarker = DraftWatermarker(
GumbelWatermarker(key=42, context_width=2),
max_num_reqs=max_num_reqs,
device=torch.device("cuda"),
num_speculative_steps=num_steps,
deduplicate_contexts=deduplicate_contexts,
deduplicate_contexts_max_history=None,
)
draft_watermarker.prepare(
contexts=contexts.cuda(),
enabled=torch.tensor([True, False, True, True]).cuda(),
all_token_ids=all_token_ids.cuda(),
prompt_lens=torch.full((max_num_reqs,), 4).cuda(),
total_lens=torch.full((max_num_reqs,), 24).cuda(),
)
draft_logits = torch.zeros(max_num_reqs, num_steps, vocab_size, dtype=dtype)
draft_logits = draft_logits.cuda()
args = (temperature, seeds)
if fused:
sampled = draft_watermarker.sample_block(
logits.view(-1, vocab_size),
idx_mapping,
*args,
pos,
logits_cache=draft_logits,
logits_cache_col=draft_step,
)
else:
by_step = [t.view(num_reqs, num_steps) for t in (idx_mapping, pos)]
steps = draft_step.view(num_reqs, num_steps)
sampled = torch.stack(
[
draft_watermarker.sample(
logits[:, step],
by_step[0][:, step],
*args,
by_step[1][:, step],
logits_cache=draft_logits,
logits_cache_col=steps[:, step],
apply_temperature=True,
is_drafting=True,
)
for step in range(num_steps)
],
dim=1,
).view(-1)
return (
sampled.view(num_reqs, num_steps)[valid],
draft_watermarker.state.contexts[:num_reqs][valid],
draft_watermarker.state.prior_contexts,
draft_logits,
)
for fused, reference in zip(run(fused=True), run(fused=False)):
torch.testing.assert_close(fused, reference, rtol=0, atol=0, equal_nan=True)
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator = object.__new__(DSparkSpeculator)
speculator.draft_logits = torch.zeros(2, 1, 8)