Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``--mamba-ssm-cache-dtype`` must reach GLM-5.3-Flash's KDA recurrent state
(it used to be dropped, leaving the state float32 unconditionally)."""
from types import SimpleNamespace
import pytest
import torch
from vllm.models.glm5next.common.kda import Glm5NextLinearAttention
from vllm.models.glm5next.common.model import (
Glm5NextForCausalLM,
Glm5NextForConditionalGeneration,
)
@pytest.mark.parametrize(
("ssm_dtype", "expected"),
[("auto", torch.float32), ("float32", torch.float32), ("bfloat16", torch.bfloat16)],
)
def test_kda_recurrent_state_dtype_follows_ssm_cache_dtype(ssm_dtype, expected):
model_config = SimpleNamespace(dtype=torch.bfloat16)
cache_config = SimpleNamespace(
mamba_cache_dtype="auto",
mamba_ssm_cache_dtype=ssm_dtype,
use_recoverssm=False,
)
vllm_config = SimpleNamespace(model_config=model_config, cache_config=cache_config)
want = (torch.bfloat16, expected)
assert Glm5NextForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == want
assert (
Glm5NextForConditionalGeneration.get_mamba_state_dtype_from_config(vllm_config)
== want
)
# The layer's own dtype must agree with the model-level one, since the
# KV-cache spec and the kernels read them separately.
layer = Glm5NextLinearAttention.__new__(Glm5NextLinearAttention)
layer.model_config, layer.cache_config = model_config, cache_config
assert layer.get_state_dtype() == want
@@ -289,3 +289,59 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
def test_decode_weights_staged_after_loading(dist_init) -> None:
"""The fused-decode weight copies must not depend on the weight loaders.
Loaders that bypass ``weight_loader`` (e.g. ``--load-format dummy``) left
them uninitialized, so decode could read NaN. Loading runs with grad mode
on, so the copies must also stay out of autograd.
"""
_requires_kernel()
from tests.v1.attention.utils import create_vllm_config
from vllm.config import set_current_vllm_config
from vllm.model_executor.model_loader.utils import process_weights_after_loading
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
config = KimiLinearConfig(
hidden_size=256,
linear_attn_config={
"kda_layers": [1],
"full_attn_layers": [],
"head_dim": HEAD_DIM,
"num_heads": 12,
"short_conv_kernel_size": CONV_WIDTH,
"use_full_rank_gate": True,
"gate_lower_bound": GATE_LOWER_BOUND,
},
)
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", dtype=DTYPE)
with set_current_vllm_config(vllm_config), torch.device("cuda"):
layer = KimiK3DeltaAttention(
config, vllm_config, prefix="model.layers.0.self_attn"
)
assert layer.decode_conv1d_weight is not None
with torch.no_grad():
for param in layer.parameters():
param.normal_()
layer.decode_conv1d_weight.fill_(float("nan"))
layer.decode_norm_weight.fill_(float("nan"))
process_weights_after_loading(
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
)
dim = layer.local_projection_size
conv_weights = layer.conv1d.weight.squeeze(1)
expected_conv = torch.stack(
[conv_weights[i * dim : (i + 1) * dim].transpose(0, 1) for i in range(3)]
)
torch.testing.assert_close(
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
)
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
+6 -3
View File
@@ -692,8 +692,10 @@ def test_kda_spec_decode_correctness(
pytest.param(True, False, None, True, id="aligned"),
],
)
@pytest.mark.parametrize("accepted", [(2, 8), (3, 4), (5, 8)])
@torch.inference_mode()
def test_kda_recoverssm_verify_and_group_commit(
accepted: tuple[int, int],
monkeypatch: pytest.MonkeyPatch,
lower_bound: float | None,
use_request_indices: bool,
@@ -738,7 +740,6 @@ def test_kda_recoverssm_verify_and_group_commit(
state_indices = torch.tensor(
[5, 6] if align_mode else [1, 2], dtype=torch.int32, device=DEVICE
)
accepted = [2, 8]
if use_request_indices:
global_num_accepted = torch.tensor(
[0, accepted[0], 0, accepted[1]],
@@ -891,7 +892,9 @@ def test_kda_recoverssm_verify_and_group_commit(
if align_mode:
assert block_table is not None
row = request_indices[seq_idx] if use_request_indices else seq_idx
final_block = block_table[row, (4 + commit_len) // 8]
final_block = block_table[
row, {2: 0, 3: 0, 4: 0, 5: 1, 8: 1}[commit_len]
]
committed_states[final_block] = committed_state.transpose(-1, -2)
if align_mode and 4 + commit_len >= 8:
_, boundary_state = naive_recurrent_kda(
@@ -944,7 +947,7 @@ def test_kda_recoverssm_verify_and_group_commit(
if align_mode:
assert block_table is not None
row = request_indices[seq_idx] if use_request_indices else seq_idx
block = block_table[row, (4 + commit_len) // 8]
block = block_table[row, {2: 0, 3: 0, 4: 0, 5: 1, 8: 1}[commit_len]]
source_block = state_indices[seq_idx] if align_mode else block
if conv_state_dim_first:
actual_conv = layer.kv_cache[0][block, :, :history_len]
+3 -2
View File
@@ -198,7 +198,8 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
builder_config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=SimpleNamespace(linear_key_head_dim=32)
dtype=torch.bfloat16,
hf_text_config=SimpleNamespace(linear_key_head_dim=32),
),
cache_config=SimpleNamespace(
mamba_cache_mode="align",
@@ -219,7 +220,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
scheduler_config=SimpleNamespace(max_num_seqs=4),
additional_config={},
num_speculative_tokens=2,
use_v2_model_runner=False,
use_v2_model_runner=True,
)
builder = KimiK3KDAMetadataBuilder(
kv_cache_spec=kv_cache_spec,
+114 -25
View File
@@ -4,7 +4,12 @@
import contextlib
import json
import os
import random
import shutil
import socket
import subprocess
import time
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
@@ -60,6 +65,16 @@ SPEC_CONFIG = {
}
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
MOONCAKE = {
"kv_connector": "MooncakeStoreConnector",
"kv_role": "kv_both",
"kv_load_failure_policy": "recompute",
"kv_connector_extra_config": {
"load_async": True,
"lookup_async": True,
"enable_cross_layers_blocks": False,
},
}
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
OFFLOAD = {
"kv_connector": "SimpleCPUOffloadConnector",
@@ -243,10 +258,16 @@ def _tokens_text_logprobs(response: dict):
def _reset_gpu_prefix_cache(url: str) -> None:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
deadline = time.monotonic() + 60
while True:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
if response.json()["success"]:
return
assert time.monotonic() < deadline, "GPU cache reset timed out"
time.sleep(0.1)
@dataclass(frozen=True)
@@ -354,6 +375,35 @@ def _format(turns: list[Turn]) -> str:
return "\n".join(rows)
def _check_resend_reuses_prompt_checkpoint(
deployment: Deployment, prompt_len: int
) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
if deployment.offload:
_reset_gpu_prefix_cache(url)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
)
@pytest.mark.parametrize(
"prompt_len",
[
@@ -368,28 +418,67 @@ def _format(turns: list[Turn]) -> str:
],
)
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
deployment = DEPLOYMENTS["plain"]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
@pytest.fixture
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
"""Run a Mooncake master and point the servers' store config at it."""
if shutil.which("mooncake_master") is None:
pytest.skip("mooncake_master is not installed")
port = get_open_port()
log_path = tmp_path / "mooncake_master.log"
with open(log_path, "w") as log:
master = subprocess.Popen(
[
"mooncake_master",
f"--port={port}",
f"--metrics_port={get_open_port()}",
"--enable_metric_reporting=false",
"--default_kv_lease_ttl=60000",
],
stdout=log,
stderr=subprocess.STDOUT,
)
try:
deadline = time.monotonic() + 60
while True:
assert master.poll() is None, log_path.read_text()
with (
contextlib.suppress(OSError),
socket.create_connection(("127.0.0.1", port)),
):
break
assert time.monotonic() < deadline, "Mooncake master did not start"
time.sleep(0.1)
config_path = tmp_path / "mooncake.json"
config_path.write_text(
json.dumps(
{
"metadata_server": "P2PHANDSHAKE",
"master_server_address": f"127.0.0.1:{port}",
"protocol": "rdma",
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
"global_segment_size": "4GB",
"local_buffer_size": "1GB",
}
)
)
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
yield
finally:
master.kill()
master.wait()
@pytest.mark.usefixtures("mooncake_store")
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
remain reusable from the store after a GPU cache reset."""
_check_resend_reuses_prompt_checkpoint(
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
)
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from vllm.models.kimi_k3.nvidia.model import (
KimiK3ForConditionalGeneration,
KimiLinearForCausalLM,
)
pytestmark = pytest.mark.cpu_test
class _FakeKimiLinearModel(nn.Module):
def __init__(self) -> None:
super().__init__()
self.tensor_a = nn.Parameter(torch.zeros(1))
self.tensor_c = nn.Parameter(torch.zeros(1))
self.finalized_values: list[tuple[float, float]] = []
def load_weights(self, weights):
params = dict(self.named_parameters())
loaded = set()
for name, value in weights:
params[name].data.copy_(value)
loaded.add(name)
return loaded
def finalize_mega_moe_weights(self) -> None:
self.finalized_values.append((self.tensor_a.item(), self.tensor_c.item()))
# MegaMoE finalization replaces its original weight parameters.
self.tensor_a = None
self.tensor_c = None
def test_interleaved_composite_weights_finalize_kimi_once_after_loading() -> None:
language_model = object.__new__(KimiLinearForCausalLM)
nn.Module.__init__(language_model)
language_model.config = SimpleNamespace(tie_word_embeddings=False)
language_model.model = _FakeKimiLinearModel()
model = object.__new__(KimiK3ForConditionalGeneration)
nn.Module.__init__(model)
model.language_model = language_model
model.vision_tower = nn.Module()
model.vision_tower.tensor_b = nn.Parameter(torch.zeros(1))
loaded = model.load_weights(
iter(
[
("language_model.model.tensor_a", torch.tensor([1.0])),
("vision_tower.tensor_b", torch.tensor([2.0])),
("language_model.model.tensor_c", torch.tensor([3.0])),
]
)
)
assert loaded == {
"language_model.model.tensor_a",
"vision_tower.tensor_b",
"language_model.model.tensor_c",
}
assert language_model.model.finalized_values == []
model.process_weights_after_loading()
assert language_model.model.finalized_values == [(1.0, 3.0)]
assert model.vision_tower.tensor_b.item() == 2.0
@@ -342,10 +342,13 @@ def test_full_cuda_graph(
example_prompts, max_tokens, num_logprobs
)
# Leave headroom for the HF model that ran earlier in this process on a
# 32.5 GiB MIG.
with vllm_runner(
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
gpu_memory_utilization=0.9,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
@@ -55,6 +55,8 @@ def test_max_truncation_size(vllm_model):
prompt_tokens = vllm_output[0].prompt_token_ids
# Pooling requests emit an embedding rather than generated tokens, so -1
# truncates to the full context length.
assert len(prompt_tokens) == max_model_len
@@ -13,8 +13,8 @@ import torch
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
from vllm.utils.mem_utils import format_mib
@@ -0,0 +1,174 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The shared Clef pooler against a port of the checkpoint's joint schema head,
which loops over questions where the pooler batches them."""
import math
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.models.clef_joint_schema import (
JointSchemaHead,
JointSchemaPooler,
)
from vllm.platforms import current_platform
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
DEVICE_TYPE = current_platform.device_type
HIDDEN, VOCAB = 64, 500
TYPES = [0, 1, 1, 2]
COUNTS = [2, 3, 1, 5]
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
for one record, with ``spans`` the option spans of each question."""
hidden = head.hidden_norm(hidden)
memory = head.memory_projection(hidden).unsqueeze(0)
global_vector = hidden[-1]
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
lexicals = [
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
]
options = torch.cat(
[
head.option_context_projection(c)
+ head.option_lexical_projection(lex)
+ head.option_question_projection(questions[i]).unsqueeze(0)
for i, (c, lex) in enumerate(zip(contexts, lexicals))
]
).unsqueeze(0)
for layer in head.evidence_layers:
options = layer(options, memory)
routed = torch.split(options[0], [len(q) for q in spans])
base = head.question_projection(questions)
summaries = []
for field, opts in zip(base, routed):
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
summaries.append((weights[:, None] * opts).sum(0))
fields = (
base
+ head.option_summary_norm(torch.stack(summaries))
+ head.global_projection(global_vector)
+ head.type_embedding(torch.tensor(TYPES, device=hidden.device))
).unsqueeze(0)
for layer in head.layers:
fields = layer(fields, memory)
fields = head.field_norm(fields[0])
logits = []
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
anchor = F.normalize(questions[i] + global_vector, dim=-1)
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
opts = head.option_norm(opts)
field = field.expand_as(opts)
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
joint = joint + head.residual_scorer(features).squeeze(-1)
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
return logits
@pytest.fixture
def pooler():
torch.manual_seed(0)
head = JointSchemaHead(
hidden_size=HIDDEN,
width=32,
routing_layers=2,
layers=2,
heads=4,
feedforward=48,
).eval()
with torch.no_grad():
for p in head.parameters():
p.add_(torch.randn_like(p) * 0.2)
head.to(DEVICE_TYPE)
embedding = torch.randn(VOCAB, HIDDEN, device=DEVICE_TYPE)
config = SimpleNamespace(
scheduler_config=SimpleNamespace(
enable_chunked_prefill=False, async_scheduling=False
)
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=config,
):
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
return pooler, embedding
def schema(length: int):
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
pos = 20
question_spans: list[tuple[int, int]] = []
spans: list[list[tuple[int, int]]] = []
for count in COUNTS:
question_spans.append((pos, pos + 4))
pos += 6
spans.append([])
for k in range(count):
n = 1 + (k * 3) % 4
spans[-1].append((pos, pos + n))
pos += n + 1
assert pos < length
extra = {
"question_types": TYPES,
"question_spans": [[s - length, e - length] for s, e in question_spans],
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
"option_counts": COUNTS,
}
return {"joint_schema": extra}, question_spans, spans
@torch.inference_mode()
def test_pooler_matches_reference(pooler):
pooler, embedding = pooler
lengths = [120, 90, 100]
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
hidden = torch.randn(sum(lengths), HIDDEN, device=DEVICE_TYPE)
first, first_questions, first_spans = schema(lengths[0])
last, last_questions, last_spans = schema(lengths[2])
params = [
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
# A prompt without a schema, like the profiling run's.
PoolingParams(task="token_classify", use_activation=True),
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
]
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
for i, ids in enumerate(token_ids):
padded[i, : len(ids)] = ids
metadata = PoolingMetadata(
prompt_lens=torch.tensor(lengths),
prompt_token_ids=None,
prompt_token_ids_cpu=padded,
pooling_params=params,
pooling_states=[PoolingStates() for _ in lengths],
)
metadata.build_pooling_cursor(
torch.tensor(lengths).numpy(),
seq_lens_cpu=torch.tensor(lengths),
device=hidden.device,
)
probs, empty, logits = pooler(hidden, metadata)
first_hidden, _, last_hidden = torch.split(hidden, lengths)
expected = reference_logits(
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
)
expected_probs = torch.cat([q.softmax(-1) for q in expected])
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
assert empty.numel() == 0
expected = reference_logits(
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
)
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
@@ -14,12 +14,12 @@ import pytest
import torch
from PIL import Image
from vllm.entrypoints.chat_utils import (
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.platforms import current_platform
from ....conftest import VllmRunner
@@ -14,11 +14,11 @@ import pytest
import torch
from PIL import Image
from vllm.entrypoints.chat_utils import (
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from ....conftest import VllmRunner
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
The runtime reads a question's options from the 255 code logits of its readout
at the last prompt token. vLLM's classify head must give the same code logits
for the same prompt tokens, and for an image prompt that vLLM processes itself.
"""
import pytest
import torch
from transformers import AutoModel
from vllm.assets.image import ImageAsset
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "vllm-sr/d3-edge"
STATE = (
"The order arrived damaged yesterday. The customer has a receipt and asks "
"for a replacement today."
)
QUESTIONS = {
"route": {
"type": "choice",
"instructions": "Which team should handle this request?",
"criteria": {
"returns": "Refunds, replacements and damaged deliveries",
"billing": "Payments, invoices and charges",
"technical": "Product setup and faults",
},
},
"receipt": {"type": "noul", "instructions": "Does the customer have a receipt?"},
"urgency": {
"type": "score",
"instructions": "How urgent is this request?",
"criteria": ["Routine", "Soon", "Today"],
},
"department": {
"type": "choice",
"criteria": {f"department {i}": None for i in range(40)},
},
}
IMAGE_QUESTIONS = {"photo": {"type": "noul", "instructions": "Is this a photograph?"}}
def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
return torch.softmax(logits[:count].float(), dim=-1)
@pytest.mark.parametrize("model", [MODEL])
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
runtime = reference.runtime
text = runtime.prepare(STATE, QUESTIONS)
keys = text.runnable
counts = [len(text.questions[key].keys) for key in keys]
sequences = [text.sequences[key] for key in keys]
ref_logits = list(runtime.logits(sequences, counts).cpu())
image = ImageAsset("stop_sign").pil_image
visual = runtime.prepare("Describe the attached image.", IMAGE_QUESTIONS, [image])
(image_key,) = visual.runnable
image_prompt = visual.texts[image_key]
ref_logits += list(
runtime.image_logits(
[image_prompt], visual.images, [2], visual.lengths[image_key]
).cpu()
)
counts.append(2)
del reference, runtime
cleanup_dist_env_and_memory()
with vllm_runner(model, runner="pooling", max_model_len=4096) as vllm_model:
outputs = vllm_model.llm.classify(
[{"prompt_token_ids": ids} for ids in sequences]
+ [{"prompt": image_prompt, "multi_modal_data": {"image": image}}],
use_tqdm=False,
)
# The head returns raw code logits by default.
vllm_logits = [torch.tensor(output.outputs.probs) for output in outputs]
assert all(len(logits) == 255 for logits in vllm_logits)
for ref, ours, count in zip(ref_logits, vllm_logits, counts):
ref_probs, our_probs = option_probs(ref, count), option_probs(ours, count)
torch.testing.assert_close(our_probs, ref_probs, atol=0.02, rtol=0)
@@ -5,12 +5,12 @@ import pytest
from transformers import AutoModel
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.entrypoints.chat_utils import (
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageEmbedsParam,
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from ....conftest import HfRunner, VllmRunner
@@ -18,13 +18,13 @@ import torch
from PIL import Image
from transformers import AutoModel, AutoModelForSequenceClassification, AutoProcessor
from vllm.entrypoints.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.transformers_utils.processors.nemotron_vl import (
SIGLIP_MEAN,
SIGLIP_STD,
@@ -23,13 +23,13 @@ from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.entrypoints.chat_utils import _get_embeds_data
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.platforms import current_platform
from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.utils.mistral import is_mistral_tokenizer
@@ -3,13 +3,27 @@
"""DeepSeek-V4 VL processing helpers."""
from PIL import Image
from transformers import DeepseekV4Config
from vllm.models.deepseek_v4.common.mm_preprocess import (
DeepseekV4VLProcessingInfo,
DeepseekV4VLProcessor,
build_image_block_pad_free,
)
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
VISION_CONFIG = dict(
vision_n_layers=24,
vision_dim=1024,
vision_n_heads=16,
vision_inter_dim=2816,
vision_patch_size=14,
vision_rope_theta=10000.0,
vision_downsample_ratio=3,
vision_max_n_token=384,
vision_min_pixels=147456,
vision_max_wh_ratio=8,
)
class _ConfigCtx:
@@ -34,7 +48,7 @@ def test_get_image_size_with_most_features_beats_wide_real_image():
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
"""
config = DeepseekV4Config(vision_n_layers=24)
config = DeepseekV4Config(**VISION_CONFIG)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
processor = DeepseekV4VLProcessor(config)
@@ -47,7 +61,7 @@ def test_get_image_size_with_most_features_beats_wide_real_image():
def test_get_image_size_with_most_features_respects_max_wh_ratio():
config = DeepseekV4Config(vision_n_layers=24)
config = DeepseekV4Config(**VISION_CONFIG)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
dummy = info.get_image_size_with_most_features()
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for LightOnOCR precomputed image embeddings."""
import pytest
import torch
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("tensor_input", [False, True])
@pytest.mark.parametrize("prompt_as_tokens", [False, True])
def test_lightonocr_image_embeddings_preserve_lengths_and_values(
tensor_input: bool, prompt_as_tokens: bool
):
"""Reserve each image's feature count and pass projected embeddings through."""
ctx = build_model_context(
_LIGHTON_MODEL_ID,
limit_mm_per_prompt={"image": 2},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
ctx.model_config.get_multimodal_config().enable_mm_embeds = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
hidden_size = ctx.model_config.get_hidden_size()
feature_sizes = [3, 3] if tensor_input else [3, 7]
embeddings = [
torch.full((size, hidden_size), float(i))
for i, size in enumerate(feature_sizes)
]
image_input = torch.stack(embeddings) if tensor_input else embeddings
prompt = " ".join([hf_processor.image_token] * len(embeddings))
if prompt_as_tokens:
prompt = ctx.get_tokenizer().encode(prompt)
items = processor.info.parse_mm_data({"image": image_input})
processed = processor(prompt, mm_items=items)
assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum(
feature_sizes
)
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Nemotron-Parse's device-side normalisation path."""
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ...utils import build_model_context
_MODEL_ID = "nvidia/NVIDIA-Nemotron-Parse-2.0"
def _noise_image(width: int, height: int) -> Image.Image:
rng = np.random.default_rng(0)
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(("width", "height"), [(1240, 1754), (310, 470)])
def test_mm_device_do_normalize(width: int, height: int):
device = current_platform.device_type
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
assert ctx.model_config.multimodal_config.mm_device_do_normalize
images = [_noise_image(width, height)]
prompt = [0]
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
assert raw_values.dtype == torch.uint8
assert raw_values.shape == normalized_values.shape
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
)
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("mm_device_do_normalize", [False, True])
@pytest.mark.parametrize("layout", ["ndarray_hwc", "tensor_hwc", "tensor_chw"])
def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
ctx.model_config.multimodal_config.mm_device_do_normalize = mm_device_do_normalize
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = _noise_image(310, 470)
array = np.asarray(image)
item = {
"ndarray_hwc": array,
"tensor_hwc": torch.from_numpy(array),
"tensor_chw": torch.from_numpy(array).permute(2, 0, 1),
}[layout]
def pixel_values(img: object) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [img]})
return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
@pytest.mark.usefixtures("default_vllm_config")
def test_explicit_kwargs_without_device_normalize():
ctx = build_model_context(
_MODEL_ID,
limit_mm_per_prompt={"image": 1},
mm_processor_kwargs={"do_rescale": False, "do_normalize": False},
)
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
# The remote processor ignores these kwargs and still normalises on the host.
assert values.is_floating_point()
@@ -0,0 +1,181 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs."""
import numpy as np
import pytest
import torch
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen3_asr import Qwen3ASRProcessor
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.transformers_utils.processor import cached_processor_from_config
from .test_common import _assert_inputs_equal
_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>"
@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"])
def asr_processor(request):
config = ModelConfig(
request.param,
dtype="float32",
max_model_len=4096,
limit_mm_per_prompt={"audio": 2},
mm_processor_cache_gb=0,
)
processor = MULTIMODAL_REGISTRY.create_processor(config)
native = Qwen3ASRProcessor.from_pretrained(request.param)
return processor, native
@pytest.mark.parametrize(
"samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001]
)
@pytest.mark.parametrize("num_audios", [1, 2])
def test_audio_processing_matches_hf(asr_processor, samples, num_audios):
"""Each clip must match HF independently of other clips in the request."""
processor, native = asr_processor
rng = np.random.RandomState(42)
audios = [
rng.randn(samples + index * 160).astype(np.float32)
for index in range(num_audios)
]
expected = [
native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios
]
actual = processor(
_AUDIO_PROMPT * num_audios,
mm_items=processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
expected_ids = [
token for item in expected for token in item["input_ids"][0].tolist()
]
assert actual["prompt_token_ids"] == expected_ids
masks = [item["input_features_mask"][0] for item in expected]
expected_features = torch.cat(
[
item["input_features"][0, :, mask.bool()]
for item, mask in zip(expected, masks)
],
dim=1,
)
actual_features = actual["mm_kwargs"].get_data()
torch.testing.assert_close(
actual_features["input_audio_features"], expected_features, rtol=0, atol=0
)
torch.testing.assert_close(
actual_features["audio_feature_lengths"],
torch.stack([mask.sum() for mask in masks]),
)
@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]])
def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices):
"""Partial cache hits must not change features, masks, or prompt expansion."""
processor, _ = asr_processor
config = processor.info.ctx.model_config
config.get_multimodal_config().mm_processor_cache_gb = 1
cache = MultiModalProcessorOnlyCache(config)
rng = np.random.RandomState(42)
audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)]
for index in cached_indices:
processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": audios[index]}),
hf_processor_mm_kwargs={},
cache=cache,
)
for batch in (audios, audios[::-1]):
kwargs = dict(
mm_items=processor.info.parse_mm_data({"audio": batch}),
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
processor(_AUDIO_PROMPT * 2, **kwargs),
processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache),
)
def test_text_only_processing(asr_processor):
"""Empty media input must not invoke the audio feature extractor."""
processor, native = asr_processor
prompt = "Transcribe the audio."
actual = processor(
prompt,
mm_items=processor.info.parse_mm_data({}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt)
assert actual["mm_kwargs"].get_data() == {}
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}])
def test_hf_processor_without_audio(asr_processor, hf_data):
"""The media hook must not require audio or call HF for absent waveforms."""
processor, _ = asr_processor
assert not processor._call_hf_processor(hf_data, {})
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}])
def test_postprocessing_without_audio_features(asr_processor, hf_data):
"""Non-audio output must pass through even when HF input is nonempty."""
processor, _ = asr_processor
processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])})
actual = processor._postprocess_hf_mm_data(hf_data, {}, processed)
assert actual is processed
assert set(actual) == {"input_ids"}
torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]]))
def test_chat_template_processing(asr_processor):
"""The generic processor loader must still render audio chat prompts."""
processor, native = asr_processor
chat_processor = cached_processor_from_config(processor.info.ctx.model_config)
messages = [
{"role": "system", "content": ""},
{"role": "user", "content": [{"type": "audio", "audio": "unused"}]},
]
prompt = chat_processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
expected_prompt = native.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
assert prompt == expected_prompt
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
expected = native(text=expected_prompt, audio=audio, return_tensors="pt")
actual = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audio}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist()
def test_precomputed_audio_features(asr_processor):
"""Precomputed features must bypass waveform extraction unchanged."""
processor, _ = asr_processor
processor.info.ctx.get_mm_config().enable_mm_embeds = True
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
expected = processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": audio}),
hf_processor_mm_kwargs={},
)
features = expected["mm_kwargs"].get_data()
actual = processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": features}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features)
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
f"Got short={short_tokens}, long={long_tokens}"
)
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
"""Audio and video tokens interleave by the fps the video was processed
at, as in HF, not by the default of one frame a second."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
rng = np.random.RandomState(42)
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
audio = rng.rand(32000).astype(np.float32)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
mm_kwargs: dict[str, Any] = {
"use_audio_in_video": True,
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [video], "audio": [(audio, 16000)]}
),
hf_processor_mm_kwargs=mm_kwargs,
)
expected = processor.info.get_hf_processor()(
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
)
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
@@ -15,6 +15,7 @@ from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
@@ -336,65 +337,136 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
],
)
@pytest.mark.parametrize("model_kwargs", [False, True])
@pytest.mark.parametrize("use_uuids", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
@pytest.mark.skip_global_cleanup
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
model_kwargs: bool,
use_uuids: bool,
scoped: bool,
) -> None:
"""List-valued video kwargs must be scalarized per item without mutation.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
"""Per-video options must survive duplicate items and partial cache hits."""
if scoped:
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
mm_processor_cache_gb=1,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = (
"<|vision_start|><|video_pad|><|vision_end|>"
"<|vision_start|><|video_pad|><|vision_end|>"
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = [
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
for _ in range(3)
]
for i, (frames, _) in enumerate(videos):
frames.fill(i)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
def run(indices, *, cache=None, cached_only=False):
return processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [None if cached_only else videos[i] for i in indices]}
),
mm_uuid_items=(
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
),
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
cache=cache,
)
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
expected = run(indices)
actual = run(indices, cache=cache)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert actual["mm_kwargs"] == expected["mm_kwargs"]
assert actual["mm_hashes"] == expected["mm_hashes"]
assert len(set(actual["mm_hashes"]["video"])) == 2
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
if use_uuids:
cached_only = run([2, 0], cache=cache, cached_only=True)
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
@pytest.mark.parametrize("length", [0, 1])
@pytest.mark.parametrize("use_cache", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
def test_processor_rejects_short_per_video_options(
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
) -> None:
"""Reject malformed option lists with and without a warm processor cache."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
def process(values: list[float | int]) -> None:
kwargs = {key: values}
processor(
prompt,
mm_items=processor.info.parse_mm_data({"video": videos}),
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
cache=cache,
)
if use_cache:
process([value, value])
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
process([value] * length)
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
def test_processor_profiles_fewer_videos_than_configured_options(
kwargs: dict[str, list[float] | list[int]],
) -> None:
"""Profiling may use fewer videos than the configured per-video list."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=kwargs,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
profiled = processor.get_dummy_mm_inputs({"video": 1})
assert len(profiled["mm_placeholders"]["video"]) == 1
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
def _build_video_embeds_mm_data(
@@ -97,6 +97,7 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", _MissingMultiModalModule())
object.__setattr__(model, "vision_final_layernorm", _MissingMultiModalModule())
object.__setattr__(model, "sound_encoder", None)
language_weight = object()
@@ -106,6 +107,8 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
("mlp1.0.weight", object()),
("vision_model.radio_model.encoder.weight", object()),
("sound_encoder.encoder.weight", object()),
("vision_projector.vision_final_layernorm.weight", object()),
("vision_projector.vision_final_layernorm.bias", object()),
]
)
@@ -120,6 +123,7 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", vision_model)
object.__setattr__(model, "vision_final_layernorm", None)
object.__setattr__(model, "sound_encoder", None)
language_weight = object()
@@ -145,12 +149,65 @@ def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
object.__setattr__(model, "language_model", language_model)
object.__setattr__(model, "mlp1", _AdapterModule())
object.__setattr__(model, "vision_model", vision_model)
object.__setattr__(model, "vision_final_layernorm", None)
object.__setattr__(model, "sound_encoder", None)
with pytest.raises(AssertionError):
model.load_weights([("sound_encoder.encoder.weight", object())])
def _vision_norm_model(use_layernorm=True):
model = object.__new__(NemotronH_Nano_VL_V2)
torch.nn.Module.__init__(model)
model.model_config = _ImageOnlyModelConfig()
model.language_model = _LanguageModel()
model.vision_model = _VisionModel()
model.sound_encoder = None
model.mlp1 = torch.nn.Identity()
model.vision_final_layernorm = (
torch.nn.LayerNorm(4, eps=1e-6, dtype=torch.bfloat16) if use_layernorm else None
)
model.patch_size = 2
model.downsample_ratio = 0.5
model.ps_version = "v2"
model.video_temporal_patch_size = 2
return model
@pytest.mark.parametrize("mode", ["image", "dynamic", "video"])
@pytest.mark.parametrize("use_layernorm", [True, False])
def test_vision_layernorm_is_loaded_and_applied_before_pixel_shuffle(
mode, use_layernorm
):
model = _vision_norm_model(use_layernorm)
weight = torch.tensor([2.0, 3.0, 4.0, 5.0], dtype=torch.bfloat16)
bias = torch.tensor([0.5, -0.5, 1.0, -1.0], dtype=torch.bfloat16)
model.load_weights(
[
("vision_projector.vision_final_layernorm.weight", weight),
("vision_projector.vision_final_layernorm.bias", bias),
]
if use_layernorm
else []
)
features = torch.arange(16, dtype=torch.bfloat16).reshape(1, 4, 4)
expected = (
torch.nn.functional.layer_norm(features, (4,), weight, bias, eps=1e-6)
if use_layernorm
else features
)
expected = model.pixel_shuffle(expected.reshape(1, 2, 2, 4)).reshape(1, 1, 16)
model.vision_model = lambda *args, **kwargs: (None, features)
pixels = torch.zeros(2 if mode == "video" else 1, 3, 4, 4)
if mode == "dynamic":
output = model.extract_feature_dynamic(pixels, imgs_sizes=[(4, 4)])
else:
output = model.extract_feature(
pixels, num_frames=2 if mode == "video" else None
)
torch.testing.assert_close(output, expected)
def _make_mm_items_with_video_bytes(
video_bytes: bytes,
) -> MultiModalDataItems:
+61 -23
View File
@@ -12,6 +12,7 @@ import pytest
import torch
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config import CacheConfig, ParallelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
@@ -53,23 +54,54 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
return Qwen4ExpTextConfig(**values)
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
def test_moe_sp_rejects_pipeline_parallel() -> None:
"""Automatic MoE SP must validate HC topology even without --enable-hc-sp."""
config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(num_experts=4, ple_layer_ids=[]),
),
cache_config=CacheConfig(),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
pipeline_parallel_size=2,
data_parallel_size=2,
enable_expert_parallel=True,
all2all_backend="allgather_reducescatter",
),
)
with (
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
pytest.raises(ValueError, match="MoE SP requires PP=1"),
):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
@pytest.mark.parametrize("use_sp", [False, True])
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
model.use_hc_sequence_parallel = use_sp
model.use_sequence_parallel = False
torch.nn.Module.__init__(model)
model.hc_count = 2
model.hidden_size = 4
model.num_mtp_layers = 1
model.layers = [
lambda **kwargs: (
kwargs["hidden_states"],
kwargs["hidden_states"],
torch.zeros(kwargs["hidden_states"].shape[0], 2),
),
]
model.pre_fc_norm_embedding = torch.nn.Identity()
model.pre_fc_norm_hidden = torch.nn.Identity()
model.fc_embedding = torch.nn.Identity()
model.fc_hidden = torch.nn.Identity()
def decoder(**kwargs):
"""Check that the embedding residual shares the hidden state's token slice."""
hidden = kwargs["hidden_states"]
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
model.layers = [decoder]
model.hyper_connection_mixer = SimpleNamespace(
combine_and_mix=lambda hidden_states, block_output, injection: (
hidden_states,
@@ -77,24 +109,32 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
None,
),
)
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
multi_hidden = torch.arange(24, dtype=torch.float32).reshape(3, 8)
expected_sample = multi_hidden.unflatten(-1, (2, 4)).mean(-2)
pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
with patch(
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
return_value=pp_group,
def gather(local):
"""Supply rank 1's last token and padding for a simulated two-rank gather."""
assert local.shape[0] == 2
remote = torch.cat([expected_sample[2:], multi_hidden[2:]], dim=-1)
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
with (
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
positions=torch.arange(2),
intermediate_tensors={"hidden_states": multi_hidden},
positions=torch.arange(3),
hidden_states=multi_hidden,
inputs_embeds=torch.zeros(3, 4),
)
torch.testing.assert_close(
sample_hidden,
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
)
assert returned_multi_hidden is multi_hidden
torch.testing.assert_close(sample_hidden, expected_sample)
torch.testing.assert_close(returned_multi_hidden, multi_hidden)
assert sample_hidden.is_contiguous()
assert returned_multi_hidden.is_contiguous()
@spawn_new_process_for_each_test
@@ -155,9 +195,7 @@ def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> Non
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
multimodal_config=None,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
speculative_config=None,
)
with patch.object(
+228 -47
View File
@@ -48,6 +48,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLEFileGatherEmbedding,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLENvFp4EmbeddingMethod,
Qwen4ExpPLEPinnedHostEmbedding,
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
@@ -181,9 +182,10 @@ def _mock_etp_group(
monkeypatch: pytest.MonkeyPatch,
world_size: int = 1,
all_reduce=lambda tensor: tensor,
rank: int = 0,
) -> None:
group = SimpleNamespace(
rank_in_group=0,
rank_in_group=rank,
world_size=world_size,
all_reduce=all_reduce,
)
@@ -332,6 +334,162 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
)
def _make_nvfp4_ngram_embedding(monkeypatch, *, rank=0, device="cpu", pinned=False):
_mock_etp_group(monkeypatch, world_size=2, rank=rank)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_rank", lambda: rank
)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
)
cls = Qwen4ExpPLEPinnedHostEmbedding if pinned else Qwen4ExpPLEDeviceEmbedding
with torch.device(device):
embedding = cls(
8,
160,
params_dtype=torch.bfloat16,
padding_size=2,
prefix="test.ple_embedding.ngram_embedding",
embedding_method=Qwen4ExpPLENvFp4EmbeddingMethod(),
num_ngram_heads=2,
max_total_tokens=4,
)
module = _make_ngram_embedding_for_load_test()
module.ngram_embedding = embedding
module.split_ngram_parts = 3
codes = torch.arange(8 * 80).reshape(8, 80).to(torch.uint8)
scales = ((torch.arange(80).reshape(8, 10) + 1) / 4).to(torch.float8_e4m3fn)
tensors = []
# Checkpoint shards cross ETP boundaries; scales can arrive before weights.
for shard in (2, 0, 1):
for suffix, tensor in (("weight_scale", scales), ("weight", codes)):
tensors.append(
(f"ngram_embedding.shard_{shard}.{suffix}", tensor[shard * 3 :][:3])
)
tensors.append(("ngram_embedding.weight_scale_2", torch.tensor(0.37)))
return module, tensors, codes, scales
def _reference_nvfp4_ple(codes, scales):
nibbles = torch.stack((codes & 15, codes >> 4), dim=-1).reshape(8, 160)
values = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
decoded = values[(nibbles & 7).long()] * torch.where(nibbles < 8, 1.0, -1.0)
return (decoded * scales.float().repeat_interleave(16, -1) * 0.37).bfloat16()
@pytest.mark.parametrize("rank", [0, 1])
def test_nvfp4_ple_loads_streamed_shards_across_etp_boundaries(monkeypatch, rank):
module, tensors, codes, scales = _make_nvfp4_ngram_embedding(monkeypatch, rank=rank)
loaded = set()
for start in range(0, len(tensors), 3):
loaded.update(module.load_weights(iter(tensors[start : start + 3])))
layer = module.ngram_embedding
layer.embedding_method.process_weights_after_loading(layer)
assert loaded == {
"ngram_embedding.weight",
"ngram_embedding.weight_scale",
"ngram_embedding.weight_scale_2",
}
assert layer.weight.dtype == torch.uint8
assert layer.weight_scale.dtype == torch.float8_e4m3fn
assert layer.weight.numel() + layer.weight_scale.numel() == 4 * (80 + 10)
torch.testing.assert_close(layer.weight, codes[rank * 4 :][:4])
torch.testing.assert_close(
layer.weight_scale.float(), scales[rank * 4 :][:4].float()
)
ids = torch.tensor([[3, 4], [7, 0], [4, 3]])
output = layer(ids)
expected = _reference_nvfp4_ple(codes, scales)[ids]
expected[(ids < rank * 4) | (ids >= (rank + 1) * 4)] = 0
torch.testing.assert_close(output, expected, atol=0, rtol=0)
def test_nvfp4_ple_rejects_missing_global_scale(monkeypatch):
module, tensors, _, _ = _make_nvfp4_ngram_embedding(monkeypatch)
module.load_weights(tensors[:-1])
layer = module.ngram_embedding
with pytest.raises(ValueError, match="missing its global scale"):
layer.embedding_method.process_weights_after_loading(layer)
def test_nvfp4_ple_resolves_from_mixed_precision_quantized_layers():
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
config = ModelOptMixedPrecisionConfig.from_config(
{
"quantization": {
"quant_algo": "MIXED_PRECISION",
"kv_cache_quant_algo": None,
"quantized_layers": {
prefix: {"quant_algo": "NVFP4", "group_size": 16},
"model.language_model.layers.0.mlp.experts": {
"quant_algo": "NVFP4",
"group_size": 16,
},
},
}
}
)
assert isinstance(
Qwen4ExpPLEEmbeddingMethod.from_quant_config(config, prefix),
Qwen4ExpPLENvFp4EmbeddingMethod,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")
@pytest.mark.parametrize("pinned", [False, True])
def test_nvfp4_ple_cuda_lookup_and_graph_replay(monkeypatch, pinned):
# Rank 1 exercises both the vocab offset and out-of-shard masking.
rank = 1
module, tensors, codes, scales = _make_nvfp4_ngram_embedding(
monkeypatch, rank=rank, device="cuda", pinned=pinned
)
module.load_weights(iter(tensors))
layer = module.ngram_embedding
layer.embedding_method.process_weights_after_loading(layer)
ids = torch.tensor([[3, 4], [7, 0]], device="cuda")
lookup_fn = layer._lookup if pinned else torch.compile(layer, fullgraph=True)
def lookup():
return lookup_fn(ids)
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
lookup()
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = lookup()
for new_ids in ([[3, 4], [7, 0]], [[4, 3], [0, 7]]):
ids.copy_(torch.tensor(new_ids, device="cuda"))
graph.replay()
cpu_ids = ids.cpu()
expected = _reference_nvfp4_ple(codes, scales)[cpu_ids]
expected[(cpu_ids < rank * 4) | (cpu_ids >= (rank + 1) * 4)] = 0
torch.testing.assert_close(output.cpu(), expected, atol=0, rtol=0)
if pinned:
layer.start_prefetch(ids)
torch.testing.assert_close(
layer.finalize_prefetch(total_tokens=ids.shape[0]).cpu(),
expected.flatten(-2),
atol=0,
rtol=0,
)
assert layer.weight.device.type == ("cpu" if pinned else "cuda")
assert layer.weight_scale.device == layer.weight.device
assert layer.weight_scale_2.device.type == "cuda"
assert lookup_fn(ids[:0]).shape == (0, 2, 160)
padded_ids = torch.tensor([[8, -1]], device="cuda")
torch.testing.assert_close(
lookup_fn(padded_ids),
torch.zeros(1, 2, 160, device="cuda", dtype=torch.bfloat16),
atol=0,
rtol=0,
)
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
def test_etp_lookup_gathers_and_returns_dp_local_rows(
monkeypatch: pytest.MonkeyPatch,
@@ -618,46 +776,65 @@ def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
)
def test_pinned_embedding_forward_finalizes_prefetched_output(
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("reduce_results", [False, True])
def test_pinned_embedding_finalize_prefetch_optionally_reduces_output(
monkeypatch: pytest.MonkeyPatch,
dtype: torch.dtype,
reduce_results: bool,
) -> None:
"""Deferring reduction must preserve full token rows and wait for the lookup."""
total_tokens = 3
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
values = torch.arange(1, 13).reshape(4, 3)
embedding._prefetch_buffer = torch.stack(
(values, torch.zeros_like(values)), dim=1
).to(dtype)
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
def finalize_prefetched(
prefetch_output: torch.Tensor,
output: torch.Tensor,
) -> None:
assert prefetch_output is embedding._prefetch_buffer
output.copy_(expected)
embedding.tp_size = 2
embedding.reduce_results = reduce_results
embedding.etp_data_parallel_size = 1
embedding._prefetch_stream = object()
waited_streams: list[object] = []
reduced_dtypes = []
monkeypatch.setattr(
embedding,
"_finalize_prefetch",
finalize_prefetched,
torch.cuda,
"current_stream",
lambda: SimpleNamespace(wait_stream=waited_streams.append),
)
output = embedding(hidden_states)
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
assert waited_streams == [embedding._prefetch_stream]
reduced_dtypes.append(tensor.dtype)
# The remote rank owns the other head's vocabulary rows.
return tensor + tensor.flip(1)
assert output.dtype == embedding._prefetch_buffer.dtype
assert torch.equal(output, expected)
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
output = embedding.finalize_prefetch(total_tokens=total_tokens)
expected = (
values.unsqueeze(1).expand(-1, 2, -1).to(dtype)
if reduce_results
else embedding._prefetch_buffer
)
expected = expected[:total_tokens].flatten(-2)
comm_dtype = torch.int8 if dtype == torch.float8_e4m3fn else dtype
assert reduced_dtypes == ([comm_dtype] if reduce_results else [])
assert waited_streams == [embedding._prefetch_stream]
assert output.dtype == dtype
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
"""Forward before any start_prefetch must fail loudly, not read garbage."""
def test_pinned_embedding_finalize_prefetch_requires_prior_start_prefetch() -> None:
"""Finalizing before start_prefetch must fail instead of reading garbage."""
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = None
embedding._prefetch_stream = None
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="prior start_prefetch"):
embedding(hidden_states)
embedding.finalize_prefetch(total_tokens=2)
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
@@ -788,15 +965,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
def test_pinned_embedding_start_prefetch_allocates_lazily(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
Exercises the full pinned-prefetch contract end to end: first call
allocates the side stream and buffer with the constructor geometry,
and forward finalizes the lookup into the returned output.
"""
"""Allocate prefetch storage on first use and finalize the lookup results."""
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
@@ -832,8 +1004,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
embedding.start_prefetch(hidden_states, ngram_ids)
embedding.start_prefetch(ngram_ids)
buffer = embedding._prefetch_buffer
assert buffer is not None
@@ -842,7 +1013,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
assert buffer.device.type == "cuda"
assert embedding._prefetch_stream is not None
output = embedding(hidden_states)
output = embedding.finalize_prefetch(total_tokens=ngram_ids.shape[0])
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
@@ -859,7 +1030,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
"quantized_layers": {
prefix: {"quant_algo": "FP8"},
"model.language_model.layers.2.moe.gate_proj": {
"quant_algo": "NVFP4"
"quant_algo": "MXFP8"
},
},
}
@@ -870,13 +1041,12 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
assert isinstance(
# A quant_algo the PLE table cannot load must not fall back to unquantized.
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
"model.language_model.layers.2.moe.gate_proj",
),
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
)
def _patch_ple_construction(
@@ -919,6 +1089,7 @@ def _make_file_gather_embedding(
)
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
module.use_sequence_parallel = False
module.split_ngram_parts = 4
module.eos_token_id = 0
module.ngram_size = 2
@@ -1215,7 +1386,7 @@ def test_file_gather_forward_reads_staging_without_host_work(
step = (torch.arange(6), torch.tensor([0, 6]), torch.tensor([[1]]))
hidden_states = torch.zeros(6, 8, dtype=torch.bfloat16)
assert module.start_prefetch(hidden_states, *step) is None
assert module.start_prefetch(*step) is None
output = module(hidden_states, *step)
assert torch.all(staging.view(torch.uint8) == 0xFF)
@@ -1500,7 +1671,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
prefetched: list[torch.Tensor] = []
module.ngram_embedding = SimpleNamespace(
supports_prefetch=True,
start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)),
start_prefetch=lambda ids: prefetched.append(weak_ref_tensor(ids)),
)
expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params)
@@ -1508,7 +1679,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
# like a later CUDA graph segment would.
pool = torch.cuda.MemPool()
with torch.cuda.use_mem_pool(pool):
module.start_prefetch(None, input_ids, query_start_loc, ngram_context)
module.start_prefetch(input_ids, query_start_loc, ngram_context)
torch.full_like(expected, -1)
assert torch.equal(prefetched[0], expected)
@@ -2210,12 +2381,18 @@ def _make_conv_metadata(
],
)
@pytest.mark.parametrize(
"layer_cls", [Qwen4ExpPLELayer, Qwen4ExpPLELayerAMD], ids=["nvidia", "amd"]
("layer_cls", "fuse_residual"),
[
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
],
)
def test_fused_conv_correctness(
case: _ConvBatchCase,
state_layout: str,
layer_cls: type[nn.Module],
fuse_residual: bool,
) -> None:
device = torch.device("cuda")
metadata, num_real_tokens = _make_conv_metadata(case, device)
@@ -2257,12 +2434,15 @@ def test_fused_conv_correctness(
)
null_state = conv_state[NULL_BLOCK_ID].clone()
residual_kernel = residual.clone()
residual_reference = residual.clone()
# The unfused kernel must overwrite the buffer without reading its contents.
residual_reference = (
residual.clone() if fuse_residual else torch.zeros_like(residual)
)
module._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual_kernel,
outer_residual=outer_residual,
outer_residual=outer_residual if fuse_residual else None,
metadata=metadata,
conv_state=conv_state,
conv_weights=weights,
@@ -2276,7 +2456,8 @@ def test_fused_conv_correctness(
conv_state_len=module.conv_state_len,
dilation=module.short_conv_dilation,
)
residual_reference = outer_residual + residual_reference
if fuse_residual:
residual_reference = outer_residual + residual_reference
assert torch.equal(
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
@@ -2286,7 +2467,7 @@ def test_fused_conv_correctness(
if case.graph_padding:
assert torch.equal(
residual_kernel[num_real_tokens:],
(outer_residual + residual)[num_real_tokens:],
residual_reference[num_real_tokens:],
)
+267
View File
@@ -0,0 +1,267 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Multi-GPU equivalence tests for Qwen4Exp HC and MoE sequence parallelism."""
from dataclasses import MISSING, fields
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.multiprocessing as mp
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
from tests.utils import multi_gpu_test
from vllm.config import (
EngramConfig,
ParallelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
tensor_model_parallel_all_reduce,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
from vllm.utils.network_utils import get_open_port
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
from .test_ple import _ConvBatchCase, _make_conv_metadata
class _TPAttentionProjection(nn.Module):
"""Keep real TP communication without setting up an attention KV cache."""
def __init__(self, config, *, reduce_results: bool, **kwargs) -> None:
super().__init__()
self.proj = RowParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
input_is_parallel=False,
reduce_results=reduce_results,
)
def forward(self, hidden_states: torch.Tensor):
return self.proj(hidden_states)[0]
def _make_config(rank: int, *, hc_sp: bool = False, moe_sp: bool = False) -> VllmConfig:
"""Configure HC SP explicitly or MoE SP through TP=2, DP=2 and EP."""
config = VllmConfig(
parallel_config=ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=2 if moe_sp else 1,
data_parallel_rank=rank // 2,
enable_expert_parallel=moe_sp,
enable_hc_sp=hc_sp,
all2all_backend="allgather_reducescatter",
),
)
config.model_config = SimpleNamespace(
architecture="Qwen4ExpForCausalLM",
is_moe=True,
sleep_mode_offload_cudagraph=False,
hf_text_config=Qwen4ExpTextConfig(
vocab_size=128,
eos_token_id=0,
hidden_size=128,
hc_count=2,
hc_lowrank=16,
num_hidden_layers=1,
layer_types=["linear_attention"],
num_experts=4,
num_experts_per_tok=2,
moe_intermediate_size=256,
shared_expert_intermediate_size=256,
ple_layer_ids=[] if moe_sp else [1],
ple_embed_dim=128,
heads_per_ngram=2,
ngram_vocab_size_base=32,
make_ngram_vocab_size_divisible_by=64,
),
dtype=torch.bfloat16,
)
config.engram_config = EngramConfig(cpu_offload=False)
return config
def _make_decoder(config: VllmConfig, rank: int) -> Qwen4ExpDecoderLayer:
"""Use the production constructor and identical initial weights for both modes."""
with (
set_current_vllm_config(config),
patch.object(qwen4_model, "QwenGatedDeltaNetAttention", _TPAttentionProjection),
):
layer = Qwen4ExpDecoderLayer(
config, config.model_config.hf_text_config.layer_types[0], "model.layers.0"
)
torch.manual_seed(12)
for param in layer.parameters():
param.normal_(std=0.08)
# Distinct rank contributions expose missing or duplicate reductions.
layer.linear_attn.proj.weight.add_(rank * 0.01)
experts = layer.mlp.experts.routed_experts
if config.parallel_config.enable_expert_parallel:
experts.w13_weight.add_(rank * 0.01)
experts.quant_method.process_weights_after_loading(experts)
return layer
def _check_decoder_ple(rank: int) -> None:
"""Compare HC SP with full tokens across a PLE prefill and cached decode."""
configs = [_make_config(rank, hc_sp=enabled) for enabled in (False, True)]
layers = [_make_decoder(config, rank) for config in configs]
states = []
for layer in layers:
ple = layer.ple
assert ple is not None
state = torch.zeros(2, ple.hc_hidden_size, ple.conv_state_len)
if not is_conv_state_dim_first():
state = state.transpose(-1, -2).contiguous()
ple.kv_cache = (state,)
states.append(state)
# Three prefill tokens straddle the TP=2 boundary and require SP padding.
for case in (
_ConvBatchCase(prefill_query_lens=(3,)),
_ConvBatchCase(num_decodes=1),
):
layout, num_tokens = _make_conv_metadata(case, torch.device("cuda", rank))
metadata_args = {
f.name: None
for f in fields(PleShortConvAttentionMetadata)
if f.default is MISSING and f.default_factory is MISSING
}
metadata = PleShortConvAttentionMetadata(**(metadata_args | vars(layout)))
ids = torch.randint(0, 128, (num_tokens,))
positions = torch.arange(num_tokens).expand(3, -1)
qsl = torch.tensor([0, num_tokens], dtype=torch.int32)
context = torch.zeros(1, layers[0].config.ngram_size - 1, dtype=torch.int64)
hidden = torch.randn(num_tokens, 256)
block = torch.randn(num_tokens, 128)
injection = torch.randn(num_tokens, 2)
outputs = []
for config, layer in zip(configs, layers):
assert layer.ple is not None
inputs = (hidden, block, injection)
if config.parallel_config.enable_hc_sp:
inputs = tuple(sp_shard(x) for x in inputs)
with (
set_current_vllm_config(config),
set_forward_context({layer.ple.prefix: metadata}, config),
):
layer.ple.start_prefetch(ids, qsl, context)
output = layer(
*inputs,
positions,
input_ids=ids,
query_start_loc=qsl,
ngram_context=context,
)
if config.parallel_config.enable_hc_sp:
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
# A later MoE call may reuse workspace-backed output storage.
outputs.append(tuple(x.clone() for x in output))
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
torch.testing.assert_close(states[1], states[0], atol=0.002, rtol=0.02)
def _check_moe_sp(rank: int, config: VllmConfig) -> None:
"""Compare native SP with full-token GR and the MoE wrapper's public interface."""
layer = _make_decoder(config, rank)
dp_rank = config.parallel_config.data_parallel_rank
counts = torch.tensor([3, 5], device="cpu")
num_tokens = int(counts[dp_rank])
torch.manual_seed(30 + dp_rank)
hidden = torch.randn(num_tokens, 256)
positions = torch.arange(num_tokens)
# Include runner padding in addition to the padding introduced by SP.
padding = sp_padding_mask(positions == num_tokens - 1, hidden)
with (
set_current_vllm_config(config),
set_forward_context(
None,
config,
num_tokens=num_tokens,
num_tokens_across_dp=counts,
is_padding=padding,
),
):
# The reference keeps full GR rows. MoE handles its own chunk and gather.
full_hidden, attn_input, injection = layer.attn_hyper_connection.mix(hidden)
attn_output = tensor_model_parallel_all_reduce(layer.linear_attn(attn_input))
full_hidden, moe_input, injection = layer.mlp_hyper_connection.combine_and_mix(
full_hidden, attn_output, injection
)
expected = tuple(
x.clone() for x in (full_hidden, layer.mlp(moe_input), injection)
)
output = layer(
sp_shard(hidden),
None,
None,
positions,
input_ids=None,
query_start_loc=None,
ngram_context=None,
)
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
"""Initialize rank-local runtime state and run one equivalence check."""
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
config = _make_config(rank, moe_sp=moe_sp)
init_distributed_environment(
world_size=4 if moe_sp else 2,
rank=rank,
local_rank=rank,
distributed_init_method=f"tcp://127.0.0.1:{port}",
)
try:
# Spawned ranks bypass the GPU worker's workspace initialization.
init_workspace_manager(torch.device(f"cuda:{rank}"))
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=2)
with torch.device(f"cuda:{rank}"), torch.inference_mode():
torch.set_default_dtype(torch.bfloat16)
if moe_sp:
_check_moe_sp(rank, config)
else:
_check_decoder_ple(rank)
finally:
reset_workspace_manager()
cleanup_dist_env_and_memory()
@multi_gpu_test(num_gpus=2)
def test_hc_sp_decoder_and_ple() -> None:
"""HC SP preserves decoder output and PLE cache across a token-shard boundary."""
mp.spawn(_run_sp, args=(get_open_port(), False), nprocs=2)
@multi_gpu_test(num_gpus=4)
def test_moe_sp_equivalence(monkeypatch) -> None:
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
+32 -11
View File
@@ -197,6 +197,10 @@ class _HfExamplesInfo:
_TEXT_GENERATION_EXAMPLE_MODELS = {
# [Decoder-only]
"AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
"AliceAIForCausalLM": _HfExamplesInfo(
"yandex/AliceAI-Foundation-80B-A3B-Base",
trust_remote_code=True,
),
"ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
"ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
"AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
@@ -266,6 +270,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
),
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
"ExaoneMoEForCausalLM": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B",
hf_overrides={"architectures": ["ExaoneMoEForCausalLM"]},
),
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
@@ -356,6 +364,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"KimiLinearForCausalLM": _HfExamplesInfo(
"moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True
),
"Kolibri1ForCausalLM": _HfExamplesInfo(
"Aleph-Alpha/Kolibri-1", min_transformers_version="5.20.0"
),
"LagunaForCausalLM": _HfExamplesInfo("poolside/Laguna-XS.2"),
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
"Lfm2MoeForCausalLM": _HfExamplesInfo(
@@ -718,6 +729,11 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
"BertForSequenceClassification": _HfExamplesInfo(
"cross-encoder/ms-marco-MiniLM-L-6-v2"
),
"D3ForSequenceClassification": _HfExamplesInfo(
"vllm-sr/d3-edge",
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
max_model_len=4096,
),
"GPT2ForSequenceClassification": _HfExamplesInfo(
"nie3e/sentiment-polish-gpt2-small"
),
@@ -798,6 +814,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Salesforce/blip2-opt-2.7b",
extras={"6b": "Salesforce/blip2-opt-6.7b"},
),
"ClefFlashForJointSchema": _HfExamplesInfo(
"Cloudflare/clef-flash",
max_model_len=4096,
),
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025",
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
@@ -1114,6 +1134,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
),
"MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_7",
min_transformers_version="5.18.0",
is_available_online=False,
),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
trust_remote_code=True,
@@ -1342,6 +1367,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
),
"ClefOmniForJointSchema": _HfExamplesInfo(
"Cloudflare/clef-omni",
max_model_len=4096,
),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
@@ -1417,7 +1446,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"CohereLabs/cohere-transcribe-03-2026", trust_remote_code=True
),
"NemotronParseForConditionalGeneration": _HfExamplesInfo(
"nvidia/NVIDIA-Nemotron-Parse-v1.2", trust_remote_code=True
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
extras={"2.0": "nvidia/NVIDIA-Nemotron-Parse-2.0"},
trust_remote_code=True,
),
"WhisperForConditionalGeneration": _HfExamplesInfo(
"openai/whisper-large-v3-turbo",
@@ -1429,16 +1460,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
# [Medusa]
"MedusaModel": _HfExamplesInfo(
"JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
),
# Temporarily disabled.
# TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
# "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
# "JackFram/llama-160m",
# speculative_model="ibm-ai-platform/llama-160m-accelerator"
# ),
# [DFlash]
"DFlashDraftModel": _HfExamplesInfo(
"Qwen/Qwen3-4B",
+231
View File
@@ -0,0 +1,231 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.model_executor.models import alice_ai as alice_ai_module
from vllm.model_executor.models.alice_ai import (
AliceAIDecoderBlock,
AliceAIForCausalLM,
AliceAIModel,
_AttentionResidualFinal,
_load_split_kda_conv_weights,
)
from vllm.model_executor.offloader import PrefetchOffloader, UVAOffloader
@pytest.mark.parametrize(
"backend,error",
[
("uva", "requires UVA"),
("prefetch", "does not support prefetch"),
],
)
def test_unsupported_offloading_is_rejected(monkeypatch, backend, error):
from vllm.model_executor.offloader import uva
monkeypatch.setattr(uva, "is_uva_available", lambda: False)
monkeypatch.setenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0")
if backend == "uva":
offloader = UVAOffloader(cpu_offload_max_bytes=1024)
else:
offloader = PrefetchOffloader.__new__(PrefetchOffloader)
monkeypatch.setattr(alice_ai_module, "get_offloader", lambda: offloader)
config = SimpleNamespace(lora_config=None)
with pytest.raises(NotImplementedError, match=error):
AliceAIForCausalLM(vllm_config=config, prefix="target")
@pytest.mark.parametrize(
"source,target,shard_id",
[
("linear_attn.q_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 0),
("linear_attn.k_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 1),
("linear_attn.v_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 2),
("linear_attn.b_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 3),
("linear_attn.f_a_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 4),
("self_attn.q_proj.weight", "self_attn.qkv_proj.weight", "q"),
("linear_attn.a_log_bias", "linear_attn.A_log", None),
],
)
def test_weight_mapping_preserves_shard_ids(source, target, shard_id):
tensor = torch.empty(1)
[(mapped, mapped_tensor)] = list(
AliceAIModel.hf_to_vllm_mapper.apply([(f"model.layers.0.{source}", tensor)])
)
assert mapped == f"model.layers.0.{target}"
assert getattr(mapped_tensor, "shard_id", None) == shard_id
def test_split_kda_conv_weights_use_fused_loader_shard_ids() -> None:
calls: list[tuple[torch.Tensor, int]] = []
param = torch.nn.Parameter(torch.empty(3, 1, 1))
def weight_loader(
target: torch.Tensor,
weight: torch.Tensor,
shard_id: int,
) -> None:
assert target is param
calls.append((weight, shard_id))
param.weight_loader = weight_loader
target_name = "model.layers.0.linear_attn.conv1d.weight"
weights = [
("model.layers.0.linear_attn.q_conv1d.weight", torch.full((1, 1), 1.0)),
("model.layers.0.linear_attn.k_conv1d.weight", torch.full((1, 1), 2.0)),
("model.layers.0.linear_attn.v_conv1d.weight", torch.full((1, 1), 3.0)),
("model.embed_tokens.weight", torch.full((1, 1), 4.0)),
]
loaded: set[str] = set()
remaining = list(
_load_split_kda_conv_weights(weights, {target_name: param}, loaded)
)
assert loaded == {target_name}
assert [shard_id for _, shard_id in calls] == [0, 1, 2]
assert [weight.item() for weight, _ in calls] == [1.0, 2.0, 3.0]
assert remaining == [weights[-1]]
def test_kda_cache_contract_is_fp32_recurrent() -> None:
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
dtype=torch.bfloat16,
hf_text_config=SimpleNamespace(
linear_num_key_heads=32,
linear_key_head_dim=128,
linear_conv_kernel_dim=4,
),
),
cache_config=SimpleNamespace(mamba_cache_dtype="auto"),
parallel_config=SimpleNamespace(tensor_parallel_size=1),
speculative_config=None,
)
assert AliceAIForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == (
torch.bfloat16,
torch.float32,
)
conv_shape = (12288, 3) if is_conv_state_dim_first() else (3, 12288)
assert AliceAIForCausalLM.get_mamba_state_shape_from_config(vllm_config) == (
conv_shape,
(32, 128, 128),
)
class _FixedOutput(torch.nn.Module):
def __init__(self, value: float) -> None:
super().__init__()
self.value = value
self.inputs: list[torch.Tensor] = []
def forward(
self, hidden_states: torch.Tensor, positions: torch.Tensor | None = None
) -> torch.Tensor:
del positions
self.inputs.append(hidden_states.detach().clone())
return torch.full_like(hidden_states, self.value)
def _make_state_layer(
layer_idx: int,
block_size: int,
attention_value: float,
mlp_value: float,
) -> AliceAIDecoderBlock:
layer = AliceAIDecoderBlock.__new__(AliceAIDecoderBlock)
torch.nn.Module.__init__(layer)
layer.layer_idx = layer_idx
layer.layer_type = "linear_attention"
layer.layer_scale = False
layer.attn_res_block_size = block_size
layer.attn_res_rms_norm_eps = 1e-6
layer.input_layernorm = torch.nn.Identity()
layer.post_attention_layernorm = torch.nn.Identity()
layer.linear_attn = _FixedOutput(attention_value)
layer.mlp = _FixedOutput(mlp_value)
if layer_idx > 0:
layer.attn_res_proj = torch.nn.Linear(1, 1, bias=False)
layer.attn_res_norm_weight = torch.nn.Parameter(torch.ones(1))
layer.mlp_res_proj = torch.nn.Linear(1, 1, bias=False)
layer.mlp_res_norm_weight = torch.nn.Parameter(torch.ones(1))
return layer
def _make_final_attn_res() -> _AttentionResidualFinal:
final = _AttentionResidualFinal.__new__(_AttentionResidualFinal)
torch.nn.Module.__init__(final)
final.rms_norm_eps = 1e-6
final.res_proj = torch.nn.Linear(1, 1, bias=False)
final.res_norm_weight = torch.nn.Parameter(torch.ones(1))
return final
def test_attn_res_block_grouping_and_final_norm(
monkeypatch,
) -> None:
calls: list[list[torch.Tensor]] = []
def reference_attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
block_write_idx,
eps,
output_norm_eps,
):
del delta, norm_weight, qk_weight, output_norm_weight
del block_write_idx, eps, output_norm_eps
sources = [blocks[:, index] for index in range(num_blocks)] + [prefix]
calls.append([source.detach().clone() for source in sources])
return torch.stack(sources).sum(dim=0)
monkeypatch.setattr(alice_ai_module, "attn_res", reference_attn_res)
monkeypatch.setattr(
alice_ai_module,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
raw_outputs = [(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)]
model = AliceAIModel.__new__(AliceAIModel)
torch.nn.Module.__init__(model)
model.attn_res_block_size = 4
model.start_layer = 0
model.end_layer = len(raw_outputs)
model.layers = torch.nn.ModuleList(
[
_make_state_layer(index, 4, attention, mlp)
for index, (attention, mlp) in enumerate(raw_outputs)
]
)
model.config = SimpleNamespace(num_hidden_layers=len(raw_outputs))
model.attnres_final = _make_final_attn_res()
model.norm = torch.nn.Linear(1, 1, bias=False)
with torch.no_grad():
model.norm.weight.fill_(2)
output = AliceAIModel.forward(
model,
input_ids=None,
positions=torch.arange(2),
inputs_embeds=torch.full((2, 1), 10.0),
)
torch.testing.assert_close(output, torch.full((2, 1), 130.0))
torch.testing.assert_close(
model.layers[0].linear_attn.inputs[0], torch.full((2, 1), 10.0)
)
assert [len(sources) for sources in calls] == [2] * 8 + [3, 3]
assert [source[0, 0].item() for source in calls[-1]] == [10, 36, 19]
+93
View File
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only tests for AliceAI configuration."""
from types import SimpleNamespace
import pytest
from vllm.model_executor.models.config import AliceAIForCausalLMConfig
def _make_config(**overrides) -> SimpleNamespace:
return SimpleNamespace(
**{
"kda_allow_negative_eigenvalues": False,
"router_score_function": "sigmoid",
"router_bias_correction": True,
**overrides,
}
)
def test_model_config_warns_when_mtp_is_ignored(caplog) -> None:
config = _make_config(mtp_num_hidden_layers=1)
with caplog.at_level("WARNING"):
AliceAIForCausalLMConfig.verify_and_update_model_config(
SimpleNamespace(hf_config=config)
)
assert "ignores the MTP configuration and weights" in caplog.text
@pytest.mark.parametrize(
"overrides,error_type,error",
[
({}, None, None),
(
{"kda_allow_negative_eigenvalues": True},
NotImplementedError,
"kda_allow_negative_eigenvalues=false",
),
(
{"router_score_function": "softmax"},
ValueError,
"router_score_function='sigmoid'",
),
(
{"router_bias_correction": False},
ValueError,
"router_bias_correction=true",
),
],
)
def test_model_config_rejects_unsupported_alice_contract(
overrides, error_type, error
) -> None:
model_config = SimpleNamespace(hf_config=_make_config(**overrides))
if error is None:
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
else:
with pytest.raises(error_type, match=error):
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
@pytest.mark.parametrize(
"pipeline_parallel_size,speculative_method,error",
[
(1, None, None),
(2, None, "pipeline_parallel_size=1"),
(1, "eagle3", "eagle3"),
(1, "dflash", "dflash"),
(1, "dspark", "dspark"),
(1, "extract_hidden_states", "extract_hidden_states"),
],
)
def test_runtime_config_rejects_unsupported_alice_modes(
pipeline_parallel_size, speculative_method, error
) -> None:
vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(pipeline_parallel_size=pipeline_parallel_size),
speculative_config=(
None
if speculative_method is None
else SimpleNamespace(method=speculative_method)
),
)
if error is None:
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
else:
with pytest.raises(NotImplementedError, match=error):
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
@@ -0,0 +1,329 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""What the AMD replay forward does with a prefill longer than the window.
The cut in this fixture is layer 1, so replay starts at layer 1. That is the
same boundary as production layer 20, on a 4-layer stand-in. The final
``mhc_post`` and the aux capture at that cut both run on the window rows.
"""
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41.decoder_replay_layers import ReplayBatch
from vllm.sequence import IntermediateTensors
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
WINDOW = 128
FULL = 300
# Trailing window of a 300-token prefill.
ROWS = list(range(FULL - WINDOW, FULL))
HC, HIDDEN = 4, 8
DEVICE = torch.device("cuda")
# gfx942 writes the V4 record (584 B). Family-100 writes the V4.1 record (528 B).
# The SWA layer uses block size 32, which is the size the KV-only insert checks.
_BLOCK = 32
_HEAD_DIM = 512
class _RocmKvInsert:
"""SWA cache from ``forward_kv``, checked against the KV-only insert."""
def __init__(self, num_tokens: int):
from vllm.models.deepseek_v41.attention import _use_v41_mxfp8_kv_record
if not hasattr(torch.ops._C, "fused_deepseek_v4_kv_rope_insert"):
pytest.skip("fused DeepseekV4 KV insert op is not built")
self.num_tokens = num_tokens
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
bytes_per = 528 if self.kv_mxfp8 else 584
num_blocks = (num_tokens + _BLOCK - 1) // _BLOCK + 1
cache = torch.full(
(num_blocks, _BLOCK, bytes_per),
0xA5,
dtype=torch.uint8,
device=DEVICE,
)
self.poison = cache.clone()
self.kv = torch.randn(
num_tokens, _HEAD_DIM, dtype=torch.bfloat16, device=DEVICE
)
prefix = "replay.swa"
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=DEVICE)
self.metadata = {
prefix: SimpleNamespace(slot_mapping=slot_mapping, block_size=_BLOCK)
}
self.context = ForwardContext({}, self.metadata, {})
inv_freq = 1.0 / (
10000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device=DEVICE) / 64)
)
freqs = torch.outer(
torch.arange(num_tokens + 8, dtype=torch.float32, device=DEVICE), inv_freq
)
self.attn = SimpleNamespace(
kv_mxfp8=self.kv_mxfp8,
compressor=None,
indexer=None,
rotary_emb=SimpleNamespace(
cos_sin_cache=torch.cat((freqs.cos(), freqs.sin()), dim=-1)
),
swa_cache_layer=SimpleNamespace(
prefix=prefix, kv_cache=cache, block_size=_BLOCK
),
_run_parallel_input_projections=lambda hidden: (hidden, None, None),
_split_qkv_and_norm=lambda qr_kv: (None, None, self.kv[: qr_kv.shape[0]]),
)
self.last_rows: int | None = None
self.last_positions: torch.Tensor | None = None
def write(
self,
x,
positions,
input_ids,
pre_mix,
post_mix,
res_mix,
residual,
):
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV41ROCMAiterMLAAttention,
)
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
assert (
DeepseekV41ROCMAiterMLAAttention.forward_kv
is DeepseekV4Attention.forward_kv
)
self.last_rows = x.shape[0]
self.last_positions = positions
DeepseekV41ROCMAiterMLAAttention.forward_kv(self.attn, positions, x)
def assert_matches_forward(self):
assert self.last_rows == self.num_tokens
assert self.last_positions is not None
written = self.attn.swa_cache_layer.kv_cache
assert not torch.equal(written, self.poison)
ref = self.poison.clone()
meta = self.metadata[self.attn.swa_cache_layer.prefix]
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
self.kv,
ref,
meta.slot_mapping,
self.last_positions,
self.attn.rotary_emb.cos_sin_cache,
_BLOCK,
None,
self.kv_mxfp8,
)
assert torch.equal(written, ref)
assert torch.equal(written[-1], self.poison[-1])
class _Layer(nn.Module):
def __init__(
self,
idx: int,
compressed: str | None = None,
indexer: str | None = None,
):
super().__init__()
self.idx = idx
self.seen: list[torch.Tensor] = []
self.attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix=f"layers.{idx}.swa"),
compressed_cache_prefix=compressed,
indexer=(
None
if indexer is None
else SimpleNamespace(k_cache=SimpleNamespace(prefix=indexer))
),
compress_ratio=1,
)
def forward(
self,
hidden_states,
positions,
input_ids,
pre_mix,
post_mix,
res_mix,
residual,
engram_hashes,
engram_mask,
):
self.seen.append(hidden_states[:, 0, 0].detach().clone())
if residual is None:
residual = hidden_states
if pre_mix is None:
pre_mix = hidden_states[:, :, 0]
if post_mix is None:
post_mix = pre_mix
if res_mix is None:
res_mix = pre_mix
return hidden_states, residual, post_mix, res_mix, pre_mix
def _model(monkeypatch):
import vllm.models.deepseek_v41.amd.model as amd
monkeypatch.setattr(
amd,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=False, is_last_rank=False),
)
monkeypatch.setattr(amd.EngramLayout, "from_config", lambda config: None)
monkeypatch.setattr(amd, "MHCPostOp", lambda: SimpleNamespace())
# Layer 0 sits before the cut, so its keys are not collected. Layer 2
# shares layer 1's compressed cache. Layer 3 has neither.
layers = [
_Layer(0, "layers.0.compressed", "layers.0.indexer"),
_Layer(1, "layers.1.compressed", "layers.1.indexer"),
_Layer(2, "layers.1.compressed", "layers.2.indexer"),
_Layer(3),
]
# Layer 1 is the KV source, so the replay callback is this layer's write.
insert = _RocmKvInsert(FULL)
layers[1].write_kv = insert.write
monkeypatch.setattr(amd, "make_layers", lambda n, factory, prefix: (0, n, layers))
config = SimpleNamespace(
vocab_size=32,
hc_eps=1e-6,
hc_mult=HC,
hidden_size=HIDDEN,
rms_norm_eps=1e-6,
index_topk=4,
kv_source_layer_ids=[1],
num_hidden_layers=4,
sliding_window=WINDOW,
engram_layer_ids=(),
candidate_source_layer_id=-1,
candidate_topk_blocks=0,
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(hf_config=config, dtype=torch.bfloat16),
quant_config=None,
cache_config=SimpleNamespace(swa_bounded_replay=True),
use_v2_model_runner=True,
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
use_ubatching=False,
enable_expert_parallel=False,
),
speculative_config=None,
scheduler_config=SimpleNamespace(max_num_batched_tokens=FULL),
)
model = amd.DeepseekV4Model(vllm_config=vllm_config)
# Production requests aux at the first replay layer. Here that id is the cut.
model.aux_hidden_state_layers = (model.decoder_replay_start,)
model.topk_indices_buffer = model.topk_indices_buffer.to(DEVICE)
calls: list[torch.Tensor] = []
def mhc_post(hidden, residual, post_mix, res_mix):
calls.append(hidden[:, 0, 0].detach().clone())
return hidden
model.mhc_post = mhc_post
saved_aux: list[list[torch.Tensor]] = []
run_layers = model._run_layers
def wrapped(*args, **kwargs):
result = run_layers(*args, **kwargs)
# Snapshot: forward() extends the early list with the scattered aux.
saved_aux.append(list(result[5]))
return result
model._run_layers = wrapped
scattered: list[tuple[torch.Tensor, ...]] = []
replay = model.decoder_replay_layers
run = replay._run
def wrapped_run(*args, **kwargs):
result = run(*args, **kwargs)
scattered.append(result)
return result
replay._run = wrapped_run
return model, layers, calls, saved_aux, scattered, insert
def _forward(model, insert: _RocmKvInsert):
token_ids = torch.arange(FULL, device=DEVICE, dtype=torch.float32)
hidden = token_ids[:, None, None].expand(FULL, HC, HIDDEN).contiguous()
pre_mix = torch.ones(FULL, HC, device=DEVICE)
rows = torch.tensor(ROWS, device=DEVICE)
model.decoder_replay_layers.replay_batch = ReplayBatch(
rows,
ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}),
)
positions = torch.arange(FULL, device=DEVICE)
with override_forward_context(insert.context):
out = model(
input_ids=positions.to(torch.int32),
positions=positions,
intermediate_tensors=IntermediateTensors(
{"hidden_states": hidden, "pre_mix": pre_mix}
),
)
return out
def test_amd_replay_forward_trims_to_the_window(monkeypatch):
model, layers, calls, saved_aux, scattered, insert = _model(monkeypatch)
replay_layers = model.decoder_replay_layers
assert model.decoder_replay_start == 1
assert replay_layers.first_swa_prefix == "layers.1.swa"
assert replay_layers.metadata_prefixes == {
"layers.1.swa",
"layers.1.compressed",
"layers.1.indexer",
"layers.2.swa",
"layers.2.indexer",
"layers.3.swa",
}
assert replay_layers.replay_batch is None
out = _forward(model, insert)
insert.assert_matches_forward()
replay = model.decoder_replay_layers.replay_batch
assert replay is not None and replay.rows.tolist() == ROWS
early = [layer.seen[0] for layer in layers[: model.decoder_replay_start]]
late = [layer.seen[0] for layer in layers[model.decoder_replay_start :]]
full_ids = torch.arange(FULL, device=DEVICE, dtype=torch.float32)
window_ids = full_ids[ROWS]
assert [t.tolist() for t in early] == [full_ids.tolist()]
assert [t.tolist() for t in late] == [
window_ids.tolist(),
window_ids.tolist(),
window_ids.tolist(),
]
# Scatter writes the replay rows back and leaves the trimmed rows at zero.
hidden = out["hidden_states"][:, 0, 0]
assert torch.equal(hidden[ROWS], window_ids)
assert hidden[: FULL - WINDOW].eq(0).all()
# calls[0] is the cut-aux capture. calls[1] is the final mhc_post.
# Both run inside the replay batch, before the scatter back to FULL.
assert len(calls) == 2
assert [t.shape[0] for t in calls] == [WINDOW, WINDOW]
assert torch.equal(calls[0], window_ids)
assert torch.equal(calls[1], window_ids)
# The early loop skips this id. The replay loop does not see it either.
assert saved_aux[0] == []
assert saved_aux[1] == []
_hidden, _pre_mix, aux = scattered[0]
assert aux.shape[0] == FULL
assert torch.equal(aux[ROWS, 0], window_ids)
assert aux[: FULL - WINDOW].eq(0).all()
@@ -8,6 +8,7 @@ import pytest
import torch
from vllm.config import CompilationConfig, CUDAGraphMode
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.forward_context import (
BatchDescriptor,
ForwardContext,
@@ -144,8 +145,18 @@ def test_piecewise_graphs_break_out_by_size():
write_batch_kv.assert_called_once_with(*states)
def test_replay_graph_matches_eager(monkeypatch):
"""The replay graph of the next captured size pads and runs the rows."""
@pytest.mark.parametrize(
("moe_backend", "ids_dtype"),
[
("auto", torch.int32),
*((b, torch.int64) for b in sorted(NATIVE_MEGA_MOE_BACKENDS)),
*((b, torch.int32) for b in sorted(FLASHINFER_MOE_EP_BACKENDS)),
],
ids=str,
)
def test_replay_graph_matches_eager(monkeypatch, moe_backend, ids_dtype):
"""The replay graph of the next captured size pads and runs the rows. Input ids
arrive as the model passes them: int32, cast to int64 for native MegaMoE only."""
monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock)
capture = torch.cuda.stream(torch.cuda.Stream())
monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture)
@@ -154,9 +165,10 @@ def test_replay_graph_matches_eager(monkeypatch):
# Mocking it true would capture into the cuMem pool, which breaks on ROCm.
cfg.use_cumem_cudagraph_pool = False
cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64)
cfg.cache_config.use_kda_recoverssm = False
cfg.cache_config.use_recoverssm = False
cfg.model_config.hf_config = MagicMock(hc_mult=HC, hidden_size=HIDDEN)
cfg.model_config.dtype = torch.float32
cfg.kernel_config.moe_backend = moe_backend
def run_layers(hidden, positions, ids, pre, post, mix, residual):
return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None]
@@ -173,6 +185,7 @@ def test_replay_graph_matches_eager(monkeypatch):
torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype)
for b in manager.input_buffers
]
states[2] = torch.arange(12, device=DEVICE, dtype=ids_dtype) # input_ids
rows = torch.tensor([11, 2, 7], device=DEVICE)
desc = manager.dispatch(2, 3, None, 0)
with override_forward_context(_graph_context(desc.num_tokens)):
@@ -290,6 +290,81 @@ def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129]
def test_zero_prefix_window_warms_gather_and_skips_pad(monkeypatch):
"""ROCm leaves the prefix-cache window at 0. Replay still needs replay_start
and a warmed gather kernel. The pad kernel stays on the prefix-cache path."""
import vllm.models.deepseek_v41.nvidia.model_state as model_state
calls = {"pad_warmup": 0, "gather_warmup": 0, "pad_launch": 0}
def pad_warmup(*args, **kwargs):
calls["pad_warmup"] += 1
def gather_warmup(*args, **kwargs):
calls["gather_warmup"] += 1
class _Pad:
warmup = staticmethod(pad_warmup)
def __getitem__(self, grid):
def launch(*args, **kwargs):
calls["pad_launch"] += 1
return launch
monkeypatch.setattr(model_state, "_pad_replayed_slots_kernel", _Pad())
monkeypatch.setattr(
model_state._gather_replay_batch_kernel, "warmup", gather_warmup
)
cfg = MagicMock()
cfg.model_config.enable_prompt_embeds = False
cfg.model_config.uses_mrope = False
cfg.model_config.is_multimodal_model = False
cfg.scheduler_config.max_num_seqs = 16
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
groups = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(
layer_names=["swa"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=False, prefix_replay_tokens=0
),
),
SimpleNamespace(
layer_names=["mla"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=True, prefix_replay_tokens=0
),
),
]
)
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
return {"swa": kw["model_specific_attn_metadata"]}
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
state._replay_start_np[0] = 16
batch = _input_batch([8], [24], [True])
state.prepare_attn(
batch,
CUDAGraphMode.NONE,
BLOCK_TABLES,
_slot_mappings(batch.num_tokens_after_padding),
[],
groups,
)
assert state._replay[0] == 0
assert calls == {"pad_warmup": 0, "gather_warmup": 1, "pad_launch": 0}
assert state._replay_start[:1].tolist() == [16]
def test_dp_ranks_pad_to_one_replay_graph(dp_state):
"""Every rank pads its replay batch to the graph fitting the largest rank's."""
dp_state.replay_cudagraphs = MagicMock()
@@ -29,6 +29,7 @@ def _make_experts(
intermediate_size=INTER,
max_num_batched_tokens=3000,
swiglu_limit=None,
fuse_shared_expert=False,
):
monkeypatch.setattr(
mega_moe_module,
@@ -47,6 +48,7 @@ def _make_experts(
hidden_size=HIDDEN,
intermediate_size=intermediate_size,
swiglu_limit=swiglu_limit,
fuse_shared_expert=fuse_shared_expert,
)
@@ -108,6 +110,49 @@ def test_experts_load_into_local_slots(monkeypatch):
assert torch.all(w2 == 10 * expert + 3)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"tokens_across_dp, expected",
[([4, 128], "decode"), ([4, 129], "prefill")],
)
def test_runtime_selected_by_max_tokens_across_dp(
monkeypatch, tokens_across_dp, expected
):
experts = _make_experts(monkeypatch)
experts._runtimes = [(128, "decode"), (4096, "prefill")]
monkeypatch.setattr(mega_moe_module, "is_forward_context_available", lambda: True)
monkeypatch.setattr(
mega_moe_module,
"get_forward_context",
lambda: SimpleNamespace(
dp_metadata=SimpleNamespace(
num_tokens_across_dp_cpu=torch.tensor(tokens_across_dp)
)
),
)
# Every rank must take the same runtime, so a small local batch still gets
# the prefill runtime when another DP rank is over the decode capacity.
assert experts.select_runtime(4) == expected
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
def test_shared_expert_route_appended(monkeypatch):
experts = _make_experts(
monkeypatch, ep_rank=3, ep_size=8, num_experts=384, fuse_shared_expert=True
)
ids = torch.randint(0, 384, (37, 2), device="cuda", dtype=torch.int32)
ids[::5, 1] = -1
weights = torch.rand(37, 2, device="cuda")
out_weights, out_ids = experts.append_shared_expert(weights, ids)
shared = torch.full((37, 1), 3 * 49 + 48, device="cuda", dtype=torch.int32)
expected_ids = torch.cat([torch.where(ids >= 0, ids + ids // 48, ids), shared], 1)
torch.testing.assert_close(out_ids, expected_ids)
torch.testing.assert_close(
out_weights, torch.cat([weights, torch.ones(37, 1, device="cuda")], 1)
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
@pytest.mark.parametrize("hash_routing", [False, True])
@pytest.mark.parametrize("num_tokens", [1, 37])
@@ -42,10 +42,6 @@ def _disable_distributed_loader_paths(monkeypatch: pytest.MonkeyPatch):
"fused_moe_make_expert_params_mapping",
lambda *args, **kwargs: [],
)
monkeypatch.setattr(
dspark_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(dspark_module, "get_tensor_model_parallel_rank", lambda: 0)
@pytest.mark.cpu_test
@@ -142,6 +142,7 @@ def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
is_act_and_mul=True,
activation=MoEActivation.SILU,
has_hash_routing=False,
require_decomposed_backend=False,
routing_method=RoutingMethodType.DeepseekV4,
router_logits_dtype=torch.float32,
hidden_dim=256,
+362 -233
View File
@@ -5,6 +5,7 @@ from types import SimpleNamespace
import pytest
import torch
from transformers import DeepseekV4Config
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
DeepGemmSm100MegaMoEBackend,
@@ -12,8 +13,9 @@ from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_weight_scale_block,
from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
is_mixture_of_experts,
)
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
@@ -23,7 +25,6 @@ from vllm.models.deepseek_v4.nvidia.model import (
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
@@ -38,6 +39,15 @@ pytestmark = pytest.mark.skipif(
)
@pytest.fixture(autouse=True)
def sm100_mega_moe_backend(monkeypatch):
"""Build the SM100 kernel layout on any GPU; kernels are gated per test."""
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.get_deep_gemm_mega_moe_backend",
lambda *args: DeepGemmSm100MegaMoEBackend(),
)
@pytest.fixture
def v41_moe_config(dist_init):
return SimpleNamespace(
@@ -298,18 +308,6 @@ def test_deepseek_v4_mega_moe_expert_mapping():
]
def test_deepseek_v4_mega_moe_ue8m0_uint8_to_float():
raw = torch.tensor([0, 126, 127, 128], dtype=torch.uint8)
decoded = DeepseekV4MegaMoEExperts._ue8m0_uint8_to_float(raw)
assert torch.equal(decoded.view(torch.int32), raw.to(torch.int32) << 23)
assert decoded[0].item() == 0.0
assert decoded[1].item() == 0.5
assert decoded[2].item() == 1.0
assert decoded[3].item() == 2.0
@pytest.mark.parametrize("use_kimi", [False, True])
def test_deep_gemm_mega_moe_capture_precedes_eplb(monkeypatch, use_kimi):
experts_cls = DeepseekV4MegaMoEExperts
@@ -422,16 +420,141 @@ def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
return_success=True,
)
assert torch.equal(experts.w13_weight[0, :128], w1)
assert torch.equal(experts.w13_weight[0, 128:], w3)
assert torch.equal(experts.w2_weight[0], w2)
interleaved = experts.w13_weight[0].view(torch.uint8).view(16, 2, 8, 64)
assert torch.equal(interleaved[:, 0], w1.view(16, 8, 64))
assert torch.equal(interleaved[:, 1], w3.view(16, 8, 64))
assert torch.equal(experts.w2_weight[0].view(torch.uint8), w2)
assert torch.count_nonzero(experts.w13_weight[1]) == 0
def _reference_mega_moe_layout(w13, s13, w2, s2):
"""CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales."""
def interleave(t):
g, n, k = t.shape
halves = t.view(g, 2, n // 16, 8, k)
return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k)
def utccp(sf):
g, mn, k = sf.shape
return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k)
return (
interleave(w13),
utccp(interleave(s13.view(torch.int32))),
w2,
utccp(s2.view(torch.int32)),
)
def _make_mega_moe_experts(num_local_experts, num_logical_experts):
return DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=num_local_experts,
num_local_experts=num_local_experts,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=384,
num_logical_experts=num_logical_experts,
)
def _load_mega_moe_checkpoint_shuffled(experts, num_logical_experts, seed):
hidden, inter = experts.hidden_size, experts.intermediate_size
shapes = {
"w13_weight": (num_logical_experts, 2 * inter, hidden // 2),
"w13_weight_scale": (num_logical_experts, 2 * inter, hidden // 32),
"w2_weight": (num_logical_experts, hidden, inter // 2),
"w2_weight_scale": (num_logical_experts, hidden, inter // 32),
}
generator = torch.Generator().manual_seed(seed)
ckpt = {
name: torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
for name, shape in shapes.items()
}
shards = []
for expert_id in range(num_logical_experts):
for name, tensor in ckpt.items():
if name.startswith("w2_"):
shards.append((name, tensor[expert_id], "w2", expert_id))
else:
shards.append((name, tensor[expert_id, :inter], "w1", expert_id))
shards.append((name, tensor[expert_id, inter:], "w3", expert_id))
for i in torch.randperm(len(shards), generator=generator).tolist():
name, loaded_weight, shard_id, expert_id = shards[i]
if name.endswith("scale"):
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
assert experts.weight_loader(
getattr(experts, name),
loaded_weight,
f"experts.{name}",
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
return ckpt
def _expected_mega_moe_layout(ckpt, num_local_experts, num_logical_experts):
physical = torch.arange(num_local_experts) % num_logical_experts
ckpt = {name: tensor[physical] for name, tensor in ckpt.items()}
expected = _reference_mega_moe_layout(*ckpt.values())
if is_deep_gemm_supported():
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
w13, s13, w2, s2 = (t.cuda() for t in ckpt.values())
hidden, inter = w2.shape[1], w2.shape[2] * 2
scales = [
deep_gemm.transform_sf_into_required_layout(
(sf.to(torch.int32) << 23).view(torch.float32),
mn,
k,
(1, 32),
num_local_experts,
)
for sf, mn, k in ((s13, 2 * inter, hidden), (s2, hidden, inter))
]
(l1, l1_sf), (l2, l2_sf) = deep_gemm.transform_weights_for_mega_moe(
(w13.view(torch.int8), scales[0]), (w2.view(torch.int8), scales[1])
)
for actual, reference in zip((l1, l1_sf, l2, l2_sf), expected):
assert torch.equal(actual.cpu().view(reference.dtype), reference)
return expected
def _assert_mega_moe_layout(experts, expected):
actual = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
for tensor, reference in zip(actual, expected):
assert torch.equal(tensor.view(reference.dtype), reference)
def test_deepseek_v4_mega_moe_loads_shards_into_kernel_layout():
"""Shards in any order, including EPLB replicas, land in DeepGEMM's layout."""
experts = _make_mega_moe_experts(num_local_experts=4, num_logical_experts=3)
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 3, seed=0)
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 4, 3))
# DeepGEMM's MN-major scale strides.
assert experts.w13_weight_scale.stride() == (768 * 2, 1, 768)
assert experts.w2_weight_scale.stride() == (256 * 3, 1, 256)
def test_deepseek_v4_mega_moe_reload_overwrites_kernel_layout_in_place():
"""A second load must update the live kernel tensors without reallocating."""
experts = _make_mega_moe_experts(num_local_experts=2, num_logical_experts=2)
_load_mega_moe_checkpoint_shuffled(experts, 2, seed=0)
data_ptrs = [param.data_ptr() for param in experts.parameters()]
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 2, seed=1)
assert [param.data_ptr() for param in experts.parameters()] == data_ptrs
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 2, 2))
@pytest.mark.parametrize("intermediate_size", [512, 2304])
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(
monkeypatch, intermediate_size
):
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(intermediate_size):
"""Keep native widths so V4.1 routed and shared experts can fuse."""
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
@@ -445,216 +568,149 @@ def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(
hidden_size=128,
intermediate_size=intermediate_size,
)
originals = []
for param in (
experts.w13_weight,
experts.w13_weight_scale,
experts.w2_weight,
experts.w2_weight_scale,
):
param.data.random_(1, 128)
originals.append(param.data.clone())
monkeypatch.setattr(
experts, "_ensure_backend", lambda: DeepGemmSm100MegaMoEBackend()
)
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm",
lambda: SimpleNamespace(
transform_sf_into_required_layout=lambda sf, *args: sf,
transform_weights_for_mega_moe=lambda l1, l2: (l1, l2),
),
)
experts.finalize_weights()
assert experts.intermediate_size == intermediate_size
assert experts.w13_weight.shape == (2, 2 * intermediate_size, 64)
assert experts.w13_weight_scale.shape == (2, 2 * intermediate_size, 1)
assert experts.w2_weight.shape == (2, 128, intermediate_size // 2)
assert experts.w2_weight_scale.shape == (2, 128, intermediate_size // 128)
transformed = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
for actual, original in zip(transformed, originals):
expected = (
original
if actual.dtype == torch.int8
else experts._ue8m0_uint8_to_float(original)
)
assert torch.equal(actual.view(expected.dtype), expected)
for actual, param in zip(transformed, experts.parameters()):
assert actual.data_ptr() == param.data_ptr()
transformed_l1 = experts._transformed_l1_weights
experts.finalize_weights()
assert experts._transformed_l1_weights is transformed_l1
def test_deepseek_v4_mega_moe_loads_shared_experts_into_kernel_layout(monkeypatch):
"""Shared expert shards land directly in the MegaMoE kernel layout."""
monkeypatch.setattr(
DeepGemmSm100MegaMoEBackend, "supports_shared_experts", lambda *args: True
)
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=384,
num_shared_experts=1,
)
assert experts.has_fused_shared_experts
hidden, inter = 256, 384
generator = torch.Generator().manual_seed(42)
def rand(*shape):
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
w1, w3, w2 = rand(inter, hidden), rand(inter, hidden), rand(hidden, inter)
s1, s3, s2 = (
rand(inter, hidden // 32),
rand(inter, hidden // 32),
rand(hidden, inter // 32),
)
fp8, e8m0 = torch.float8_e4m3fn, torch.float8_e8m0fnu
# The down shard arrives as `.w2.` (MTP, DSpark) or `.down_proj.` (mapped).
for name, data in [
("shared_experts.w1.weight", w1.view(fp8)),
("shared_experts.w3.weight", w3.view(fp8)),
("shared_experts.down_proj.weight", w2.view(fp8)),
("shared_experts.w1.weight_scale", s1.view(e8m0)),
("shared_experts.w3.weight_scale", s3.view(e8m0)),
("shared_experts.w2.weight_scale", s2.view(e8m0)),
]:
loaded = experts.load_shared_expert_weight(f"ffn.{name}", data)
# Weight tracking checks this name against the registered parameters.
assert loaded in {f"ffn.experts.{n}" for n, _ in experts.named_parameters()}
def l1_halves():
return experts.shared_l1_weight.view(torch.uint8).view(-1, 2, 8, hidden)
# Every tensor matches DeepGEMM's batched transform of the checkpoint.
expected = _reference_mega_moe_layout(
*(t.unsqueeze(0) for t in (torch.cat([w1, w3]), torch.cat([s1, s3]), w2, s2))
)
actual = (
experts.shared_l1_weight,
experts.shared_l1_weight_scale,
experts.shared_l2_weight,
experts.shared_l2_weight_scale,
)
for tensor, reference in zip(actual, expected):
assert torch.equal(tensor.view(reference.dtype), reference[0])
# A reload of one shard overwrites in place and leaves the other half alone.
ptr = experts.shared_l1_weight.data_ptr()
w1b = rand(inter, hidden)
experts.load_shared_expert_weight("shared_experts.w1.weight", w1b.view(fp8))
assert experts.shared_l1_weight.data_ptr() == ptr
assert torch.equal(l1_halves()[:, 0], w1b.view(-1, 8, hidden))
assert torch.equal(l1_halves()[:, 1], w3.view(-1, 8, hidden))
with pytest.raises(ValueError):
experts.load_shared_expert_weight("shared_experts.gate_up_proj.weight", w1)
def _fake_deep_gemm(with_shared: bool):
def old_buffer(group): ...
def old_kernel(y): ...
def new_buffer(group, num_shared_experts=0): ...
def new_kernel(y, shared_l1_weights=None, shared_l2_weights=None): ...
return SimpleNamespace(
get_symm_buffer_for_mega_moe=new_buffer if with_shared else old_buffer,
fp8_fp4_mega_moe=new_kernel if with_shared else old_kernel,
get_block_m_for_mega_moe=lambda *args: 0,
)
@pytest.mark.parametrize(
"hidden_size,intermediate_size,block_size,mxfp8,packed",
"weight_dtype,scale_dtype,scale_shape,with_shared,fused",
[
pytest.param(128, 512, 128, False, False, id="aligned-block128"),
pytest.param(128, 512, 128, False, True, id="aligned-block128-packed"),
pytest.param(128, 512, 128, True, False, id="aligned-block128-mxfp8"),
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
pytest.param(128, 2304, 128, False, False, id="native-block128"),
pytest.param(128, 2304, 128, False, True, id="native-block128-packed"),
pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash-mxfp8"),
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), True, True),
(torch.float8_e4m3fn, torch.uint8, (256, 8), True, True),
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), False, False),
(torch.bfloat16, None, None, True, False),
(torch.float8_e4m3fn, torch.float32, (2, 2), True, False),
(torch.float8_e4m3fn, torch.uint8, (256, 16), True, False),
],
)
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
monkeypatch, hidden_size, intermediate_size, block_size, mxfp8, packed
def test_deepseek_v4_mega_moe_decides_shared_fusion_at_construction(
monkeypatch, weight_dtype, scale_dtype, scale_shape, with_shared, fused
):
"""Shared fusion preserves checkpoint weights and scales at native widths."""
if packed and not is_deep_gemm_supported():
pytest.skip("Packing the shared scales requires DeepGEMM")
class FakeDeepGemm:
transformed_dims: list[tuple[int, int]] = []
scale_inputs: list[tuple[int, ...]] = []
scale_values: list[torch.Tensor] = []
@staticmethod
def get_symm_buffer_for_mega_moe(*args, num_shared_experts=0, **kwargs):
return None
@staticmethod
def get_block_m_for_mega_moe(*args, **kwargs):
return 128
@staticmethod
def fp8_fp4_mega_moe(
y,
l1_weights,
l2_weights,
sym_buffer,
shared_l1_weights=None,
shared_l2_weights=None,
**kwargs,
):
return None
@classmethod
def transform_sf_into_required_layout(cls, sf, mn, k, *args, **kwargs):
cls.scale_inputs.append(tuple(sf.shape))
cls.scale_values.append(sf)
return torch.empty((sf.shape[0], mn, k // 128), dtype=torch.int32)
@classmethod
def transform_weights_for_mega_moe(cls, l1_weights, l2_weights):
cls.transformed_dims.append((l1_weights[0].dim(), l2_weights[0].dim()))
if l1_weights[0].dim() == 2:
return (l1_weights[0].clone(), l1_weights[1]), l2_weights
return l1_weights, l2_weights
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
"""Fuse only shared experts the installed kernel and the 1x32 layout fit."""
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm", lambda: _fake_deep_gemm(with_shared)
)
def linear():
scale = (
None if scale_dtype is None else torch.empty(scale_shape).to(scale_dtype)
)
return SimpleNamespace(
weight=torch.empty(256, 256, dtype=weight_dtype), weight_scale_inv=scale
)
shared = SimpleNamespace(gate_up_proj=linear(), down_proj=linear())
experts = DeepseekV4MegaMoEExperts(
vllm_config,
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=1,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
hidden_size=256,
intermediate_size=128,
num_shared_experts=1,
)
monkeypatch.setattr(
experts, "_ensure_backend", lambda: DeepGemmSm100MegaMoEBackend()
shared_experts=shared,
)
def fp8_parameter(*shape):
return torch.nn.Parameter(
torch.empty(*shape, dtype=torch.uint8)
.random_(1, 119)
.view(torch.float8_e4m3fn),
requires_grad=False,
)
def scale_parameter(*shape, dtype=torch.int32):
return torch.nn.Parameter(torch.ones(*shape, dtype=dtype), requires_grad=False)
shared_experts = SimpleNamespace(
gate_up_proj=SimpleNamespace(
weight=fp8_parameter(2 * intermediate_size, hidden_size),
weight_block_size=(block_size, block_size),
weight_scale_inv=scale_parameter(
2 * intermediate_size // block_size,
hidden_size // block_size,
dtype=torch.float8_e8m0fnu,
),
),
down_proj=SimpleNamespace(
weight=fp8_parameter(hidden_size, intermediate_size),
weight_block_size=(block_size, block_size),
weight_scale_inv=scale_parameter(
hidden_size // block_size,
intermediate_size // block_size,
dtype=torch.float8_e8m0fnu,
),
),
)
if mxfp8:
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
linear.weight_block_size = (1, 32)
linear.weight_scale = torch.nn.Parameter(
linear.weight_scale_inv.view(torch.uint8)
.repeat_interleave(block_size, dim=0)
.repeat_interleave(block_size // 32, dim=1),
requires_grad=False,
)
del linear.weight_scale_inv
originals = []
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
scale = linear.weight_scale if mxfp8 else linear.weight_scale_inv
scale.data.view(torch.uint8).random_(124, 130)
block_m, block_k = linear.weight_block_size
scale_1x32 = (
experts._ue8m0_uint8_to_float(scale.view(torch.uint8))
.repeat_interleave(block_m, dim=0)
.repeat_interleave(block_k // 32, dim=1)
)
originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32))
if packed:
linear.weight_scale_inv = torch.nn.Parameter(
deepgemm_post_process_weight_scale_block(
scale.data.cuda().unsqueeze(0),
*linear.weight.shape,
quant_block_shape=linear.weight_block_size,
num_groups=1,
)
.squeeze(0)
.cpu(),
requires_grad=False,
)
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
experts.finalize_weights(shared_experts)
assert experts.has_fused_shared_experts
assert experts.intermediate_size == intermediate_size
assert FakeDeepGemm.transformed_dims == [(3, 3), (2, 2)]
assert FakeDeepGemm.scale_inputs[-2:] == [
(1, 2 * intermediate_size, hidden_size // 32),
(1, hidden_size, intermediate_size // 32),
]
for transformed, actual_scale, (weight, scale) in zip(
(
experts._transformed_shared_l1_weights,
experts._transformed_shared_l2_weights,
),
FakeDeepGemm.scale_values[-2:],
originals,
):
assert torch.equal(transformed[0].view(torch.uint8), weight)
assert torch.equal(actual_scale[0], scale)
assert shared_experts.gate_up_proj.weight.data_ptr() != original_gate_up_ptr
assert (
experts._transformed_shared_l1_weights[0].data_ptr()
== shared_experts.gate_up_proj.weight.data_ptr()
)
assert (
experts._transformed_shared_l2_weights[0].data_ptr()
== shared_experts.down_proj.weight.data_ptr()
)
transformed_l1 = experts._transformed_shared_l1_weights
experts.finalize_weights(shared_experts)
assert experts._transformed_shared_l1_weights is transformed_l1
assert experts.has_fused_shared_experts == fused
assert hasattr(experts, "shared_l1_weight") == fused
@pytest.mark.parametrize("fused", [False, True])
@@ -671,10 +727,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
class FakeExperts(torch.nn.Module):
has_fused_shared_experts = fused
_backend = DeepGemmSm100MegaMoEBackend()
def _ensure_backend(self):
return self._backend
backend = DeepGemmSm100MegaMoEBackend()
def forward(self, hidden_states, *args, **kwargs):
return torch.ones_like(hidden_states)
@@ -691,7 +744,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
moe.use_native_mega_moe = True
moe.gate = FakeGate()
moe.experts = FakeExperts()
moe.shared_experts = FakeSharedExperts()
moe.shared_experts = None if fused else FakeSharedExperts()
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = 1
moe.renormalize = True
@@ -710,7 +763,8 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
expected = 1 if fused else 3
assert torch.all(output == expected)
assert moe.shared_experts.calls == (0 if fused else 1)
if not fused:
assert moe.shared_experts.calls == 1
@pytest.mark.skipif(
@@ -882,35 +936,110 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
assert torch.all(fused_shared_x_sf[~populated] == -1)
def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
def test_deepseek_v4_pwal_hook_finalizes_mhc_broadcast():
"""The loader invokes the model-level PWAL hook for every load format,
so it must finalize megamoe + mhc broadcast weights to cover dummy
load, which skips load_weights()."""
so it must finalize mhc broadcast weights to cover dummy load, which
skips load_weights()."""
calls = []
stub = SimpleNamespace(
model=SimpleNamespace(
finalize_mega_moe_weights=lambda: calls.append("mega_moe"),
finalize_mhc_broadcast_weights=lambda: calls.append("mhc"),
)
)
DeepseekV4ForCausalLM.process_weights_after_loading(stub)
assert calls == ["mega_moe", "mhc"]
assert calls == ["mhc"]
def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
"""MTP/DSpark drafters load as their own top-level models, so each needs
its own PWAL hook now that the megamoe forward no longer finalizes
weights lazily on first use."""
calls = []
mtp = SimpleNamespace(finalize_mega_moe_weights=lambda: calls.append("mtp"))
DeepSeekV4MTP.process_weights_after_loading(mtp)
@pytest.fixture
def v4_dspark_config(dist_init):
hf_config = DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hidden_act="silu",
swiglu_limit=10.0,
norm_topk_prob=True,
topk_method="noaux_tc",
routed_scaling_factor=1.5,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
layer_types=["sliding_attention"] * 2,
q_lora_rank=32,
o_lora_rank=32,
qk_rope_head_dim=16,
o_groups=4,
sliding_window=128,
hc_sinkhorn_iters=1,
)
model_config = SimpleNamespace(
dtype=torch.bfloat16,
hf_config=hf_config,
max_model_len=2048,
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(
moe_backend="deep_gemm_mega_moe",
enable_jit_warmup=False,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(draft_model_config=model_config),
use_v2_model_runner=False,
)
dspark = SimpleNamespace(_finalize_moe=lambda: calls.append("dspark"))
DSparkDeepseekV4ForCausalLM.process_weights_after_loading(dspark)
assert calls == ["mtp", "dspark"]
def test_dspark_draft_registers_mixture_of_experts(
v4_dspark_config, monkeypatch, dist_init
):
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
if not current_platform.is_device_capability_family(100):
pytest.skip("DeepGEMM MegaMoE requires SM100")
hf_config = v4_dspark_config.model_config.hf_config
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
lambda: v4_dspark_config,
)
with (
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
torch.device("cuda"),
):
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
assert is_mixture_of_experts(draft)
assert get_mixture_of_experts_model(draft) is draft
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
assert draft.num_routed_experts == hf_config.n_routed_experts
assert draft.num_redundant_experts == 4
@pytest.mark.skipif(
+58 -4
View File
@@ -242,7 +242,14 @@ def test_v41_dspark_loads_linear_scales(
linear_scale_name=scale_name,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
layers=[
SimpleNamespace(
ffn=SimpleNamespace(
use_native_mega_moe=False,
experts=SimpleNamespace(has_fused_shared_experts=False),
)
)
],
confidence_head=None,
),
named_parameters=lambda: [(runtime_name, param)],
@@ -251,8 +258,6 @@ def test_v41_dspark_loads_linear_scales(
draft._remap_dspark_name = lambda name: (
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 4)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
dspark, "fused_moe_make_expert_params_mapping", lambda *a, **kw: []
)
@@ -266,6 +271,55 @@ def test_v41_dspark_loads_linear_scales(
torch.testing.assert_close(param, checkpoint_scale)
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
monkeypatch,
):
from vllm.models.deepseek_v4.nvidia import dspark
captured: dict[str, int] = {}
def capture_expert_mapping(*args, **kwargs):
captured.update(kwargs)
return []
draft = SimpleNamespace(
config=SimpleNamespace(
num_attention_heads=4,
n_routed_experts=8,
expert_dtype="fp4",
),
quant_config=None,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[
SimpleNamespace(
ffn=SimpleNamespace(
use_native_mega_moe=False,
use_mega_moe=False,
n_redundant_experts=4,
)
)
],
confidence_head=None,
),
named_parameters=lambda: [],
process_weights_after_loading=lambda: None,
)
draft._remap_dspark_name = lambda name: (
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
)
monkeypatch.setattr(
dspark,
"fused_moe_make_expert_params_mapping",
capture_expert_mapping,
)
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
assert captured["num_experts"] == 8
assert captured["num_redundant_experts"] == 4
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
weights = [
("mtp.0.attn.wkv.weight", torch.arange(4)),
@@ -343,7 +397,7 @@ def test_k3_dspark_mla_kv_cache_spec_groups_with_target_mla():
"""The draft's MLA layers must share a KV cache group with the target's."""
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
from vllm.v1.core.kv_cache_utils import _get_kv_cache_groups_uniform_page_size
from vllm.v1.core.kv_cache_planning import _get_kv_cache_groups_uniform_page_size
vllm_config = SimpleNamespace(
model_config=None, cache_config=SimpleNamespace(block_size=64)
+372
View File
@@ -0,0 +1,372 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode layer runs only where its kernels are exact:
decode-only steps of at most MAX_ROWS rows with causal SWA windows, eager or in
a FULL graph. A PIECEWISE capture is replayed for mixed batches, so it must
never record the mono launches. Layers outside the whole-layer kernels run the
FFN launch on the same steps, from wo_b's unreduced output."""
import sys
from types import ModuleType, SimpleNamespace
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.models.deepseek_v41.amd import mono_decode as md
M = 6
class _Runner:
def __init__(self):
self.calls = []
def forward(self, *args, **kwargs):
self.calls.append((args, kwargs))
return ("mono outputs",)
def first_pre_mix(self, rows):
return torch.zeros(rows, 4)
def ffn(self, *args):
self.calls.append((args, {}))
return ("ffn outputs",)
def _layer(comp="comp", indexer=None):
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
compressed_cache_prefix=comp,
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
compress_ratio=1,
indexer=indexer,
_compressed_kv_cache=cache,
)
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
def _swa(**overrides):
fields = dict(
num_prefills=0,
num_decodes=1,
num_decode_tokens=M,
decode_swa_width=md.SWA_WIDTH,
decode_swa_indices=torch.zeros(64, 1, md.SWA_WIDTH, dtype=torch.int32),
decode_swa_lens=torch.zeros(64, dtype=torch.int32),
token_to_req_indices=torch.zeros(64, dtype=torch.int32),
slot_mapping=torch.zeros(64, dtype=torch.int64),
block_size=32,
)
fields.update(overrides)
return SimpleNamespace(**fields)
def _inputs(rows=M):
return dict(
x=torch.zeros(rows, 5120, dtype=torch.bfloat16),
positions=torch.zeros(rows, dtype=torch.int64),
residual=torch.zeros(rows, 4, 5120, dtype=torch.bfloat16),
post_mix=torch.zeros(rows, 4, 1),
res_mix=torch.zeros(rows, 4, 4),
pre_mix=torch.zeros(rows, 4),
)
@pytest.fixture
def run(monkeypatch):
"""Call the mono path for one layer under a forward context built from
``mode`` and ``metadata``; returns (result, runner calls)."""
def _run(
mode=CUDAGraphMode.FULL,
metadata=None,
ffn_only=False,
entry=None,
layer=None,
**inputs,
):
if metadata is None:
metadata = {
"swa": _swa(),
"comp": SimpleNamespace(block_size=128, block_table=None),
}
fc = SimpleNamespace(cudagraph_runtime_mode=mode, attn_metadata=metadata)
runner = _Runner()
monkeypatch.setattr(md, "is_forward_context_available", lambda: True)
monkeypatch.setattr(md, "get_forward_context", lambda: fc)
monkeypatch.setattr(md, "_mono_runner", lambda device: runner)
monkeypatch.setattr(md.MonoDecodeLayer, "weights", lambda self, layer: None)
args = {**_inputs(), **inputs}
mono = md.MonoDecodeLayer(ffn_only)
if (entry or ("ffn" if ffn_only else "layer")) == "ffn":
# x stands for wo_b's unreduced output; the first layer's attention
# has no compressed cache
part, _ = args.pop("x"), args.pop("positions")
return mono.ffn(_layer(comp=None), part, **args), runner.calls
return mono(layer or _layer(), **args), runner.calls
return _run
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
def test_decode_step_runs_mono(run, mode):
out, calls = run(mode)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
# the caches reach the kernels as [blocks, block, record] bytes
assert args[8].shape == (4, 32, md.RECORD) and args[8].stride() == (
128 * md.RECORD,
md.RECORD,
1,
)
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
assert kwargs["fold"] and "select" not in kwargs
def test_first_layer_runs_mono_from_the_embedding(run):
"""The first layer's residual is the embedding broadcast to the four
streams, which K1 takes as it is (no owed post to fold)."""
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
assert out == ("mono outputs",) and len(calls) == 1
args, kwargs = calls[0]
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
def test_index_source_selects_between_the_kernels(run):
"""An index source hands the runner its compressor and indexer to run
between K1 and K2."""
out, calls = run(layer=_layer(indexer=object()))
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
def test_padded_decode_step_runs_mono(run):
"""A FULL graph pads the step to its capture size: the kernels take it and
skip the pad rows (their slot is -1)."""
swa = _swa(num_decode_tokens=M - 2)
comp = SimpleNamespace(block_size=128, block_table=None)
out, calls = run(metadata={"swa": swa, "comp": comp})
assert out == ("mono outputs",) and len(calls) == 1
@pytest.mark.parametrize(
"case",
[
dict(mode=CUDAGraphMode.PIECEWISE),
dict(
metadata={
"swa": _swa(num_prefills=1),
"comp": SimpleNamespace(block_size=128),
}
),
dict(
metadata={
"swa": _swa(decode_swa_width=256),
"comp": SimpleNamespace(block_size=128),
}
),
dict(
metadata={
"swa": _swa(num_decode_tokens=M + 1),
"comp": SimpleNamespace(block_size=128),
}
),
dict(metadata={}), # a profile / dummy run: no attention metadata
dict(**_inputs(md.MAX_ROWS + 6)),
# a later pipeline stage's first layer: its stream arrives without
# the mixes
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
],
ids=[
"piecewise",
"prefill",
"noncausal-window",
"tokens-past-rows",
"no-metadata",
"rows",
"stage-input",
],
)
def test_other_steps_take_the_original_path(run, case):
out, calls = run(**case)
assert out is None and not calls
def _decoder_layer(engram=None, **attn):
attn = (
dict(
layer_id=21,
compressor=None,
indexer=None,
compress_ratio=1,
kv_mxfp8=False,
kv_cache_dtype="fp8_ds_mla",
)
| attn
)
backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
quant_method = SimpleNamespace(mxfp4_backend=backend)
ffn = SimpleNamespace(
experts=SimpleNamespace(
routed_experts=SimpleNamespace(quant_method=quant_method)
),
shared_experts=object(),
gate=SimpleNamespace(tid2eid=None),
n_routed_experts=384,
n_activated_experts=6,
routed_scaling_factor=1.5,
swiglu_limit=10.0,
scoring_func="sqrtsoftplus",
renormalize=True,
)
return SimpleNamespace(
attn=SimpleNamespace(**attn),
ffn=ffn,
engram=engram,
use_sequence_parallel=False,
fuse_seam_norm=True,
)
def _indexer():
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
return object.__new__(DeepseekV41RocmMxfp4Indexer)
@pytest.mark.parametrize(
"layer, path",
[
(_decoder_layer(), "whole"),
(_decoder_layer(compress_ratio=0), "whole"),
(_decoder_layer(engram=object()), "whole"),
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
(_decoder_layer(indexer=_indexer()), "whole"),
(_decoder_layer(compressor=object()), "ffn"),
(_decoder_layer(indexer=object()), "ffn"),
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
(_decoder_layer(layer_id=40), None),
],
ids=[
"standard",
"window",
"engram",
"kv-source",
"index-source",
"compressor-only",
"other-indexer",
"kv",
"draft",
],
)
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
index source's compressor and indexer run between them); a layer outside
them keeps vLLM's attention and runs its FFN half as one launch."""
_deployment(monkeypatch)
mono = md.MonoDecodeLayer.create(layer, _config())
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
assert got == path
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
def test_decode_step_runs_ffn_launch(run, mode):
out, calls = run(mode, ffn_only=True, metadata={"swa": _swa()})
assert out == ("ffn outputs",) and len(calls) == 1
assert calls[0][0][1].shape == (M, 5120) # wo_b's unreduced output
def test_ffn_layer_never_runs_whole(run):
out, calls = run(ffn_only=True, entry="layer")
assert out is None and not calls
@pytest.mark.parametrize(
"case",
[
dict(mode=CUDAGraphMode.PIECEWISE),
dict(metadata={"swa": _swa(num_prefills=1)}),
dict(metadata={"swa": _swa(num_decode_tokens=M + 1)}),
dict(metadata={}),
dict(**_inputs(md.MAX_ROWS + 6)),
],
ids=["piecewise", "prefill", "tokens-past-rows", "no-metadata", "rows"],
)
def test_other_steps_reduce_wo_b_themselves(run, case):
"""None: the layer all-reduces wo_b's partial and runs the FFN as before."""
case.setdefault("metadata", {"swa": _swa()})
out, calls = run(ffn_only=True, **case)
assert out is None and not calls
def _config(monokernel=True, **parallel):
parallel = (
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
| parallel
)
return SimpleNamespace(
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
parallel_config=SimpleNamespace(**parallel),
kernel_config=SimpleNamespace(monokernel=monokernel),
)
def _deployment(monkeypatch, cdna=4, cus=256):
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
import vllm.platforms.rocm as rocm
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(md, "_compute_units", lambda: cus)
name = "vllm.models.deepseek_v41.amd.mono.runner"
runner = ModuleType(name)
runner.BLOCKS = 256 # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, name, runner)
def test_off_without_the_flag(monkeypatch):
"""No --monokernel: no mono layer, whatever the deployment."""
_deployment(monkeypatch)
assert (
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
)
@pytest.mark.parametrize("cdna", [3, 5])
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
_deployment(monkeypatch, cdna=cdna)
with pytest.raises(ValueError, match="needs CDNA4"):
md.MonoDecodeLayer.create(_decoder_layer(), _config())
def _other_backend():
layer = _decoder_layer()
experts = layer.ffn.experts.routed_experts
experts.quant_method.mxfp4_backend = Mxfp4MoeBackend.TRITON
return layer
@pytest.mark.parametrize(
"case, match",
[
(dict(parallel=dict(enable_expert_parallel=True)), "no expert"),
(dict(parallel=dict(data_parallel_size=2)), "no expert"),
(dict(parallel=dict(enable_eplb=True)), "no expert"),
(dict(a4w4=True), "A8W4"),
(dict(layer=_other_backend), "AITER_MXFP4_BF16"),
(dict(cus=128), "256 compute units"),
],
ids=["expert-parallel", "data-parallel", "eplb", "a4w4", "moe-backend", "cus"],
)
def test_opt_in_outside_the_kernels_moe_raises(monkeypatch, case, match):
"""The kernels read all 384 experts on each rank in AITER's A8W4 layout, with
all 256 CTAs resident: any other deployment is an error, not garbage."""
_deployment(monkeypatch, cus=case.get("cus", 256))
if case.get("a4w4"):
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1")
layer = case.get("layer", _decoder_layer)()
with pytest.raises(ValueError, match=match):
md.MonoDecodeLayer.create(layer, _config(**case.get("parallel", {})))
+340
View File
@@ -0,0 +1,340 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
and off on the same random weights: every decoder layer's outputs at every
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
checks the kernels against vLLM's ops one stage at a time, this runs
``DeepseekV4DecoderLayer.forward`` as the engine does -- the model's hooks,
the per-step path choice, the weights taken off the layer, the metadata and
the indexer's top-k -- so a change to the decoder layer that the kernels do
not follow fails here.
Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
- 0: the first layer, sliding window only (K1 takes the embedding settled)
- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
and K2)
- 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
- 3: ratio-1 KV, index and candidate source
- 4: ratio-1 index source over 3's candidates and KV
"""
import zlib
import pytest
import torch
from vllm.model_executor.model_loader import register_model_loader
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.platforms import current_platform
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
LOAD_FORMAT = "dsv41_mono_random"
TRUNCATED = {
"num_hidden_layers": 5,
# one entry a layer, the three MTP layers' included
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
"kv_source_layer_ids": [1, 3],
# 4: an indexer of its own over 3's candidates (a REINDEX layer)
"index_source_layer_ids": [1, 3, 4],
"candidate_source_layer_id": 3,
# an Engram on a whole layer (K1 then takes its residual settled), its
# tables shrunk: 24 prime-sized ranges from 1000
"engram_layer_ids": [2],
"engram_num_embeddings": [32768],
"engram_vocab_size": 1000,
"dspark_target_layer_ids": [2, 3, 4],
}
WHOLE, FFN = (0, 1, 2, 3, 4), ()
STEPS = 5 # decode steps after the prefill's token
def _on_cdna4() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() == 4
@torch.no_grad()
def _fill(name: str, t: torch.Tensor) -> None:
"""A random value for every weight, seeded by its name (alike on every
rank and in every run), in the ranges the checkpoint's formats hold."""
if t.device.type == "meta":
return # materialized and filled by the dummy loader's own path
g = torch.Generator(device=t.device).manual_seed(zlib.crc32(name.encode()))
def randn(scale=1.0, shift=0.0):
return torch.randn(t.shape, generator=g, device=t.device) * scale + shift
if t.dtype == torch.uint8 and "scale" in name:
# E8M0 2^-12 .. 2^-8, alike over each 32 rows as in the checkpoint (the
# loader folds per-row scales into 32 x 32 blocks only then). Larger
# random layers make the stack chaotic: atomics' order alone moves
# vLLM's own logits run to run.
rows = t.shape[-2] if t.dim() > 1 else 1
shape = (*t.shape[:-2], -(-rows // 32), t.shape[-1]) if t.dim() > 1 else t.shape
e8m0 = torch.randint(115, 120, shape, generator=g, device=t.device)
if t.dim() > 1:
e8m0 = e8m0.repeat_interleave(32, dim=-2)[..., :rows, :]
t.copy_(e8m0)
elif t.dtype in (torch.uint8, torch.float4_e2m1fn_x2):
u8 = t.view(torch.uint8)
u8.copy_(torch.randint(0, 256, u8.shape, generator=g, device=t.device))
elif t.dtype == torch.float8_e4m3fn:
t.copy_(randn().clamp(-6, 6))
elif not torch.is_floating_point(t):
return
elif "norm" in name:
t.copy_(randn(0.1, 1.0))
elif name.endswith(("hc_attn_scale", "hc_ffn_scale")):
t.copy_(randn(0.5).abs() + 0.5)
elif name.endswith(("hc_attn_base", "hc_ffn_base", "attn_sink")):
t.copy_(randn(0.1))
elif name.endswith("embed_tokens.weight"):
t.copy_(randn())
else:
t.copy_(randn(0.02))
@register_model_loader(LOAD_FORMAT)
class RandomWeightLoader(DummyModelLoader):
"""Dummy loading with weights in the checkpoint's ranges, before post-load
processing, so every processed copy follows (plain dummy weights leave the
E8M0 scales zero, and both paths would agree on zeros)."""
def load_weights(self, model, model_config) -> None:
from vllm.distributed import get_tensor_model_parallel_world_size
super().load_weights(model, model_config)
for name, t in model.state_dict().items():
_fill(name, t)
# the routed experts' intermediate past this rank's share is the
# loader's zero padding (576 -> 640 at TP4): the kernels skip it
inter = model_config.hf_text_config.moe_intermediate_size
inter //= get_tensor_model_parallel_world_size()
for m in model.modules():
w13, w2 = getattr(m, "w13_weight", None), getattr(m, "w2_weight", None)
if w13 is None or w2 is None or w13.device.type == "meta":
continue
padded = w13.shape[1] // 2
if padded > inter:
w13, w2 = w13.data.view(torch.uint8), w2.data.view(torch.uint8)
for half in (0, padded):
w13[:, half + inter : half + padded] = 0
w2[..., inter // 2 :] = 0
class MonoProbe:
"""Worker extension (``worker_extension_cls``): it also brings this module,
and so the loader, into the workers."""
def _decoder_layers(self) -> dict:
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
model = self.model_runner.model # type: ignore[attr-defined]
return {
m.attn.layer_id: m
for m in model.modules()
if isinstance(m, DeepseekV4DecoderLayer)
}
def capture_steps(self, lens: list[int]) -> None:
"""Keep each decoder layer's outputs (x, residual) for every decode
token taken in a decode-only step (the steps the mono path takes),
keyed by (layer, prompt length, position): a request's step and row
can differ between engines (requests reach the engine core while it
runs), its prompt length and position cannot. ``lens``: the prompt
lengths, all distinct; a row's request is the longest prompt not past
its position."""
from vllm.forward_context import get_forward_context
self._captured: dict = {}
lens = sorted(lens)
def hook(layer_id):
def keep(module, args, kwargs, out):
md = get_forward_context().attn_metadata
if not isinstance(md, dict):
return
swa = md.get(module.attn.swa_cache_layer.prefix)
if swa is None or swa.num_prefills != 0:
return
pos = kwargs.get("positions", args[1] if len(args) > 1 else None)
for row, p in enumerate(pos.tolist()):
n = max(n for n in lens if n <= p)
kept = [t[row].float().cpu() for t in out[:2]]
self._captured.setdefault((layer_id, n, p), kept)
return keep
def gap_hook(layer_id, layer):
def keep(ffn, args, kwargs):
# vLLM's path only (the mono kernels route inside): each row's
# top-6 margin, sqrt(softplus(logits)) + bias as vLLM routes
md = get_forward_context().attn_metadata
if not isinstance(md, dict):
return
swa = md.get(layer.attn.swa_cache_layer.prefix)
if swa is None or swa.num_prefills != 0:
return
h = args[0] if args else kwargs["hidden_states"]
logits = h.float() @ ffn.gate.weight.float().T
score = torch.nn.functional.softplus(logits).sqrt()
score = score + ffn.gate.e_score_correction_bias.float()
top = score.topk(7, dim=-1).values
gap = (top[:, 5] - top[:, 6]).tolist()
pos = self._positions
for row, p in enumerate(pos.tolist()):
n = max(n for n in lens if n <= p)
self._gaps.setdefault((layer_id, n, p), gap[row])
return keep
def pos_hook(module, args, kwargs):
self._positions = kwargs.get(
"positions", args[1] if len(args) > 1 else None
)
self._gaps: dict = {}
for i, layer in self._decoder_layers().items():
layer.register_forward_hook(hook(i), with_kwargs=True)
layer.register_forward_pre_hook(pos_hook, with_kwargs=True)
gap = gap_hook(i, layer)
layer.ffn.register_forward_pre_hook(gap, with_kwargs=True)
def save_capture(self, path: str) -> None:
rank = self.rank # type: ignore[attr-defined]
torch.save((self._captured, self._gaps), f"{path}.rank{rank}")
def mono_layers(self) -> dict:
from vllm.models.deepseek_v41.amd import mono_decode
layers = {
i: None
if m.mono is None
else ["ffn" if m.mono.ffn_only else "whole", m.mono._weights is not None]
for i, m in self._decoder_layers().items()
}
runner = mono_decode._runner
return {
"layers": layers,
"epoch": -1 if runner is None else int(runner.epoch[0]),
}
def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
"""Greedy decode of ``prompts`` with the mono path on or off: (each
prompt's output, the workers' mono layers); each decoder layer's outputs
for the decode tokens of decode-only steps, and vLLM's routing margins,
saved under ``path``."""
from vllm import SamplingParams
from vllm.inputs import TokensPrompt
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
with vllm_runner(
MODEL,
tensor_parallel_size=tp,
load_format=LOAD_FORMAT,
hf_overrides=TRUNCATED,
worker_extension_cls=f"{__name__}.MonoProbe",
moe_backend="aiter",
monokernel=mono,
language_model_only=True,
tokenizer_mode="deepseek_v41",
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
block_size=128,
enforce_eager=True,
max_model_len=4096,
max_num_seqs=len(prompts),
# a prompt prefills in one step, so its decode tokens never share one
max_num_batched_tokens=16384,
enable_prefix_caching=False,
gpu_memory_utilization=0.5,
) as runner:
runner.llm.collective_rpc("capture_steps", args=([len(p) for p in prompts],))
params = SamplingParams(temperature=0.0, max_tokens=STEPS + 1)
outs = runner.llm.generate(
[TokensPrompt(prompt_token_ids=p) for p in prompts], params
)
runner.llm.collective_rpc("save_capture", args=(path,))
probe = runner.llm.collective_rpc("mono_layers")
return [o.outputs[0] for o in outs], probe
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
return ((a - b).norm() / b.norm()).item()
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
@pytest.mark.parametrize("tp", [2, 4])
def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
if torch.accelerator.device_count() < tp:
pytest.skip(f"needs {tp} GPUs")
pytest.importorskip("flydsl")
g = torch.Generator().manual_seed(0)
# contexts past both ratios' 512 compressed entries, so top-k selects
lens = torch.randperm(1700, generator=g)[:8].add(300).tolist() # distinct
prompts = [torch.randint(1000, 60000, (n,), generator=g).tolist() for n in lens]
on, probe_on = _decode(
vllm_runner, monkeypatch, tp, True, prompts, f"{tmp_path}/on"
)
off, probe_off = _decode(
vllm_runner, monkeypatch, tp, False, prompts, f"{tmp_path}/off"
)
# both layer kinds took the mono path on every rank, at every decode step
launches = len(WHOLE + FFN) # one launch (pair) a layer a step
for rank in probe_on:
assert rank["layers"] == {
i: ["whole" if i in WHOLE else "ffn", True] for i in WHOLE + FFN
}, rank
assert rank["epoch"] >= launches * STEPS, rank
assert rank["epoch"] % launches == 0, rank
assert all(set(r["layers"].values()) == {None} for r in probe_off)
# each decode token's input matches between the runs while the greedy
# tokens before it do: (prompt length, position) pairs to compare
same: set[tuple[int, int]] = set()
for prompt, a, b in zip(prompts, on, off):
for step in range(STEPS):
if a.token_ids[: step + 1] != b.token_ids[: step + 1]:
break
same.add((len(prompt), len(prompt) + step))
# each layer's outputs for those tokens, mono against vLLM's path, token by
# token. Layer 0 sees the same inputs on both paths, but its FFN input
# differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
# amplify through the MoE: 3%. Each later layer takes the previous one's
# slightly different output, which random weights amplify (2-4% by layer
# 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
# another expert: a 0.4% input difference moves scores near 0.8 by about
# 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
# input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
# Once it has, its residual stream differs and the later layers are exempt
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
# longer follow is off by far more, on every token.
for rank in range(tp):
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
want, gaps = torch.load(f"{tmp_path}/off.rank{rank}")
keys: list[tuple[int, int]] = sorted(
same & {k[1:] for k in got} & {k[1:] for k in want}
)
assert len(keys) >= len(prompts), keys
flipped: set = set()
for i in sorted(WHOLE + FFN):
for k in keys:
if k in flipped:
continue
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
err = _rel(res, res_ref)
assert err < 0.03, (rank, i, k, "residual", err)
bound = 0.03 if i == 0 else 0.08
err = _rel(x, x_ref)
if err >= bound:
tie = 5e-3 if i == 0 else 1e-2
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
flipped.add(k)
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
+533
View File
@@ -0,0 +1,533 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Numerics of the DeepSeek-V4.1 mono decode kernels at TP2 on CDNA4 against the
ops vLLM runs for the same decode step: random weights in the checkpoint's
formats, the step's KV caches as vLLM's insert writes them.
- The whole layer (K1 + K2) against the mHC seams, the ROCm attention op chain,
the TP all-reduce and ``DeepseekV4MoE``.
- The FFN launch against the same FFN half, from wo_b's unreduced output.
Each stage is compared on its own input, so a failure names the stage; the MoE
is checked on the kernels' own FFN input, where routing cannot differ."""
import pytest
import ray
import torch
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_process_parallel,
)
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
HIDDEN, Q_RANK, HEAD_DIM, NOPE, ROPE = 5120, 1280, 512, 448, 64
N_HEADS, O_GROUPS, O_RANK = 64, 8, 1024
WINDOW, TOPK, SWA_BLOCK, CACHE_BLOCK = 128, 512, 32, 128
RECORD, DATA, ALIGN = 584, 576, 576
REPLAYS = 300
HC = 4
def _on_cdna4() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() == 4
def _cos(a: torch.Tensor, b: torch.Tensor) -> float:
a, b = a.float().reshape(-1), b.float().reshape(-1)
return torch.nn.functional.cosine_similarity(a, b, dim=0).item()
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
"""Relative L2 error of ``a`` against ``b``: unlike the cosine, it sees a
wrong scale."""
a, b = a.float().reshape(-1), b.float().reshape(-1)
return ((a - b).norm() / b.norm()).item()
# ---------------------------------------------------------------- caches
def _paged_cache(num_blocks: int, block: int, device) -> torch.Tensor:
"""[blocks, block, 584] u8 over pages of round_up(block * 584, 576) bytes,
the way vLLM binds a packed V4 cache."""
page = -(-block * RECORD // ALIGN) * ALIGN
raw = torch.zeros(num_blocks * page, dtype=torch.uint8, device=device)
return torch.as_strided(raw, (num_blocks, block, RECORD), (page, RECORD, 1))
def _rope(kv: torch.Tensor, positions: torch.Tensor, cos_sin: torch.Tensor):
"""GPT-J RoPE of the last 64 dims (fp32 math, bf16 out)."""
cs = cos_sin[positions.long()]
cos, sin = cs[:, :32], cs[:, 32:]
x = kv.float()
e, o = x[:, NOPE::2].clone(), x[:, NOPE + 1 :: 2].clone()
x[:, NOPE::2] = e * cos - o * sin
x[:, NOPE + 1 :: 2] = e * sin + o * cos
return x.to(torch.bfloat16)
def _write_records(cache, block: int, slots, kv):
"""fp8_ds_mla records of roped bf16 kv rows at ``slots``: the 448 nope dims
in e4m3 with a UE8M0 per 64, the rope dims in bf16, as vLLM's insert."""
nope = kv.float()[:, :NOPE].view(-1, 7, 64)
exp = torch.ceil(torch.log2(nope.abs().amax(-1).clamp_min(1e-4) / 448.0))
q = (nope * torch.exp2(-exp)[..., None]).clamp(-448, 448).reshape(-1, NOPE)
data = torch.cat(
[
q.to(torch.float8_e4m3fn).view(torch.uint8),
kv[:, NOPE:].contiguous().view(torch.uint8),
],
1,
)
scales = torch.zeros(kv.shape[0], 8, dtype=torch.uint8, device=kv.device)
scales[:, :7] = (exp + 127).clamp(0, 255).to(torch.uint8)
n = cache.untyped_storage().nbytes() - cache.storage_offset()
flat = torch.as_strided(cache, (n,), (1,))
slots = slots.to(kv.device).long()
b, o = slots // block, slots % block
d0 = b * cache.stride(0) + o * DATA
s0 = b * cache.stride(0) + block * DATA + o * 8
ar = torch.arange(DATA, device=kv.device)
flat[(d0[:, None] + ar).reshape(-1)] = data.reshape(-1)
ar = torch.arange(8, device=kv.device)
flat[(s0[:, None] + ar).reshape(-1)] = scales.reshape(-1)
def _normed_kv(n: int, kv_norm, g, device):
x = torch.randn(n, HEAD_DIM, generator=g).to(device)
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True)) * kv_norm.float()
return x.to(torch.bfloat16)
def _decode_step(ratio: int, R: int, T: int, kv_norm, cos_sin, seed: int):
"""R requests x T tokens past random contexts: the step's metadata as
vLLM's builders lay it out, its window rows and (ratio > 0) the compressed
cache filled, a random top-512 a token."""
g = torch.Generator().manual_seed(seed)
dev = kv_norm.device
ctx = [int(c) for c in torch.randint(2000, 40000, (R,), generator=g)]
M = R * T
spans = [
(max(0, c - WINDOW - T) // SWA_BLOCK, (c + T - 1) // SWA_BLOCK) for c in ctx
]
n_swa = 1 + sum(hi - lo + 1 for lo, hi in spans)
perm = (torch.randperm(n_swa - 1, generator=g) + 1).tolist()
bt = torch.zeros(R, max(hi for _, hi in spans) + 1, dtype=torch.int64)
for r, (lo, hi) in enumerate(spans):
for lb in range(lo, hi + 1):
bt[r, lb] = perm.pop()
swa = _paged_cache(n_swa, SWA_BLOCK, dev)
def slot(r, p):
return int(bt[r, p // SWA_BLOCK]) * SWA_BLOCK + p % SWA_BLOCK
pos = [ctx[r] + t for r in range(R) for t in range(T)]
t2r = [r for r in range(R) for _ in range(T)]
swa_idx = torch.full((M, WINDOW), -1, dtype=torch.int32)
swa_len = torch.zeros(M, dtype=torch.int32)
for i in range(M):
ps = range(max(pos[i] - WINDOW + 1, 0), pos[i] + 1)
swa_idx[i, : len(ps)] = torch.tensor([slot(t2r[i], p) for p in ps])
swa_len[i] = len(ps)
hist = [
(r, p) for r in range(R) for p in range(max(0, ctx[r] - WINDOW - T), ctx[r])
]
hp = torch.tensor([p for _, p in hist], device=dev)
kv = _rope(_normed_kv(len(hist), kv_norm, g, dev), hp, cos_sin)
_write_records(swa, SWA_BLOCK, torch.tensor([slot(r, p) for r, p in hist]), kv)
step = dict(
M=M,
positions=torch.tensor(pos, dtype=torch.int64, device=dev),
slot_mapping=torch.tensor(
[slot(t2r[i], pos[i]) for i in range(M)], dtype=torch.int64, device=dev
),
token_to_req=torch.tensor(t2r, dtype=torch.int32, device=dev),
swa_indices=swa_idx.to(dev),
swa_lens=swa_len.to(dev),
swa_cache=swa,
topk=None,
comp_cache=None,
comp_block_table=None,
comp_entries=0,
)
if ratio:
entries = CACHE_BLOCK // ratio
lens = [(c + T) // ratio for c in ctx]
nblk = [-(-n // entries) for n in lens]
perm = (torch.randperm(sum(nblk), generator=g) + 1).tolist()
cbt = torch.zeros(R, max(nblk), dtype=torch.int32)
for r in range(R):
for b in range(nblk[r]):
cbt[r, b] = perm.pop()
comp = _paged_cache(1 + sum(nblk), entries, dev)
for r in range(R):
idx = torch.arange(lens[r])
slots = cbt[r, idx // entries].long() * entries + idx % entries
kv = _rope(
_normed_kv(lens[r], kv_norm, g, dev), (idx * ratio).to(dev), cos_sin
)
_write_records(comp, entries, slots, kv)
topk = torch.full((M, TOPK), -1, dtype=torch.int32)
for i in range(M):
cand = (pos[i] + 1) // ratio
n = min(cand, TOPK)
topk[i, :n] = torch.randperm(cand, generator=g)[:n].to(torch.int32)
step.update(
topk=topk.to(dev),
comp_cache=comp,
comp_block_table=cbt.to(dev),
comp_entries=entries,
)
return step
# ---------------------------------------------------------------- weights
def _mxfp8(n: int, k: int, g, device):
"""A random MXFP8 weight [n, k] e4m3 and its E8M0 32 x 32 block scales."""
w = torch.randn(n, k, generator=g).clamp(-6, 6).to(torch.float8_e4m3fn)
s = torch.randint(118, 123, (n // 32, k // 32), generator=g, dtype=torch.uint8)
return w.to(device), s.to(device)
def _random_moe(moe, gd, gr, inter: int) -> None:
"""Random parameters in the loaded layout, before post-load processing: this
rank's shards (``gd``, a device generator) of the MXFP4 routed experts
(random nibbles, E8M0 scales; the intermediate past ``inter`` zero, as the
loader pads it) and the MXFP8 shared expert (one E8M0 a 32 x 32 block); the
replicated router from ``gr``, seeded alike on every rank."""
u8 = torch.uint8
for lin in (moe.shared_experts.gate_up_proj, moe.shared_experts.down_proj):
n = lin.weight.shape[0]
w = torch.randn(lin.weight.shape, generator=gd, device=gd.device)
lin.weight.data.copy_(w.clamp(-6, 6).to(lin.weight.dtype))
sc = lin.weight_scale.data.view(u8)
rows = -(-n // 32) if sc.shape[0] == n else sc.shape[0]
blocks = torch.randint(
118, 123, (rows, sc.shape[1]), generator=gd, device=gd.device, dtype=u8
)
sc.copy_(blocks.repeat_interleave(32, 0)[:n] if sc.shape[0] == n else blocks)
e = moe.experts.routed_experts
for name, lo, hi in (("weight", 0, 256), ("weight_scale", 118, 122)):
for w in ("w13", "w2"):
t = getattr(e, f"{w}_{name}").data.view(u8)
t.copy_(
torch.randint(lo, hi, t.shape, generator=gd, device=gd.device, dtype=u8)
)
padded = e.w13_weight.shape[1] // 2
if padded > inter:
w13, w2 = e.w13_weight.data.view(u8), e.w2_weight.data.view(u8)
for half in (0, padded):
w13[:, half + inter : half + padded] = 0
w2[..., inter // 2 :] = 0
for t in (moe.gate.weight, moe.gate.e_score_correction_bias):
t.data.copy_(torch.randn(t.shape, generator=gr, device=gr.device) * 0.02)
# ---------------------------------------------------------------- the worker
@ray.remote(num_gpus=1, max_calls=1)
def _mono_numerics(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.worker.workspace import init_workspace_manager
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
# the checkpoint's MXFP4 experts run on AITER's MoE; its flags are read
# at import, so refresh them
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
rocm_aiter_ops.refresh_env_variables()
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
init_workspace_manager(device)
_check_layers(rank, tp_size, device)
def _check_layers(rank: int, tp: int, device) -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.engine.arg_utils import EngineArgs
from vllm.forward_context import set_forward_context
from vllm.model_executor.kernels.linear.mxfp8.rocm_block32_gemm import (
rocm_mxfp8_block32_gemm,
)
from vllm.model_executor.layers.mhc import MHCPreDelayedOp
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.model_loader.utils import process_weights_after_loading
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.deepseek_v41.amd.model import DeepseekV4MoE
from vllm.models.deepseek_v41.amd.mono.layer import scratch_layout
from vllm.models.deepseek_v41.amd.mono.runner import (
AttnWeights,
DSV41MonoLayer,
MonoLayerWeights,
)
from vllm.models.deepseek_v41.amd.rocm import (
compute_global_topk_ragged_indices_and_indptr,
)
from vllm.models.deepseek_v41.common.rope import build_deepseek_v4_rope
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_ragged_indices_from_dense,
rocm_inverse_rope_mxfp8_rows,
)
vc = EngineArgs(
model=MODEL,
tensor_parallel_size=tp,
moe_backend="aiter",
load_format="dummy",
max_model_len=65536,
language_model_only=True,
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
).create_engine_config()
cfg = vc.model_config.hf_config
cfg = getattr(cfg, "text_config", cfg)
# replicated tensors alike on every rank; each rank's own TP shards
g = torch.Generator().manual_seed(1000)
gs = torch.Generator().manual_seed(1001 + rank)
H, G = N_HEADS // tp, O_GROUPS // tp
bf = torch.bfloat16
with set_current_vllm_config(vc), set_default_torch_dtype(bf), device:
moe = DeepseekV4MoE(vc, prefix="model.layers.21.ffn")
mhc = MHCPreDelayedOp()
_random_moe(
moe,
torch.Generator(device=device).manual_seed(2001 + rank),
torch.Generator(device=device).manual_seed(2000),
inter=2304 // tp,
)
process_weights_after_loading(moe, vc.model_config, device)
def f32(*shape, s=1.0, gen=g):
return (torch.randn(*shape, generator=gen) * s).to(device)
mix = (2 + HC) * HC
seams = {
sub: dict(
fn=f32(mix, HC * HIDDEN, s=0.01),
scale=f32(3, s=0.5).abs() + 0.5,
base=f32(mix, s=0.1),
norm=(1 + f32(HIDDEN, s=0.1)).to(bf),
)
for sub in ("attn", "ffn")
}
def seam(sub, residual, x, post, comb, pre):
s = seams[sub]
return mhc(
residual, s["fn"], s["scale"], s["base"], cfg.rms_norm_eps,
cfg.hc_eps, cfg.hc_eps, 2.0, cfg.hc_sinkhorn_iters,
pre_mix=pre, sublayer_out=x, post_layer_mix=post, comb_res_mix=comb,
norm_weight=s["norm"], norm_eps=cfg.rms_norm_eps,
) # fmt: skip
def moe_out(x):
with set_forward_context(None, vc):
ids = torch.zeros(x.shape[0], dtype=torch.int64, device=device)
return moe(x.clone(), ids)
wqkv, wqkv_s = _mxfp8(1792, HIDDEN, g, device)
wq_b, wq_b_s = _mxfp8(H * HEAD_DIM, Q_RANK, gs, device)
wo_a, wo_a_s = _mxfp8(G * O_RANK, H // G * HEAD_DIM, gs, device)
wo_b, wo_b_s = _mxfp8(HIDDEN, G * O_RANK, gs, device)
q_norm, kv_norm = (1 + f32(Q_RANK, s=0.1)).to(bf), (1 + f32(HEAD_DIM, s=0.1)).to(bf)
sink = f32(H, gen=gs)
e, sh = moe.experts.routed_experts, moe.shared_experts
u8 = lambda t: t.view(torch.uint8) # noqa: E731
runner = DSV41MonoLayer(tp, rank, get_tp_group().cpu_group, device)
gw, bias = moe.gate.weight.float(), moe.gate.e_score_correction_bias.float()
def top6(x):
"""The top-6 experts as vLLM routes: sqrt(softplus(logits)) + bias."""
sc = torch.nn.functional.softplus(x.float() @ gw.T).sqrt() + bias
return sc.topk(6, dim=1).indices.sort(1).values
steps = [] # each case's launches, on fixed buffers, for the replay stress
for ratio, R, T in ((1, 1, 6), (2, 8, 6)):
with set_current_vllm_config(VllmConfig()):
rope = build_deepseek_v4_rope(
cfg, head_dim=HEAD_DIM, rope_head_dim=ROPE,
max_position_embeddings=cfg.max_position_embeddings,
compress_ratio=ratio,
) # fmt: skip
cos_sin = rope.cos_sin_cache.to(device=device, dtype=torch.float32)
s = _decode_step(ratio, R, T, kv_norm, cos_sin, seed=7 + ratio)
M = s["M"]
mw = MonoLayerWeights(
attn=AttnWeights(
layer_id=21, wqkv=wqkv, wqkv_scale=wqkv_s, q_norm=q_norm,
kv_norm=kv_norm, wq_b=wq_b, wq_b_scale=wq_b_s, wo_a=wo_a,
wo_a_scale=wo_a_s, wo_b=wo_b, wo_b_scale=wo_b_s, attn_sink=sink,
cos_sin=cos_sin, ratio=ratio,
),
hc_attn_fn=seams["attn"]["fn"], hc_attn_scale=seams["attn"]["scale"],
hc_attn_base=seams["attn"]["base"], attn_norm=seams["attn"]["norm"],
hc_ffn_fn=seams["ffn"]["fn"], hc_ffn_scale=seams["ffn"]["scale"],
hc_ffn_base=seams["ffn"]["base"], ffn_norm=seams["ffn"]["norm"],
gate_w=moe.gate.weight, bias=moe.gate.e_score_correction_bias,
w13=e.w13_weight, w13_s=e.w13_weight_scale, w2=e.w2_weight,
w2_s=e.w2_weight_scale, sgu=sh.gate_up_proj.weight,
sgu_s=u8(sh.gate_up_proj.weight_scale), sw2=sh.down_proj.weight,
sw2_s=u8(sh.down_proj.weight_scale),
) # fmt: skip
# the layer's inputs at the attention seam (the same on every rank)
gi = torch.Generator().manual_seed(11)
res0 = torch.randn(M, HC, HIDDEN, generator=gi)
res0 = (
(res0 * torch.rsqrt(res0.pow(2).mean(-1, keepdim=True))).to(bf).to(device)
)
x0 = (torch.randn(M, HIDDEN, generator=gi) * 0.5).to(bf).to(device)
comb0 = torch.rand(M, HC, HC, generator=gi) + 0.1
for _ in range(20):
comb0 = comb0 / comb0.sum(-1, keepdim=True)
comb0 = comb0 / comb0.sum(-2, keepdim=True)
comb0 = comb0.to(device)
post0 = (torch.sigmoid(torch.randn(M, HC, 1, generator=gi)) * 2).to(device)
pre0 = (torch.sigmoid(torch.randn(M, HC, generator=gi)) + 1e-6).to(device)
# vLLM's decode chain
res, post, comb, xa, attn_pre = seam("attn", res0, x0, post0, comb0, pre0)
swa_ind, swa_ptr = build_ragged_indices_from_dense(
s["swa_indices"].reshape(M, -1), s["swa_lens"]
)
top_ind = top_ptr = None
if ratio:
valid = torch.ones(M, dtype=torch.bool, device=device)
top_ind, top_ptr, _ = compute_global_topk_ragged_indices_and_indptr(
s["topk"], s["token_to_req"], s["comp_block_table"],
s["comp_entries"], valid,
) # fmt: skip
x8, xs = mxfp8_e4m3_quantize(xa)
qr_kv = rocm_mxfp8_block32_gemm(x8, xs, wqkv, wqkv_s, bf)
qr, kv = fused_q_kv_rmsnorm(
qr_kv[:, :Q_RANK], qr_kv[:, Q_RANK:], q_norm, kv_norm, cfg.rms_norm_eps
)
q8, qs = mxfp8_e4m3_quantize(qr)
q = rocm_mxfp8_block32_gemm(q8, qs, wq_b, wq_b_s, bf).view(M, H, HEAD_DIM)
swa2d = s["swa_cache"].view(s["swa_cache"].shape[0], -1)
q_roped = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q, kv, swa2d, s["slot_mapping"], s["positions"], cos_sin, H,
cfg.rms_norm_eps, SWA_BLOCK, False, False, True, False,
) # fmt: skip
attn = torch.empty(M, H, HEAD_DIM, dtype=bf, device=device)
rocm_aiter_ops.triton_sparse_mla_fwd(
q_roped, s["swa_cache"], attn, HEAD_DIM**-0.5, swa_ptr, swa_ind,
attn_sink=sink, extra_kv_buffer=s["comp_cache"],
extra_kv_indptr=top_ptr, extra_kv_indices=top_ind,
) # fmt: skip
a8 = torch.empty(M, H * HEAD_DIM, dtype=torch.float8_e4m3fn, device=device)
a_s = torch.empty(M, H * HEAD_DIM // 32, dtype=torch.uint8, device=device)
rocm_inverse_rope_mxfp8_rows(attn, s["positions"], cos_sin, ROPE, a8, a_s)
z = torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(a8, a_s, wo_a, wo_a_s, G, O_RANK)
z8, zs = mxfp8_e4m3_quantize(z)
part = rocm_mxfp8_block32_gemm(z8, zs, wo_b, wo_b_s, bf).contiguous()
a = tensor_model_parallel_all_reduce(part.clone())
res2, post2, comb2, xf, ffn_pre = seam("ffn", res, a, post, comb, attn_pre)
out = moe_out(xf)
# ---- the whole layer: K1 + K2
o = runner.forward(
mw, x0, res0, post0, comb0, pre0, s["positions"], s["slot_mapping"],
s["swa_cache"], s["swa_indices"], s["swa_lens"], s["token_to_req"],
topk_indices=s["topk"], comp_cache=s["comp_cache"],
comp_block_table=s["comp_block_table"],
) # fmt: skip
off = scratch_layout(M, tp)["normed"][0]
normed = runner.scratch(M)[off : off + M * HIDDEN * 2].view(bf).view(M, HIDDEN)
tag = f"ratio {ratio}, {R} x {T}"
assert _cos(runner.res_mid[:M], res) > 0.99999, tag
for got, want in (
(runner.post_a, post),
(runner.comb_a, comb),
(runner.pre_a, attn_pre),
):
torch.testing.assert_close(
got[:M].view_as(want), want, rtol=1e-4, atol=1e-5
)
same_in = moe_out(normed)
# relative errors about 3x what the kernels show (0.3-0.5%)
assert _cos(o[1], res2) > 0.9999 and _rel(o[1], res2) < 0.01, tag
assert _cos(normed, xf) > 0.999 and _rel(normed, xf) < 0.012, tag
# TODO: back to cos > 0.9999, rel < 0.015 once
# https://github.com/ROCm/aiter/issues/6548 is fixed: aiter's fused-FP8
# stage-1 MoE kernels use another MXFP8 scale rule (1.53% at TP4)
assert _cos(o[0], same_in) > 0.9998 and _rel(o[0], same_in) < 0.02, tag
# per token, where both FFN inputs pick the same experts
same = (top6(xf) == top6(normed)).all(1)
assert same.float().mean() >= 0.5, tag
tok = torch.nn.functional.cosine_similarity(o[0].float(), out.float(), dim=1)
assert tok[same].min() > 0.99, tag
# ---- the FFN launch, from the same unreduced wo_b output
f = runner.ffn(mw, part, res, post, comb, attn_pre)
torch.testing.assert_close(f[1], res2, rtol=2**-8, atol=1e-6)
torch.testing.assert_close(normed, xf, rtol=2**-7, atol=1e-6)
# TODO: back to 0.9999 / 0.015 with the one above (ROCm/aiter#6548)
assert _cos(f[0], out) > 0.9998 and _rel(f[0], out) < 0.02, tag
torch.accelerator.synchronize()
whole, ffn = runner._outs(M, res0), runner._outs(M, res)
args = (s["positions"], s["slot_mapping"], s["swa_cache"], s["swa_indices"])
meta = dict(
topk_indices=s["topk"], comp_cache=s["comp_cache"],
comp_block_table=s["comp_block_table"],
) # fmt: skip
def step(mw=mw, x0=x0, res0=res0, post0=post0, comb0=comb0, pre0=pre0,
args=args, s=s, meta=meta, whole=whole, part=part, res=res,
post=post, comb=comb, attn_pre=attn_pre, ffn=ffn): # fmt: skip
runner.forward(
mw, x0, res0, post0, comb0, pre0, *args, s["swa_lens"],
s["token_to_req"], **meta, outs=whole,
) # fmt: skip
runner.ffn(mw, part, res, post, comb, attn_pre, outs=ffn)
return [*whole, *ffn]
steps.append(step)
# ---- the hand-off protocol under back-to-back launches: both step widths
# alternating in one graph, every replay bit for bit the first's
outs = [t for step in steps for t in step()]
torch.accelerator.synchronize()
ref = [t.clone() for t in outs]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for step in steps:
step()
for i in range(REPLAYS):
graph.replay()
if i % 50 == 49 or i == REPLAYS - 1:
torch.accelerator.synchronize()
bad = [
j for j, (t, r) in enumerate(zip(outs, ref)) if not torch.equal(t, r)
]
assert not bad, f"replay {i}: outputs {bad} differ from the first run"
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
@pytest.mark.parametrize("tp", [2, 4])
def test_mono_decode_layer_matches_vllm(monkeypatch: pytest.MonkeyPatch, tp: int):
if torch.accelerator.device_count() < tp:
pytest.skip(f"needs {tp} GPUs")
pytest.importorskip("flydsl")
multi_process_parallel(monkeypatch, tp, 1, _mono_numerics)
+2 -4
View File
@@ -14,10 +14,8 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
from vllm.v1.core.kv_cache_utils import (
generate_scheduler_kv_cache_config,
get_kv_cache_configs,
)
from vllm.v1.core.kv_cache_planning import get_kv_cache_configs
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
from vllm.v1.engine.core import EngineCore as V1EngineCore
from ..utils import (
@@ -12,9 +12,13 @@ import gc
import weakref
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.models.interfaces import _language_model_by_module
from vllm.model_executor.models.interfaces import (
SupportsMultiModal,
_language_model_by_module,
)
pytestmark = pytest.mark.cpu_test
@@ -57,3 +61,42 @@ def test_cached_entry_is_dropped_with_its_key():
gc.collect()
assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module)
@pytest.mark.parametrize("marked", [False, True])
@pytest.mark.parametrize("text_model", ["clip", "siglip"])
def test_text_tower_without_loader_hook_is_found(text_model, marked, caplog):
from vllm.model_executor.models.clip import CLIPTextTransformer
from vllm.model_executor.models.siglip import SiglipTextTransformer
cls = CLIPTextTransformer if text_model == "clip" else SiglipTextTransformer
# Only token embeddings are needed to exercise text tower discovery.
tower = cls.__new__(cls)
nn.Module.__init__(tower)
tower.embeddings = nn.Module()
tower.embeddings.token_embedding = nn.Embedding(4, 2)
model = nn.Module()
model.text_model = tower
model._language_model_names = ["text_model"] if marked else []
caplog.clear()
language_model = SupportsMultiModal.get_language_model(model)
assert language_model is tower
input_ids = torch.tensor([0, 1])
torch.testing.assert_close(
language_model.embed_input_ids(input_ids),
tower.embeddings.token_embedding.weight[:2],
)
assert not caplog.records
@pytest.mark.parametrize("marked", [False, True])
@pytest.mark.parametrize("method", ["embed_input_ids", "forward"])
def test_language_model_discovery_rejects_noncallable_methods(method, marked):
model = _MultiModalModel()
model._language_model_names = ["language_model"] if marked else []
setattr(model.language_model, method, None)
with pytest.raises(NotImplementedError, match="No language model found"):
SupportsMultiModal.get_language_model(model)
+16 -3
View File
@@ -113,9 +113,6 @@ def test_registry_model_property(model_arch, is_mm, score_type):
@pytest.mark.parametrize(
"model_arch,is_pp",
[
# TODO(woosuk): Re-enable this once the MLP Speculator is supported
# in V1.
# ("MLPSpeculatorPreTrainedModel", False),
("DeepseekV2ForCausalLM", True),
("Qwen2VLForConditionalGeneration", True),
],
@@ -127,6 +124,22 @@ def test_registry_is_pp(model_arch, is_pp):
assert model_info.supports_pp is is_pp
@pytest.mark.parametrize(
"model_arch,supported",
[
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
("Qwen4ExpMTP", not current_platform.is_rocm()),
("LlamaForCausalLM", False),
],
)
def test_registry_supports_hc_sp(model_arch, supported):
"""HC SP is opt-in for each model implementation, including MTP."""
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
assert model_info is not None
assert model_info.supports_hc_sp is supported
@create_new_process_for_each_test()
@pytest.mark.parametrize(
"model_arch",
+248 -13
View File
@@ -29,6 +29,42 @@ class TopKRouter(nn.Module):
return logits, value, index
class SelectionScoredRouter(TopKRouter):
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight).float()
value, index = torch.topk(logits, self.top_k, dim=-1)
value = (
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
).type_as(hidden_states)
return logits, value, index
class EpsilonRenormRouter(TopKRouter):
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits)
value, index = torch.topk(scores, self.top_k, dim=-1)
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
value /= denominator
return logits, value, index
class UntraceableRouter(TopKRouter):
"""Scores and selects, then loses the trace before the renormalization."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
if value.sum() > 0: # data-dependent branch: tracing stops here
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class ScaledRouter(TopKRouter):
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
@@ -108,6 +144,16 @@ class DisconnectedRouter(TopKRouter):
return logits, value, index
class ExtraInputRouter(TopKRouter):
"""Router taking a block-owned correction bias as a second input (HYV3)."""
def forward(self, hidden_states, bias=0.0):
logits = F.linear(hidden_states, self.weight)
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
value = logits.sigmoid().gather(1, index)
return logits, value, index
class MoEExperts(nn.Module):
"""Packed experts (3D weights); only its name (`experts`) matters here."""
@@ -130,17 +176,19 @@ class MoEBlock(nn.Module):
def _shared(self, x, logits):
"""The term added to the experts' output (none for a plain block)."""
return 0
return None
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
logits, weights, index = self.gate(x)
out = self.experts(x, index, weights) + self._shared(x, logits)
out = self.experts(x, index, weights)
if (shared := self._shared(x, logits)) is not None:
out = out + shared
return out.reshape(hidden_states.shape)
class MoEBlockNoShared(MoEBlock):
"""No shared-expert child but a gate-derived add -> trace skipped, still fuses."""
"""No shared-expert child but a gate-derived add -> declined, not misread."""
def _shared(self, x, logits):
return logits.sum()
@@ -257,6 +305,115 @@ class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
self.extra = BufferScale()
class MoEBlockExpertsKeywords(MoEBlock):
"""Passes the experts' routing by keyword -> still fuses."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, weights=weights, index=index)
class ScaledExperts(MoEExperts):
def forward(self, hidden_states, index, weights, scale=1.0):
return hidden_states * scale
class MoEBlockExpertsExtraArg(MoEBlock):
"""Passes the experts an extra argument the fused forward would drop."""
def __init__(self):
super().__init__()
self.experts = ScaledExperts()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights, scale=2.0)
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
def forward(self, hidden_states):
_, weights, index = self.gate(hidden_states)
x = hidden_states.view(-1, hidden_states.shape[-1])
out = self.experts(x, index, weights).view(*hidden_states.shape)
return out + self.shared_expert(hidden_states)
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
def forward(self, hidden_states):
return super().forward(hidden_states) / 2
class MoEBlockScaled(MoEBlockSharedNoGate):
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
"""An unrecognized router with a scaled experts' output."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
"""An unrecognized router with a shared expert."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockGateExtraInput(MoEBlock):
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
def __init__(self):
super().__init__(ExtraInputRouter)
self.register_buffer("e_score_correction_bias", torch.zeros(8))
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x, self.e_score_correction_bias)
return self.experts(x, index, weights)
class MoEBlockCastCombine(MoEBlockSharedNoGate):
"""Combines the outputs in fp32 (HYV3)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
return out.to(x.dtype)
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
def __init__(self):
super().__init__()
self.fc1_latent_proj = nn.Identity()
self.fc2_latent_proj = nn.Identity()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.fc2_latent_proj(
self.experts(self.fc1_latent_proj(x), index, weights)
)
return out + self.shared_expert(x)
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_detects_router(sigmoid):
with torch.device("meta"):
@@ -268,6 +425,19 @@ def test_moe_fuser_detects_router(sigmoid):
assert fuser.shared_name is None and fuser.shared_gate_name is None
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
"""Scoring the selection rather than all logits still allows the fuser to match."""
with torch.device("meta"):
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.renormalize is (not sigmoid)
assert fuser.router_dtype == torch.float32
def test_moe_fuser_matches_scaled_router():
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
with torch.device("meta"):
@@ -275,6 +445,27 @@ def test_moe_fuser_matches_scaled_router():
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
@pytest.mark.parametrize(
"router_cls,expected",
[
(TopKRouter, True), # divides by the sum of the selection
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
(ScaledRouter, False), # scales the weights instead of renormalizing
(UntraceableRouter, None), # trace stops before it could say
],
)
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
"""The router's graph, not the config, decides whether vLLM renormalizes: it
is the code being replaced, and it already bakes in `norm_topk_prob` because
the flag is a Python bool that fx resolves at trace time. Absence of the
division only means "no" on a complete trace."""
with torch.device("meta"):
block = MoEBlock(router_cls)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.renormalize is expected
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
@@ -315,16 +506,6 @@ def test_moe_fuser_detects_shared_experts():
assert fuser.shared_gate_name == "shared_expert_gate"
def test_moe_fuser_skips_shared_detection_without_extra_children():
"""With only experts and gate, shared-expert detection (and its block trace)
is skipped, so a gate-derived add is not misread as a shared expert."""
with torch.device("meta"):
block = MoEBlockNoShared()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name is None and fuser.shared_gate_name is None
def test_moe_fuser_shared_without_gate():
with torch.device("meta"):
block = MoEBlockSharedNoGate()
@@ -352,6 +533,10 @@ def test_moe_fuser_detects_non_glu_shared_expert():
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
MoEBlockNoShared, # gate-derived term added to the experts' output
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
MoEBlockGateExtraInput, # gate input the fused forward drops
],
)
def test_moe_fuser_declines_unsupported(block_cls):
@@ -374,3 +559,53 @@ def test_moe_fuser_router_requires_connected_dataflow():
with torch.device("meta"):
block = MoEBlock(DisconnectedRouter)
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_unrecognized_router_moves_shared_expert():
"""Without vLLM routing, the shared expert still moves to vLLM."""
with torch.device("meta"):
block = MoEBlockSharedBiasedRouter()
fuser = MoEBlockFuser.match(block, "experts")
assert fuser.scoring_func is None
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize(
("block_cls", "scoring_func"),
[
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
(MoEBlock, "softmax"), # no shared output to divide
],
)
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
assert fuser.scoring_func == scoring_func
def test_moe_fuser_scaled_output_needs_vllm_routing():
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
with torch.device("meta"):
block = MoEBlockScaledBiasedRouter()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_matches_experts_called_by_keyword():
with torch.device("meta"):
block = MoEBlockExpertsKeywords()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_scaled_output_with_vllm_routing():
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
with torch.device("meta"):
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
assert fuser.scoring_func == "softmax"
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts")
assert fuser.shared_name == "shared_expert"
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for skipping dynamic rope's device sync where it can never rescale."""
from types import SimpleNamespace
import pytest
import torch
from transformers import HunYuanDenseV1Config, LlamaConfig
from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import (
HunYuanDenseV1RotaryEmbedding,
)
from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding
from vllm.model_executor.models.transformers.utils import (
can_enable_torch_compile,
disable_noop_dynamic_rope,
)
MAX_POSITION_EMBEDDINGS = 128
CONFIG_KWARGS = dict(
hidden_size=64,
num_attention_heads=4,
num_hidden_layers=1,
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
)
DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4}
def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace:
model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len)
return SimpleNamespace(model_config=model_config)
@pytest.mark.parametrize(
"rope_parameters,max_model_len,compilable",
[
(DYNAMIC, MAX_POSITION_EMBEDDINGS, True),
(DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False),
# The longrope layer type still syncs
(
{"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}},
MAX_POSITION_EMBEDDINGS,
False,
),
],
)
def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable):
"""Dynamic rope only blocks compilation when it can still rescale."""
config = SimpleNamespace(
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
rope_parameters=rope_parameters,
)
config.get_text_config = lambda: config
vllm_config = make_vllm_config(config, max_model_len)
assert can_enable_torch_compile(vllm_config) is compilable
@pytest.mark.parametrize(
"config,rotary_cls",
[
(LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding),
# Hunyuan's dynamic init also applies `alpha`, which must be kept
(
HunYuanDenseV1Config(
**CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3}
),
HunYuanDenseV1RotaryEmbedding,
),
],
)
def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls):
"""The update no longer syncs and the frequencies are unchanged."""
inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :]
rotary_emb = rotary_cls(config)
expected = rotary_emb(*inputs)
vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS)
disable_noop_dynamic_rope(rotary_emb, vllm_config)
def no_sync(*args, **kwargs):
raise AssertionError("dynamic rope update still runs")
monkeypatch.setattr(torch, "max", no_sync)
actual = rotary_emb(*inputs)
assert all(torch.equal(e, a) for e, a in zip(expected, actual))