Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
+325 -13
View File
@@ -1,11 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm fused KDA decode kernel must match the Triton chain it replaces.
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
The fused kernel folds the packed causal conv1d update, the gated delta-rule
recurrence and the gated output RMSNorm into one launch, and updates both the
conv state and the recurrent state in place. Every one of those outputs is
compared against the three-kernel path the AMD layer falls back to.
Two fused paths fold the packed causal conv1d update, the gated delta-rule
recurrence and the gated output RMSNorm into one launch, updating the conv
state and the recurrent state in place:
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
Every output of each is compared against the unfused path the AMD layer falls
back to, with the argument layout the layer itself uses.
"""
import pytest
@@ -14,7 +20,7 @@ import torch
from vllm.platforms import current_platform
def _on_supported_arch() -> bool:
def _hip_kernel_arch() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx942, on_gfx950
@@ -22,10 +28,26 @@ def _on_supported_arch() -> bool:
return on_gfx950() or on_gfx942()
def _aiter_gluon_kda_available() -> bool:
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
pytestmark = pytest.mark.skipif(
not _on_supported_arch(),
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
)
requires_hip_kernel_arch = pytest.mark.skipif(
not _hip_kernel_arch(),
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
)
requires_aiter_gluon_kda = pytest.mark.skipif(
not _aiter_gluon_kda_available(),
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
)
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
HEAD_DIM = 128
@@ -41,7 +63,11 @@ def _requires_kernel() -> None:
class KdaDecodeInputs:
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout."""
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
``num_spec`` > 0 sizes the conv state for speculative decoding
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
"""
def __init__(
self,
@@ -49,6 +75,7 @@ class KdaDecodeInputs:
num_heads: int,
num_slots: int,
seed: int = 0,
num_spec: int = 0,
) -> None:
torch.manual_seed(seed)
device = "cuda"
@@ -70,9 +97,15 @@ class KdaDecodeInputs:
for i in range(3)
]
)
# SD cache layout: [slots, width - 1, 3 * dim]; the layer transposes it.
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
self.conv_state = (
torch.randn(num_slots, CONV_WIDTH - 1, 3 * dim, device=device, dtype=DTYPE)
torch.randn(
num_slots,
CONV_WIDTH - 1 + num_spec,
3 * dim,
device=device,
dtype=DTYPE,
)
* 0.5
)
self.recurrent_state = (
@@ -112,6 +145,12 @@ class KdaDecodeInputs:
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
return state.transpose(-1, -2)
def empty_out(self) -> torch.Tensor:
num_tokens = self.mixed_qkv.shape[0]
return torch.empty(
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
)
def _gated_rmsnorm(
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
@@ -196,6 +235,7 @@ def _run_fused(
return out, conv_state, recurrent_state
@requires_hip_kernel_arch
@pytest.mark.parametrize("num_heads", [12, 24, 96])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
@torch.inference_mode()
@@ -214,6 +254,7 @@ def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
"""Only the slots named by state_indices may be written."""
@@ -234,6 +275,7 @@ def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
)
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_without_output_norm() -> None:
"""Omitting the gate/norm pair returns the raw recurrent output."""
@@ -265,6 +307,7 @@ def test_fused_kda_decode_without_output_norm() -> None:
assert torch.isfinite(out).all()
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_skips_null_block_padding() -> None:
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
@@ -291,6 +334,7 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_hip_kernel_arch
def test_decode_weights_staged_after_loading(dist_init) -> None:
"""The fused-decode weight copies must not depend on the weight loaders.
@@ -323,11 +367,15 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
config, vllm_config, prefix="model.layers.0.self_attn"
)
assert layer.decode_conv1d_weight is not None
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
# kernel gets an fp32 norm copy.
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
with torch.no_grad():
for param in layer.parameters():
param.normal_()
layer.decode_conv1d_weight.fill_(float("nan"))
layer.decode_norm_weight.fill_(float("nan"))
if layer.decode_norm_weight is not None:
layer.decode_norm_weight.fill_(float("nan"))
process_weights_after_loading(
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
@@ -341,7 +389,271 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
torch.testing.assert_close(
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
)
if layer.decode_norm_weight is not None:
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
assert not layer.decode_conv1d_weight.requires_grad
# --------------------------------------------------------------------------
# non-spec decode: fused_recurrent_kda_packed_decode
# --------------------------------------------------------------------------
def _run_aiter_packed_decode(
inp: KdaDecodeInputs,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
from vllm._aiter_ops import rocm_aiter_ops
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
num_tokens = inp.mixed_qkv.shape[0]
core_attn_out = inp.empty_out()
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
mixed_qkv=inp.mixed_qkv,
g=inp.g1[0],
beta=inp.beta[0],
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
ssm_state_indices=inp.state_indices[:num_tokens],
conv_state=inp.conv_state_view(conv_state),
conv_weight=inp.decode_conv1d_weight,
out_gate=inp.g2[:num_tokens],
norm_weight=inp.norm_weight_bf16,
norm_eps=NORM_EPS,
out=core_attn_out[0, :num_tokens],
)
return core_attn_out, conv_state, recurrent_state
@requires_aiter_gluon_kda
@pytest.mark.parametrize("num_heads", [12, 24])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
@torch.inference_mode()
def test_aiter_packed_decode_matches_triton_chain(
num_heads: int, num_tokens: int
) -> None:
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
# The fused kernel keeps the recurrent output in fp32 through the norm,
# while the Triton chain rounds it to BF16 in between.
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
# The conv state is a pure shift-and-append: it must be bit-exact.
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_aiter_gluon_kda
@torch.inference_mode()
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
"""Only the slots named by the state indices may be written."""
num_tokens, num_heads, num_slots = 4, 12, 9
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
touched = set(inp.state_indices.tolist())
untouched = [slot for slot in range(num_slots) if slot not in touched]
assert untouched, "test needs at least one unused slot"
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
)
@requires_aiter_gluon_kda
@torch.inference_mode()
def test_aiter_packed_decode_skips_null_block_padding() -> None:
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
must do the same rather than read-modify-write slot 0 once per padded row.
"""
num_real, num_padded, num_heads = 3, 5, 12
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
inp.state_indices[num_real:] = 0
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
torch.testing.assert_close(
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
)
assert torch.isfinite(actual_out).all()
# Slot 0 must be untouched by both paths.
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
# --------------------------------------------------------------------------
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
# --------------------------------------------------------------------------
class SpecBatch:
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
per-token slot table the GDN metadata hands the layer.
"""
def __init__(
self,
seq_lens: list[int],
num_accepted: list[int],
num_heads: int,
num_spec: int,
seed: int = 0,
) -> None:
assert len(seq_lens) == len(num_accepted)
assert max(seq_lens) <= num_spec + 1
n_seqs = len(seq_lens)
slots_per_seq = num_spec + 1
num_slots = n_seqs * slots_per_seq + 2
self.inp = KdaDecodeInputs(
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
)
# Distinct slots per (request, token), shuffled, none at slot 0.
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
self.query_start_loc = torch.tensor(
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
dtype=torch.int32,
device="cuda",
)
self.num_accepted_tokens = torch.tensor(
num_accepted, dtype=torch.int32, device="cuda"
)
self.max_query_len = slots_per_seq
def _run_spec_triton_chain(
sb: SpecBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
inp = sb.inp
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
conv_out = causal_conv1d_update(
inp.mixed_qkv,
inp.conv_state_view(conv_state),
inp.conv_weights,
None,
activation="silu",
conv_state_indices=sb.state_indices[:, 0],
num_accepted_tokens=sb.num_accepted_tokens,
query_start_loc=sb.query_start_loc,
max_query_len=sb.max_query_len,
validate_data=False,
out=torch.empty_like(inp.mixed_qkv),
)
q, k, v = (
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
)
core_attn_out, _ = fused_recurrent_kda(
q=q,
k=k,
v=v,
raw_g=inp.g1,
raw_beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
cu_seqlens=sb.query_start_loc,
ssm_state_indices=sb.state_indices,
num_accepted_tokens=sb.num_accepted_tokens,
)
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
return out, conv_state, recurrent_state
def _run_aiter_spec(
sb: SpecBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
from vllm._aiter_ops import rocm_aiter_ops
inp = sb.inp
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
q, k, v = (
x.view(1, -1, inp.num_heads, HEAD_DIM)
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
)
n_spec = q.shape[1]
core_attn_out = inp.empty_out()
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
q=q,
k=k,
v=v,
g=inp.g1,
beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
cu_seqlens=sb.query_start_loc,
ssm_state_indices=sb.state_indices,
num_accepted_tokens=sb.num_accepted_tokens,
conv_state=inp.conv_state_view(conv_state),
conv_weight=inp.decode_conv1d_weight,
out_gate=inp.g2[:n_spec].unsqueeze(0),
norm_weight=inp.norm_weight_bf16,
norm_eps=NORM_EPS,
out=core_attn_out[:, :n_spec],
)
return core_attn_out, conv_state, recurrent_state
@requires_aiter_gluon_kda
@pytest.mark.parametrize("num_heads", [12, 24])
@pytest.mark.parametrize(
"num_spec, seq_lens, num_accepted",
[
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
(6, [7, 3], [1, 4]), # longer window, two requests
],
)
@torch.inference_mode()
def test_aiter_spec_decode_matches_triton_chain(
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
) -> None:
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
# Nothing outside the slot table may change.
touched = set(sb.state_indices.flatten().tolist())
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
assert untouched
torch.testing.assert_close(
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
@@ -75,6 +75,9 @@ AITER_MODEL_LIST = [
"facebook/opt-125m", # opt
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
pytest.param(
"facebook/opt-350m", # opt with projected token embeddings
),
pytest.param(
"microsoft/phi-2", # phi
marks=[pytest.mark.core_model, pytest.mark.slow_test],
@@ -70,6 +70,10 @@ def gemma3_chat_template(content: str) -> str:
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
def deepseek_vl2_chat_template(content: str) -> str:
return f"<|User|>: {content}\n\n<|Assistant|>:"
def ernie45_vl_chat_template(content: str) -> str:
return (
f"<|begin_of_sentence|>User: {content}"
@@ -255,7 +259,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
vllm_runner_kwargs={
"load_format": "dummy",
"attention_backend": (
"ROCM_FLASHMLA_SPARSE_DSV4"
"ROCM_FLASHMLA_SPARSE_DSV41"
if current_platform.is_rocm()
else "FLASHMLA_SPARSE_DSV41"
),
@@ -356,6 +360,15 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
},
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
),
"deepseek_vl2": VitCudagraphTestConfig(
model="deepseek-ai/deepseek-vl2-tiny",
modalities=["image"],
image_prompt=deepseek_vl2_chat_template("<image>\nWhat is in this image?"),
vllm_runner_kwargs={
"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]},
},
marks=[pytest.mark.core_model],
),
"gemma4": VitCudagraphTestConfig(
model="google/gemma-4-E2B-it",
image_prompt=(
+5 -3
View File
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
The runtime reads a question's options from the 255 code logits of its readout
at the last prompt token. vLLM's classify head must give the same code logits
for the same prompt tokens, and for an image prompt that vLLM processes itself.
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
weight file without an index, next to the readout file.
"""
import pytest
@@ -14,7 +16,7 @@ from transformers import AutoModel
from vllm.assets.image import ImageAsset
from vllm.distributed import cleanup_dist_env_and_memory
MODEL = "vllm-sr/d3-edge"
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
STATE = (
"The order arrived damaged yesterday. The customer has a receipt and asks "
"for a replacement today."
@@ -47,7 +49,7 @@ def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
return torch.softmax(logits[:count].float(), dim=-1)
@pytest.mark.parametrize("model", [MODEL])
@pytest.mark.parametrize("model", MODELS)
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
runtime = reference.runtime
+14 -11
View File
@@ -70,20 +70,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
),
)
with (
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
pytest.raises(ValueError, match="MoE SP requires PP=1"),
):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
@spawn_new_process_for_each_test
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
@pytest.mark.parametrize("use_sp", [False, True])
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
backend: str, use_sp: bool
) -> None:
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
model.use_hc_sequence_parallel = use_sp
model.use_sequence_parallel = False
torch.nn.Module.__init__(model)
@@ -98,7 +99,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
def decoder(**kwargs):
"""Check that the embedding residual shares the hidden state's token slice."""
hidden = kwargs["hidden_states"]
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
if backend == "nvidia":
prev_block_output = kwargs["prev_block_output"]
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
model.layers = [decoder]
@@ -120,9 +123,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
with (
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
+53 -8
View File
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
if not current_platform.is_cuda_alike():
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
from tests.utils import multi_gpu_test
from vllm.config import (
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
from vllm.utils.network_utils import get_open_port
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
from .test_ple import _ConvBatchCase, _make_conv_metadata
if current_platform.is_rocm():
from vllm.models.qwen4_exp.amd import model as qwen4_model
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
else:
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
class _TPAttentionProjection(nn.Module):
"""Keep real TP communication without setting up an attention KV cache."""
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
set_current_vllm_config(config),
set_forward_context({layer.ple.prefix: metadata}, config),
):
layer.ple.start_prefetch(ids, qsl, context)
if hasattr(layer.ple, "start_prefetch"):
layer.ple.start_prefetch(ids, qsl, context)
output = layer(
*inputs,
positions,
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
ngram_context=context,
)
if config.parallel_config.enable_hc_sp:
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
# ROCm all-gather requires contiguous inputs.
output = tuple(
sp_all_gather(x.contiguous())[:num_tokens] for x in output
)
# A later MoE call may reuse workspace-backed output storage.
outputs.append(tuple(x.clone() for x in output))
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
query_start_loc=None,
ngram_context=None,
)
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
def _check_compiled_sp_ops() -> None:
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
sp_reduce_scatter,
sp_shard,
)
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
return gathered, reduced
# vLLM drops Dynamo guards, so one trace must serve every token count.
compiled = torch.compile(
collectives,
fullgraph=True,
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
)
traced = torch.randn(8, 16)
torch._dynamo.mark_dynamic(traced, 0)
compiled(traced)
for num_tokens in (7, 3, 1):
torch.manual_seed(num_tokens)
x = torch.randn(num_tokens, 16)
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
"""Initialize rank-local runtime state and run one equivalence check."""
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
torch.set_default_dtype(torch.bfloat16)
if moe_sp:
_check_moe_sp(rank, config)
elif compiled_ops:
_check_compiled_sp_ops()
else:
_check_decoder_ple(rank)
finally:
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
@multi_gpu_test(num_gpus=2)
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
"""Padding in compiled SP collectives must not specialize on the traced size."""
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
+9 -2
View File
@@ -462,7 +462,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"),
"OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
"OPTForCausalLM": _HfExamplesInfo(
"facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
"facebook/opt-125m",
{"350m": "facebook/opt-350m", "1b": "facebook/opt-iml-max-1.3b"},
),
"OrionForCausalLM": _HfExamplesInfo(
"OrionStarAI/Orion-14B-Chat", trust_remote_code=True
@@ -731,7 +732,13 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
),
"D3ForSequenceClassification": _HfExamplesInfo(
"vllm-sr/d3-edge",
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
extras={
"d3": "vllm-sr/d3",
"d3-flash": "vllm-sr/d3-flash",
"d3-mini": "vllm-sr/d3-mini",
"d3-nano": "vllm-sr/d3-nano",
"d3-lite": "vllm-sr/d3-lite",
},
max_model_len=4096,
),
"GPT2ForSequenceClassification": _HfExamplesInfo(
+66
View File
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.models.deepseek_v41.nvidia import model as model_module
@pytest.mark.parametrize(
"pcp_size,use_ep", [(1, False), (2, True), (2, False), (4, False)]
)
@pytest.mark.parametrize("use_sp", [False, True])
def test_pcp_gathers_routing_ids_once_after_local_input_preparation(
pcp_size, use_ep, use_sp, monkeypatch
):
"""Embedding keeps local IDs; all decoder layers receive aligned routing IDs."""
gather = Mock(side_effect=lambda ids, dim: torch.cat([ids] * pcp_size, dim=dim))
monkeypatch.setattr(
model_module, "get_pcp_group", lambda: SimpleNamespace(all_gather=gather)
)
monkeypatch.setattr(
model_module,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(model_module, "is_forward_context_available", lambda: False)
monkeypatch.setattr(model_module, "sp_shard", lambda tensor: tensor[::2])
monkeypatch.setattr(
model_module, "sp_all_gather", lambda tensor: tensor.repeat_interleave(2, dim=0)
)
monkeypatch.setattr(model_module, "hc_collapse_triton", lambda hidden, mix: hidden)
embed = Mock(side_effect=lambda ids: ids[:, None].float())
run_layers = Mock(
side_effect=lambda layer_ids, hidden, *args: (hidden, None, None, None, hidden)
)
model = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=pcp_size, enable_expert_parallel=use_ep
),
use_native_mega_moe=False,
use_sequence_parallel=use_sp,
engram_hash=None,
embed_input_ids=embed,
start_layer=0,
decoder_replay_start=2,
decoder_replay_layers=None,
aux_hidden_state_layers=(),
_mtp_hidden_buffer=None,
_run_layers=run_layers,
_collapse=lambda hidden, *args: hidden,
norm=lambda hidden: hidden,
)
for offset in (0, 10):
local_ids = torch.tensor([3, 7, 11, 15]) + offset
model_module.DeepseekV4Model.forward(model, local_ids, torch.arange(4), None)
assert embed.call_args.args[0] is local_ids
routing_ids = local_ids[::2] if use_sp else local_ids
needs_gather = pcp_size > 1 and not use_ep
if needs_gather:
torch.testing.assert_close(gather.call_args.args[0], routing_ids)
routing_ids = routing_ids.repeat(pcp_size)
torch.testing.assert_close(run_layers.call_args.args[3], routing_ids)
assert gather.call_count == 2 * int(needs_gather)
+51 -1
View File
@@ -4,7 +4,7 @@
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
@@ -16,6 +16,9 @@ from vllm.forward_context import (
ForwardContext,
override_forward_context,
)
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
@@ -63,6 +66,7 @@ def state(monkeypatch):
cfg.scheduler_config.max_num_seqs = 16
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.aux_output_config.enable_return_routed_experts = False
cfg.compilation_config.fast_moe_cold_start = False
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = DecoderReplayLayers(WINDOW, MagicMock(), MagicMock(), set(), "swa_first")
@@ -172,6 +176,31 @@ def test_replay_batch_keeps_each_request_window(state):
assert context.dp_metadata is None and context.is_padding is None
def test_replay_context_restores_tp_routing_rows(state):
"""Capture uses the compact replay batch's size, not the outer batch's."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
replay, _ = _prepare(state, batch, CUDAGraphMode.NONE)
context = replay.forward_context
assert context.is_padding is None and context.dp_metadata is None
count = len(REPLAY_ROWS)
assert context.batch_descriptor.num_tokens == count
capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
capturer.dp_rank, capturer.tp_size = 0, 2
capturer.device_buffer = torch.full((401, 1, 2), -1, device=DEVICE)
expected = torch.tensor(REPLAY_ROWS, device=DEVICE)[:, None].expand(-1, 2)
gathered = torch.cat((expected, torch.full((1, 2), -1, device=DEVICE)))
with (
override_forward_context(context),
patch(
"vllm.model_executor.layers.fused_moe.routed_experts_capturer.get_tp_group"
) as group,
):
group.return_value.all_gather.return_value = gathered
capturer.capture(0, gathered[: (count + 1) // 2])
torch.testing.assert_close(capturer.device_buffer[:count, 0], expected)
assert (capturer.device_buffer[count:] == -1).all()
def test_replay_batch_keeps_device_decode_boundaries(state):
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
@@ -195,6 +224,27 @@ def test_prompt_logprobs_requests_keep_their_rows(state):
assert build.batch.max_query_len == 300
@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE])
def test_aux_output_requests_keep_every_routing_row(state, cg_mode):
"""R3 needs every prefill row even when final logits only need the window."""
state.vllm_config.aux_output_config.enable_return_routed_experts = True
for index in range(3):
state.add_request(
index,
SimpleNamespace(
replay_start=0,
sampling_params=SimpleNamespace(prompt_logprobs=None),
),
)
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
state.decoder_replay_layers.trim_threshold = WINDOW + 1
replay, _ = _prepare(state, batch, cg_mode)
if cg_mode == CUDAGraphMode.NONE:
assert replay is None
else:
torch.testing.assert_close(replay.rows, torch.arange(401, device=DEVICE))
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
"""Adaptive verification bounds the decodes' device-side query lengths
above their CPU lengths; the replay batch keeps that bound."""
+12 -726
View File
@@ -8,10 +8,8 @@ import torch
from transformers import DeepseekV4Config
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
DeepGemmSm100MegaMoEBackend,
)
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
bind_routed_experts_capturer,
DeepGemmMegaMoEExperts,
MegaMoEActivation,
)
from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
@@ -20,17 +18,14 @@ from vllm.model_executor.models.interfaces import (
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4ForCausalLM,
DeepseekV4MegaMoEExperts,
DeepseekV4MoE,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.torch_utils import set_default_torch_dtype
pytestmark = pytest.mark.skipif(
@@ -40,11 +35,12 @@ pytestmark = pytest.mark.skipif(
@pytest.fixture(autouse=True)
def sm100_mega_moe_backend(monkeypatch):
def sm100_mega_moe_layout(monkeypatch):
"""Build the SM100 kernel layout on any GPU; kernels are gated per test."""
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.get_deep_gemm_mega_moe_backend",
lambda *args: DeepGemmSm100MegaMoEBackend(),
"vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe."
"check_deep_gemm_mega_moe_supported",
lambda *args: None,
)
@@ -125,7 +121,8 @@ def test_deepseek_v41_moe_routes_without_hash_table(
)
assert moe.gate.tid2eid is None
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
assert isinstance(moe.experts, DeepGemmMegaMoEExperts)
assert moe.experts.activation == MegaMoEActivation("swiglu", clamp=10.0)
assert moe.experts.w13_weight.shape == (num_experts, 256, 128)
assert moe.experts.top_k == top_k
assert (config.n_routed_experts, config.num_experts_per_tok) == (384, 6)
@@ -151,10 +148,9 @@ def test_deepseek_v41_moe_routes_without_hash_table(
routed = {}
def check_routing(x, weights, ids, *, activation_clamp):
def check_routing(x, weights, ids, shared_experts):
routed["ids"], routed["weights"] = ids, weights
assert activation_clamp == config.swiglu_limit
return x.clone()
return x + shared_experts(x)
monkeypatch.setattr(moe.experts, "forward", check_routing)
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
@@ -280,10 +276,10 @@ def test_deepseek_v4_mega_gate_hash_routing_correctness(
dim=-1, keepdim=True
)
def check_routing(x, weights, ids, *, activation_clamp):
def check_routing(x, weights, ids, shared_experts):
torch.testing.assert_close(ids, expected_ids)
torch.testing.assert_close(weights, expected_weights, rtol=1e-3, atol=1e-4)
return x.clone()
return x + shared_experts(x)
monkeypatch.setattr(moe.experts, "forward", check_routing)
monkeypatch.setattr(moe.shared_experts, "forward", torch.zeros_like)
@@ -308,634 +304,6 @@ def test_deepseek_v4_mega_moe_expert_mapping():
]
@pytest.mark.parametrize("use_kimi", [False, True])
def test_deep_gemm_mega_moe_capture_precedes_eplb(monkeypatch, use_kimi):
experts_cls = DeepseekV4MegaMoEExperts
if use_kimi:
from vllm.models.kimi_k3.nvidia.model import KimiK3MegaMoEExperts
experts_cls = KimiK3MegaMoEExperts
experts = experts_cls.__new__(experts_cls)
torch.nn.Module.__init__(experts)
if use_kimi:
experts.synchronize_first_launch = lambda: None
experts.prefix = "model.layers.3.ffn.experts"
experts.max_num_tokens = 4
experts.capture_fn = None
experts.get_symm_buffer = lambda: object()
experts.eplb_state = SimpleNamespace(
logical_to_physical_map=torch.empty(1),
expert_load_view=torch.empty(1),
logical_replica_count=torch.empty(1),
should_record_tensor=torch.empty(1),
num_unpadded_tokens_tensors=None,
)
topk_ids = torch.tensor([[1, 2], [3, 4]])
captured: list[tuple[int, torch.Tensor]] = []
bind_routed_experts_capturer(
SimpleNamespace(modules=lambda: [experts]),
SimpleNamespace(capture=lambda layer_id, ids: captured.append((layer_id, ids))),
)
class MappingReached(Exception):
pass
def map_ids(**kwargs):
assert captured == [(3, topk_ids)]
raise MappingReached
monkeypatch.setattr(
f"{experts_cls.__module__}.eplb_map_to_physical_and_record",
map_ids,
)
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm", lambda: SimpleNamespace()
)
with pytest.raises(MappingReached):
experts(
torch.empty(2, 8),
torch.empty(2, 2),
topk_ids,
activation_clamp=None,
)
def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
)
experts = DeepseekV4MegaMoEExperts(
vllm_config,
num_experts=4,
num_local_experts=2,
experts_start_idx=2,
top_k=2,
hidden_size=128,
intermediate_size=128,
)
nonlocal_weight = torch.ones(128, 64, dtype=torch.uint8)
assert (
experts.weight_loader(
experts.w13_weight,
nonlocal_weight,
"experts.w13_weight",
shard_id="w1",
expert_id=1,
return_success=True,
)
is False
)
w1 = torch.full((128, 64), 3, dtype=torch.uint8)
w3 = torch.full((128, 64), 7, dtype=torch.uint8)
w2 = torch.full((128, 64), 11, dtype=torch.uint8)
assert experts.weight_loader(
experts.w13_weight,
w1,
"experts.w13_weight",
shard_id="w1",
expert_id=2,
return_success=True,
)
assert experts.weight_loader(
experts.w13_weight,
w3,
"experts.w13_weight",
shard_id="w3",
expert_id=2,
return_success=True,
)
assert experts.weight_loader(
experts.w2_weight,
w2,
"experts.w2_weight",
shard_id="w2",
expert_id=2,
return_success=True,
)
interleaved = experts.w13_weight[0].view(torch.uint8).view(16, 2, 8, 64)
assert torch.equal(interleaved[:, 0], w1.view(16, 8, 64))
assert torch.equal(interleaved[:, 1], w3.view(16, 8, 64))
assert torch.equal(experts.w2_weight[0].view(torch.uint8), w2)
assert torch.count_nonzero(experts.w13_weight[1]) == 0
def _reference_mega_moe_layout(w13, s13, w2, s2):
"""CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales."""
def interleave(t):
g, n, k = t.shape
halves = t.view(g, 2, n // 16, 8, k)
return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k)
def utccp(sf):
g, mn, k = sf.shape
return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k)
return (
interleave(w13),
utccp(interleave(s13.view(torch.int32))),
w2,
utccp(s2.view(torch.int32)),
)
def _make_mega_moe_experts(num_local_experts, num_logical_experts):
return DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=num_local_experts,
num_local_experts=num_local_experts,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=384,
num_logical_experts=num_logical_experts,
)
def _load_mega_moe_checkpoint_shuffled(experts, num_logical_experts, seed):
hidden, inter = experts.hidden_size, experts.intermediate_size
shapes = {
"w13_weight": (num_logical_experts, 2 * inter, hidden // 2),
"w13_weight_scale": (num_logical_experts, 2 * inter, hidden // 32),
"w2_weight": (num_logical_experts, hidden, inter // 2),
"w2_weight_scale": (num_logical_experts, hidden, inter // 32),
}
generator = torch.Generator().manual_seed(seed)
ckpt = {
name: torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
for name, shape in shapes.items()
}
shards = []
for expert_id in range(num_logical_experts):
for name, tensor in ckpt.items():
if name.startswith("w2_"):
shards.append((name, tensor[expert_id], "w2", expert_id))
else:
shards.append((name, tensor[expert_id, :inter], "w1", expert_id))
shards.append((name, tensor[expert_id, inter:], "w3", expert_id))
for i in torch.randperm(len(shards), generator=generator).tolist():
name, loaded_weight, shard_id, expert_id = shards[i]
if name.endswith("scale"):
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
assert experts.weight_loader(
getattr(experts, name),
loaded_weight,
f"experts.{name}",
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
return ckpt
def _expected_mega_moe_layout(ckpt, num_local_experts, num_logical_experts):
physical = torch.arange(num_local_experts) % num_logical_experts
ckpt = {name: tensor[physical] for name, tensor in ckpt.items()}
expected = _reference_mega_moe_layout(*ckpt.values())
if is_deep_gemm_supported():
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
w13, s13, w2, s2 = (t.cuda() for t in ckpt.values())
hidden, inter = w2.shape[1], w2.shape[2] * 2
scales = [
deep_gemm.transform_sf_into_required_layout(
(sf.to(torch.int32) << 23).view(torch.float32),
mn,
k,
(1, 32),
num_local_experts,
)
for sf, mn, k in ((s13, 2 * inter, hidden), (s2, hidden, inter))
]
(l1, l1_sf), (l2, l2_sf) = deep_gemm.transform_weights_for_mega_moe(
(w13.view(torch.int8), scales[0]), (w2.view(torch.int8), scales[1])
)
for actual, reference in zip((l1, l1_sf, l2, l2_sf), expected):
assert torch.equal(actual.cpu().view(reference.dtype), reference)
return expected
def _assert_mega_moe_layout(experts, expected):
actual = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
for tensor, reference in zip(actual, expected):
assert torch.equal(tensor.view(reference.dtype), reference)
def test_deepseek_v4_mega_moe_loads_shards_into_kernel_layout():
"""Shards in any order, including EPLB replicas, land in DeepGEMM's layout."""
experts = _make_mega_moe_experts(num_local_experts=4, num_logical_experts=3)
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 3, seed=0)
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 4, 3))
# DeepGEMM's MN-major scale strides.
assert experts.w13_weight_scale.stride() == (768 * 2, 1, 768)
assert experts.w2_weight_scale.stride() == (256 * 3, 1, 256)
def test_deepseek_v4_mega_moe_reload_overwrites_kernel_layout_in_place():
"""A second load must update the live kernel tensors without reallocating."""
experts = _make_mega_moe_experts(num_local_experts=2, num_logical_experts=2)
_load_mega_moe_checkpoint_shuffled(experts, 2, seed=0)
data_ptrs = [param.data_ptr() for param in experts.parameters()]
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 2, seed=1)
assert [param.data_ptr() for param in experts.parameters()] == data_ptrs
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 2, 2))
@pytest.mark.parametrize("intermediate_size", [512, 2304])
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(intermediate_size):
"""Keep native widths so V4.1 routed and shared experts can fuse."""
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=128,
intermediate_size=intermediate_size,
)
assert experts.intermediate_size == intermediate_size
assert experts.w13_weight.shape == (2, 2 * intermediate_size, 64)
assert experts.w13_weight_scale.shape == (2, 2 * intermediate_size, 1)
assert experts.w2_weight.shape == (2, 128, intermediate_size // 2)
assert experts.w2_weight_scale.shape == (2, 128, intermediate_size // 128)
transformed = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
for actual, param in zip(transformed, experts.parameters()):
assert actual.data_ptr() == param.data_ptr()
def test_deepseek_v4_mega_moe_loads_shared_experts_into_kernel_layout(monkeypatch):
"""Shared expert shards land directly in the MegaMoE kernel layout."""
monkeypatch.setattr(
DeepGemmSm100MegaMoEBackend, "supports_shared_experts", lambda *args: True
)
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=384,
num_shared_experts=1,
)
assert experts.has_fused_shared_experts
hidden, inter = 256, 384
generator = torch.Generator().manual_seed(42)
def rand(*shape):
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
w1, w3, w2 = rand(inter, hidden), rand(inter, hidden), rand(hidden, inter)
s1, s3, s2 = (
rand(inter, hidden // 32),
rand(inter, hidden // 32),
rand(hidden, inter // 32),
)
fp8, e8m0 = torch.float8_e4m3fn, torch.float8_e8m0fnu
# The down shard arrives as `.w2.` (MTP, DSpark) or `.down_proj.` (mapped).
for name, data in [
("shared_experts.w1.weight", w1.view(fp8)),
("shared_experts.w3.weight", w3.view(fp8)),
("shared_experts.down_proj.weight", w2.view(fp8)),
("shared_experts.w1.weight_scale", s1.view(e8m0)),
("shared_experts.w3.weight_scale", s3.view(e8m0)),
("shared_experts.w2.weight_scale", s2.view(e8m0)),
]:
loaded = experts.load_shared_expert_weight(f"ffn.{name}", data)
# Weight tracking checks this name against the registered parameters.
assert loaded in {f"ffn.experts.{n}" for n, _ in experts.named_parameters()}
def l1_halves():
return experts.shared_l1_weight.view(torch.uint8).view(-1, 2, 8, hidden)
# Every tensor matches DeepGEMM's batched transform of the checkpoint.
expected = _reference_mega_moe_layout(
*(t.unsqueeze(0) for t in (torch.cat([w1, w3]), torch.cat([s1, s3]), w2, s2))
)
actual = (
experts.shared_l1_weight,
experts.shared_l1_weight_scale,
experts.shared_l2_weight,
experts.shared_l2_weight_scale,
)
for tensor, reference in zip(actual, expected):
assert torch.equal(tensor.view(reference.dtype), reference[0])
# A reload of one shard overwrites in place and leaves the other half alone.
ptr = experts.shared_l1_weight.data_ptr()
w1b = rand(inter, hidden)
experts.load_shared_expert_weight("shared_experts.w1.weight", w1b.view(fp8))
assert experts.shared_l1_weight.data_ptr() == ptr
assert torch.equal(l1_halves()[:, 0], w1b.view(-1, 8, hidden))
assert torch.equal(l1_halves()[:, 1], w3.view(-1, 8, hidden))
with pytest.raises(ValueError):
experts.load_shared_expert_weight("shared_experts.gate_up_proj.weight", w1)
def _fake_deep_gemm(with_shared: bool):
def old_buffer(group): ...
def old_kernel(y): ...
def new_buffer(group, num_shared_experts=0): ...
def new_kernel(y, shared_l1_weights=None, shared_l2_weights=None): ...
return SimpleNamespace(
get_symm_buffer_for_mega_moe=new_buffer if with_shared else old_buffer,
fp8_fp4_mega_moe=new_kernel if with_shared else old_kernel,
get_block_m_for_mega_moe=lambda *args: 0,
)
@pytest.mark.parametrize(
"weight_dtype,scale_dtype,scale_shape,with_shared,fused",
[
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), True, True),
(torch.float8_e4m3fn, torch.uint8, (256, 8), True, True),
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), False, False),
(torch.bfloat16, None, None, True, False),
(torch.float8_e4m3fn, torch.float32, (2, 2), True, False),
(torch.float8_e4m3fn, torch.uint8, (256, 16), True, False),
],
)
def test_deepseek_v4_mega_moe_decides_shared_fusion_at_construction(
monkeypatch, weight_dtype, scale_dtype, scale_shape, with_shared, fused
):
"""Fuse only shared experts the installed kernel and the 1x32 layout fit."""
monkeypatch.setattr(
"vllm.utils.deep_gemm._import_deep_gemm", lambda: _fake_deep_gemm(with_shared)
)
def linear():
scale = (
None if scale_dtype is None else torch.empty(scale_shape).to(scale_dtype)
)
return SimpleNamespace(
weight=torch.empty(256, 256, dtype=weight_dtype), weight_scale_inv=scale
)
shared = SimpleNamespace(gate_up_proj=linear(), down_proj=linear())
experts = DeepseekV4MegaMoEExperts(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
),
num_experts=2,
num_local_experts=2,
experts_start_idx=0,
top_k=1,
hidden_size=256,
intermediate_size=128,
num_shared_experts=1,
shared_experts=shared,
)
assert experts.has_fused_shared_experts == fused
assert hasattr(experts, "shared_l1_weight") == fused
@pytest.mark.parametrize("fused", [False, True])
def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
monkeypatch, fused
):
class FakeGate(torch.nn.Module):
weight = torch.empty(2, 128)
tid2eid = None
e_score_correction_bias = None
def forward(self, hidden_states):
return torch.empty(hidden_states.shape[0], 2), None
class FakeExperts(torch.nn.Module):
has_fused_shared_experts = fused
backend = DeepGemmSm100MegaMoEBackend()
def forward(self, hidden_states, *args, **kwargs):
return torch.ones_like(hidden_states)
class FakeSharedExperts(torch.nn.Module):
calls = 0
def forward(self, hidden_states):
self.calls += 1
return torch.full_like(hidden_states, 2)
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.use_native_mega_moe = True
moe.gate = FakeGate()
moe.experts = FakeExperts()
moe.shared_experts = None if fused else FakeSharedExperts()
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = 1
moe.renormalize = True
moe.hash_indices_dtype = torch.int64
moe.routed_scaling_factor = 1.0
moe.swiglu_limit = 10.0
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.model.fused_topk_bias",
lambda **kwargs: (
torch.ones(kwargs["hidden_states"].shape[0], 1),
torch.zeros(kwargs["hidden_states"].shape[0], 1, dtype=torch.int64),
),
)
output = moe(torch.zeros(2, 128))
expected = 1 if fused else 3
assert torch.all(output == expected)
if not fused:
assert moe.shared_experts.calls == 1
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
)
@pytest.mark.parametrize("num_tokens", [7, 63, 64, 65, 127, 128, 135, 434, 16384])
@pytest.mark.parametrize("hidden_size", [256, 5120])
def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact(
num_tokens, hidden_size
):
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(0)
hidden_states = (
torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.float32,
generator=generator,
)
* 17.0
).to(torch.bfloat16)
hidden_states[0, :32] = 0
hidden_states[1, 32:64] = 1.0e-6
hidden_states[2, 64:96] = -1.0e-6
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
ref_topk_idx = topk_ids.to(torch.int64)
ref_topk_weights = topk_weights.clone()
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_topk_idx = torch.empty_like(ref_topk_idx)
fused_topk_weights = torch.empty_like(ref_topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
)
torch.accelerator.synchronize()
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_topk_idx, ref_topk_idx)
assert torch.equal(
fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8),
)
@pytest.mark.parametrize("shared_block_m", [8, 32, 96, 128, 192])
def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
num_tokens = shared_block_m + 7
hidden_size = 256
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(shared_block_m)
hidden_states = torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.bfloat16,
generator=generator,
)
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
aligned_block_m = ((shared_block_m + 127) // 128) * 128
num_shared_rows = ((num_tokens + shared_block_m - 1) // shared_block_m) * (
aligned_block_m
)
ref_shared_x_sf = torch.zeros(
num_shared_rows,
hidden_size // 128,
dtype=torch.int32,
device=device,
)
for token_id in range(num_tokens):
m_in_block = token_id % shared_block_m
transposed_m = (
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
)
shared_row = token_id // shared_block_m * aligned_block_m + transposed_m
ref_shared_x_sf[shared_row].copy_(ref_x_sf[token_id])
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_shared_storage = torch.full(
(hidden_size // 128, num_shared_rows),
-1,
dtype=torch.int32,
device=device,
)
fused_shared_x_sf = fused_shared_storage.t()
fused_topk_idx = torch.empty_like(topk_ids, dtype=torch.int64)
fused_topk_weights = torch.empty_like(topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
shared_x_sf=fused_shared_x_sf,
shared_block_m=shared_block_m,
)
torch.accelerator.synchronize()
populated = ref_shared_x_sf != 0
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_shared_x_sf[populated], ref_shared_x_sf[populated])
assert torch.all(fused_shared_x_sf[~populated] == -1)
def test_deepseek_v4_pwal_hook_finalizes_mhc_broadcast():
"""The loader invokes the model-level PWAL hook for every load format,
so it must finalize mhc broadcast weights to cover dummy load, which
@@ -1040,85 +408,3 @@ def test_dspark_draft_registers_mixture_of_experts(
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
assert draft.num_routed_experts == hf_config.n_routed_experts
assert draft.num_redundant_experts == 4
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
)
@pytest.mark.parametrize("nonfinite_padding", [False, True])
@pytest.mark.parametrize("num_tokens", [7, 65, 135])
def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(
nonfinite_padding, num_tokens
):
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
hidden_size = 256
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(1)
hidden_states = torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.bfloat16,
generator=generator,
)
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
is_padding = torch.zeros(num_tokens, device=device, dtype=torch.bool)
is_padding[1::3] = True
is_padding[-1] = True
if nonfinite_padding:
topk_weights[is_padding] = float("nan")
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
ref_topk_idx = topk_ids.to(torch.int64)
ref_topk_idx[is_padding] = -1
ref_topk_weights = topk_weights.clone()
ref_topk_weights[is_padding] = 0.0
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_topk_idx = torch.empty_like(ref_topk_idx)
fused_topk_weights = torch.empty_like(ref_topk_weights)
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
is_padding=is_padding,
)
torch.accelerator.synchronize()
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_topk_idx, ref_topk_idx)
assert torch.equal(
fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8),
)
+3 -3
View File
@@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp):
@pytest.mark.parametrize(
"model_arch,supported",
[
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
("Qwen4ExpMTP", not current_platform.is_rocm()),
("Qwen4ExpForCausalLM", True),
("Qwen4ExpForConditionalGeneration", True),
("Qwen4ExpMTP", True),
("LlamaForCausalLM", False),
],
)