Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,11 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm fused KDA decode kernel must match the Triton chain it replaces.
|
||||
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
|
||||
|
||||
The fused kernel folds the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, and updates both the
|
||||
conv state and the recurrent state in place. Every one of those outputs is
|
||||
compared against the three-kernel path the AMD layer falls back to.
|
||||
Two fused paths fold the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, updating the conv
|
||||
state and the recurrent state in place:
|
||||
|
||||
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
|
||||
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
|
||||
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
|
||||
|
||||
Every output of each is compared against the unfused path the AMD layer falls
|
||||
back to, with the argument layout the layer itself uses.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +20,7 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_supported_arch() -> bool:
|
||||
def _hip_kernel_arch() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
@@ -22,10 +28,26 @@ def _on_supported_arch() -> bool:
|
||||
return on_gfx950() or on_gfx942()
|
||||
|
||||
|
||||
def _aiter_gluon_kda_available() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_supported_arch(),
|
||||
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
|
||||
)
|
||||
requires_hip_kernel_arch = pytest.mark.skipif(
|
||||
not _hip_kernel_arch(),
|
||||
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
|
||||
)
|
||||
requires_aiter_gluon_kda = pytest.mark.skipif(
|
||||
not _aiter_gluon_kda_available(),
|
||||
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
|
||||
)
|
||||
|
||||
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
|
||||
HEAD_DIM = 128
|
||||
@@ -41,7 +63,11 @@ def _requires_kernel() -> None:
|
||||
|
||||
|
||||
class KdaDecodeInputs:
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout."""
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
|
||||
|
||||
``num_spec`` > 0 sizes the conv state for speculative decoding
|
||||
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -49,6 +75,7 @@ class KdaDecodeInputs:
|
||||
num_heads: int,
|
||||
num_slots: int,
|
||||
seed: int = 0,
|
||||
num_spec: int = 0,
|
||||
) -> None:
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
@@ -70,9 +97,15 @@ class KdaDecodeInputs:
|
||||
for i in range(3)
|
||||
]
|
||||
)
|
||||
# SD cache layout: [slots, width - 1, 3 * dim]; the layer transposes it.
|
||||
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
|
||||
self.conv_state = (
|
||||
torch.randn(num_slots, CONV_WIDTH - 1, 3 * dim, device=device, dtype=DTYPE)
|
||||
torch.randn(
|
||||
num_slots,
|
||||
CONV_WIDTH - 1 + num_spec,
|
||||
3 * dim,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
* 0.5
|
||||
)
|
||||
self.recurrent_state = (
|
||||
@@ -112,6 +145,12 @@ class KdaDecodeInputs:
|
||||
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
|
||||
return state.transpose(-1, -2)
|
||||
|
||||
def empty_out(self) -> torch.Tensor:
|
||||
num_tokens = self.mixed_qkv.shape[0]
|
||||
return torch.empty(
|
||||
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
|
||||
)
|
||||
|
||||
|
||||
def _gated_rmsnorm(
|
||||
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
@@ -196,6 +235,7 @@ def _run_fused(
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@pytest.mark.parametrize("num_heads", [12, 24, 96])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
@@ -214,6 +254,7 @@ def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by state_indices may be written."""
|
||||
@@ -234,6 +275,7 @@ def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_without_output_norm() -> None:
|
||||
"""Omitting the gate/norm pair returns the raw recurrent output."""
|
||||
@@ -265,6 +307,7 @@ def test_fused_kda_decode_without_output_norm() -> None:
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
@@ -291,6 +334,7 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
"""The fused-decode weight copies must not depend on the weight loaders.
|
||||
|
||||
@@ -323,11 +367,15 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
config, vllm_config, prefix="model.layers.0.self_attn"
|
||||
)
|
||||
assert layer.decode_conv1d_weight is not None
|
||||
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
|
||||
# kernel gets an fp32 norm copy.
|
||||
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
|
||||
with torch.no_grad():
|
||||
for param in layer.parameters():
|
||||
param.normal_()
|
||||
layer.decode_conv1d_weight.fill_(float("nan"))
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
if layer.decode_norm_weight is not None:
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
|
||||
process_weights_after_loading(
|
||||
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
|
||||
@@ -341,7 +389,271 @@ def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
|
||||
)
|
||||
if layer.decode_norm_weight is not None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
assert not layer.decode_conv1d_weight.requires_grad
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# non-spec decode: fused_recurrent_kda_packed_decode
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_aiter_packed_decode(
|
||||
inp: KdaDecodeInputs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
num_tokens = inp.mixed_qkv.shape[0]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
|
||||
mixed_qkv=inp.mixed_qkv,
|
||||
g=inp.g1[0],
|
||||
beta=inp.beta[0],
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
ssm_state_indices=inp.state_indices[:num_tokens],
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:num_tokens],
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[0, :num_tokens],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_matches_triton_chain(
|
||||
num_heads: int, num_tokens: int
|
||||
) -> None:
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
# The fused kernel keeps the recurrent output in fp32 through the norm,
|
||||
# while the Triton chain rounds it to BF16 in between.
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
# The conv state is a pure shift-and-append: it must be bit-exact.
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by the state indices may be written."""
|
||||
num_tokens, num_heads, num_slots = 4, 12, 9
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
|
||||
|
||||
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
touched = set(inp.state_indices.tolist())
|
||||
untouched = [slot for slot in range(num_slots) if slot not in touched]
|
||||
assert untouched, "test needs at least one unused slot"
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
|
||||
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
|
||||
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
|
||||
must do the same rather than read-modify-write slot 0 once per padded row.
|
||||
"""
|
||||
num_real, num_padded, num_heads = 3, 5, 12
|
||||
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
|
||||
inp.state_indices[num_real:] = 0
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
|
||||
)
|
||||
assert torch.isfinite(actual_out).all()
|
||||
# Slot 0 must be untouched by both paths.
|
||||
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SpecBatch:
|
||||
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
|
||||
|
||||
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
|
||||
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
|
||||
per-token slot table the GDN metadata hands the layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
seq_lens: list[int],
|
||||
num_accepted: list[int],
|
||||
num_heads: int,
|
||||
num_spec: int,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
assert len(seq_lens) == len(num_accepted)
|
||||
assert max(seq_lens) <= num_spec + 1
|
||||
n_seqs = len(seq_lens)
|
||||
slots_per_seq = num_spec + 1
|
||||
num_slots = n_seqs * slots_per_seq + 2
|
||||
self.inp = KdaDecodeInputs(
|
||||
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
|
||||
)
|
||||
# Distinct slots per (request, token), shuffled, none at slot 0.
|
||||
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
|
||||
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
|
||||
self.query_start_loc = torch.tensor(
|
||||
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
self.num_accepted_tokens = torch.tensor(
|
||||
num_accepted, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
self.max_query_len = slots_per_seq
|
||||
|
||||
|
||||
def _run_spec_triton_chain(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
conv_out = causal_conv1d_update(
|
||||
inp.mixed_qkv,
|
||||
inp.conv_state_view(conv_state),
|
||||
inp.conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_state_indices=sb.state_indices[:, 0],
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
query_start_loc=sb.query_start_loc,
|
||||
max_query_len=sb.max_query_len,
|
||||
validate_data=False,
|
||||
out=torch.empty_like(inp.mixed_qkv),
|
||||
)
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
|
||||
)
|
||||
core_attn_out, _ = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
)
|
||||
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
def _run_aiter_spec(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM)
|
||||
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
|
||||
)
|
||||
n_spec = q.shape[1]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=inp.g1,
|
||||
beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:n_spec].unsqueeze(0),
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[:, :n_spec],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize(
|
||||
"num_spec, seq_lens, num_accepted",
|
||||
[
|
||||
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
|
||||
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
|
||||
(6, [7, 3], [1, 4]), # longer window, two requests
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_aiter_spec_decode_matches_triton_chain(
|
||||
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
|
||||
) -> None:
|
||||
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
|
||||
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
# Nothing outside the slot table may change.
|
||||
touched = set(sb.state_indices.flatten().tolist())
|
||||
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
|
||||
assert untouched
|
||||
torch.testing.assert_close(
|
||||
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
|
||||
@@ -75,6 +75,9 @@ AITER_MODEL_LIST = [
|
||||
"facebook/opt-125m", # opt
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"facebook/opt-350m", # opt with projected token embeddings
|
||||
),
|
||||
pytest.param(
|
||||
"microsoft/phi-2", # phi
|
||||
marks=[pytest.mark.core_model, pytest.mark.slow_test],
|
||||
|
||||
@@ -70,6 +70,10 @@ def gemma3_chat_template(content: str) -> str:
|
||||
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
|
||||
|
||||
|
||||
def deepseek_vl2_chat_template(content: str) -> str:
|
||||
return f"<|User|>: {content}\n\n<|Assistant|>:"
|
||||
|
||||
|
||||
def ernie45_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
f"<|begin_of_sentence|>User: {content}"
|
||||
@@ -255,7 +259,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"attention_backend": (
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4"
|
||||
"ROCM_FLASHMLA_SPARSE_DSV41"
|
||||
if current_platform.is_rocm()
|
||||
else "FLASHMLA_SPARSE_DSV41"
|
||||
),
|
||||
@@ -356,6 +360,15 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
},
|
||||
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
|
||||
),
|
||||
"deepseek_vl2": VitCudagraphTestConfig(
|
||||
model="deepseek-ai/deepseek-vl2-tiny",
|
||||
modalities=["image"],
|
||||
image_prompt=deepseek_vl2_chat_template("<image>\nWhat is in this image?"),
|
||||
vllm_runner_kwargs={
|
||||
"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]},
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"gemma4": VitCudagraphTestConfig(
|
||||
model="google/gemma-4-E2B-it",
|
||||
image_prompt=(
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
|
||||
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
|
||||
|
||||
The runtime reads a question's options from the 255 code logits of its readout
|
||||
at the last prompt token. vLLM's classify head must give the same code logits
|
||||
for the same prompt tokens, and for an image prompt that vLLM processes itself.
|
||||
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
|
||||
weight file without an index, next to the readout file.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +16,7 @@ from transformers import AutoModel
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "vllm-sr/d3-edge"
|
||||
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
|
||||
STATE = (
|
||||
"The order arrived damaged yesterday. The customer has a receipt and asks "
|
||||
"for a replacement today."
|
||||
@@ -47,7 +49,7 @@ def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
|
||||
return torch.softmax(logits[:count].float(), dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [MODEL])
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
|
||||
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
|
||||
runtime = reference.runtime
|
||||
|
||||
@@ -70,20 +70,21 @@ def test_moe_sp_rejects_pipeline_parallel() -> None:
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
|
||||
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=True),
|
||||
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
||||
):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(
|
||||
backend: str, use_sp: bool
|
||||
) -> None:
|
||||
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
mtp_module = f"vllm.models.qwen4_exp.{backend}.mtp"
|
||||
model = object.__new__(import_module(mtp_module).Qwen4ExpMultiTokenPredictor)
|
||||
model.use_hc_sequence_parallel = use_sp
|
||||
model.use_sequence_parallel = False
|
||||
torch.nn.Module.__init__(model)
|
||||
@@ -98,7 +99,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
def decoder(**kwargs):
|
||||
"""Check that the embedding residual shares the hidden state's token slice."""
|
||||
hidden = kwargs["hidden_states"]
|
||||
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
|
||||
if backend == "nvidia":
|
||||
prev_block_output = kwargs["prev_block_output"]
|
||||
assert prev_block_output.shape == (hidden.shape[0], model.hidden_size)
|
||||
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
||||
|
||||
model.layers = [decoder]
|
||||
@@ -120,9 +123,9 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
||||
|
||||
with (
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
|
||||
patch(f"{mtp_module}.get_pp_group", return_value=pp_group),
|
||||
patch(f"{mtp_module}.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch(f"{mtp_module}.sp_all_gather", side_effect=gather),
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
|
||||
@@ -14,8 +14,8 @@ from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import (
|
||||
@@ -38,14 +38,19 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
from .test_ple import _ConvBatchCase, _make_conv_metadata
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.qwen4_exp.amd import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.amd.model import Qwen4ExpDecoderLayer
|
||||
else:
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
|
||||
|
||||
class _TPAttentionProjection(nn.Module):
|
||||
"""Keep real TP communication without setting up an attention KV cache."""
|
||||
@@ -168,7 +173,8 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context({layer.ple.prefix: metadata}, config),
|
||||
):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
if hasattr(layer.ple, "start_prefetch"):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
output = layer(
|
||||
*inputs,
|
||||
positions,
|
||||
@@ -177,7 +183,10 @@ def _check_decoder_ple(rank: int) -> None:
|
||||
ngram_context=context,
|
||||
)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
# ROCm all-gather requires contiguous inputs.
|
||||
output = tuple(
|
||||
sp_all_gather(x.contiguous())[:num_tokens] for x in output
|
||||
)
|
||||
# A later MoE call may reuse workspace-backed output storage.
|
||||
outputs.append(tuple(x.clone() for x in output))
|
||||
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
|
||||
@@ -223,11 +232,38 @@ def _check_moe_sp(rank: int, config: VllmConfig) -> None:
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
actual = tuple(sp_all_gather(x.contiguous())[:num_tokens] for x in output)
|
||||
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
def _check_compiled_sp_ops() -> None:
|
||||
"""Compiled SP collectives stay correct for token counts unseen at trace time."""
|
||||
from vllm.models.qwen4_exp.amd.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
|
||||
def collectives(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
gathered = sp_all_gather(sp_shard(x) * 2)[: x.shape[0]]
|
||||
reduced = sp_all_gather(sp_reduce_scatter(x))[: x.shape[0]]
|
||||
return gathered, reduced
|
||||
|
||||
# vLLM drops Dynamo guards, so one trace must serve every token count.
|
||||
compiled = torch.compile(
|
||||
collectives,
|
||||
fullgraph=True,
|
||||
options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe},
|
||||
)
|
||||
traced = torch.randn(8, 16)
|
||||
torch._dynamo.mark_dynamic(traced, 0)
|
||||
compiled(traced)
|
||||
for num_tokens in (7, 3, 1):
|
||||
torch.manual_seed(num_tokens)
|
||||
x = torch.randn(num_tokens, 16)
|
||||
torch.testing.assert_close(compiled(x), collectives(x), atol=0, rtol=0)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool, compiled_ops: bool = False) -> None:
|
||||
"""Initialize rank-local runtime state and run one equivalence check."""
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
@@ -247,6 +283,8 @@ def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
if moe_sp:
|
||||
_check_moe_sp(rank, config)
|
||||
elif compiled_ops:
|
||||
_check_compiled_sp_ops()
|
||||
else:
|
||||
_check_decoder_ple(rank)
|
||||
finally:
|
||||
@@ -265,3 +303,10 @@ def test_moe_sp_equivalence(monkeypatch) -> None:
|
||||
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
|
||||
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
|
||||
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only SP ops")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_compiled_sp_ops_handle_untraced_token_counts() -> None:
|
||||
"""Padding in compiled SP collectives must not specialize on the traced size."""
|
||||
mp.spawn(_run_sp, args=(get_open_port(), False, True), nprocs=2)
|
||||
|
||||
@@ -462,7 +462,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"OlmoHybridForCausalLM": _HfExamplesInfo("allenai/Olmo-Hybrid-7B"),
|
||||
"OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
|
||||
"OPTForCausalLM": _HfExamplesInfo(
|
||||
"facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
|
||||
"facebook/opt-125m",
|
||||
{"350m": "facebook/opt-350m", "1b": "facebook/opt-iml-max-1.3b"},
|
||||
),
|
||||
"OrionForCausalLM": _HfExamplesInfo(
|
||||
"OrionStarAI/Orion-14B-Chat", trust_remote_code=True
|
||||
@@ -731,7 +732,13 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"D3ForSequenceClassification": _HfExamplesInfo(
|
||||
"vllm-sr/d3-edge",
|
||||
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
|
||||
extras={
|
||||
"d3": "vllm-sr/d3",
|
||||
"d3-flash": "vllm-sr/d3-flash",
|
||||
"d3-mini": "vllm-sr/d3-mini",
|
||||
"d3-nano": "vllm-sr/d3-nano",
|
||||
"d3-lite": "vllm-sr/d3-lite",
|
||||
},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"GPT2ForSequenceClassification": _HfExamplesInfo(
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pcp_size,use_ep", [(1, False), (2, True), (2, False), (4, False)]
|
||||
)
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_pcp_gathers_routing_ids_once_after_local_input_preparation(
|
||||
pcp_size, use_ep, use_sp, monkeypatch
|
||||
):
|
||||
"""Embedding keeps local IDs; all decoder layers receive aligned routing IDs."""
|
||||
gather = Mock(side_effect=lambda ids, dim: torch.cat([ids] * pcp_size, dim=dim))
|
||||
monkeypatch.setattr(
|
||||
model_module, "get_pcp_group", lambda: SimpleNamespace(all_gather=gather)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_module,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(model_module, "is_forward_context_available", lambda: False)
|
||||
monkeypatch.setattr(model_module, "sp_shard", lambda tensor: tensor[::2])
|
||||
monkeypatch.setattr(
|
||||
model_module, "sp_all_gather", lambda tensor: tensor.repeat_interleave(2, dim=0)
|
||||
)
|
||||
monkeypatch.setattr(model_module, "hc_collapse_triton", lambda hidden, mix: hidden)
|
||||
embed = Mock(side_effect=lambda ids: ids[:, None].float())
|
||||
run_layers = Mock(
|
||||
side_effect=lambda layer_ids, hidden, *args: (hidden, None, None, None, hidden)
|
||||
)
|
||||
model = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=pcp_size, enable_expert_parallel=use_ep
|
||||
),
|
||||
use_native_mega_moe=False,
|
||||
use_sequence_parallel=use_sp,
|
||||
engram_hash=None,
|
||||
embed_input_ids=embed,
|
||||
start_layer=0,
|
||||
decoder_replay_start=2,
|
||||
decoder_replay_layers=None,
|
||||
aux_hidden_state_layers=(),
|
||||
_mtp_hidden_buffer=None,
|
||||
_run_layers=run_layers,
|
||||
_collapse=lambda hidden, *args: hidden,
|
||||
norm=lambda hidden: hidden,
|
||||
)
|
||||
for offset in (0, 10):
|
||||
local_ids = torch.tensor([3, 7, 11, 15]) + offset
|
||||
model_module.DeepseekV4Model.forward(model, local_ids, torch.arange(4), None)
|
||||
assert embed.call_args.args[0] is local_ids
|
||||
routing_ids = local_ids[::2] if use_sp else local_ids
|
||||
needs_gather = pcp_size > 1 and not use_ep
|
||||
if needs_gather:
|
||||
torch.testing.assert_close(gather.call_args.args[0], routing_ids)
|
||||
routing_ids = routing_ids.repeat(pcp_size)
|
||||
torch.testing.assert_close(run_layers.call_args.args[3], routing_ids)
|
||||
assert gather.call_count == 2 * int(needs_gather)
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -16,6 +16,9 @@ from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
override_forward_context,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
)
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
@@ -63,6 +66,7 @@ def state(monkeypatch):
|
||||
cfg.scheduler_config.max_num_seqs = 16
|
||||
cfg.scheduler_config.max_num_batched_tokens = 1024
|
||||
cfg.parallel_config.data_parallel_size = 1
|
||||
cfg.aux_output_config.enable_return_routed_experts = False
|
||||
cfg.compilation_config.fast_moe_cold_start = False
|
||||
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
layers = DecoderReplayLayers(WINDOW, MagicMock(), MagicMock(), set(), "swa_first")
|
||||
@@ -172,6 +176,31 @@ def test_replay_batch_keeps_each_request_window(state):
|
||||
assert context.dp_metadata is None and context.is_padding is None
|
||||
|
||||
|
||||
def test_replay_context_restores_tp_routing_rows(state):
|
||||
"""Capture uses the compact replay batch's size, not the outer batch's."""
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
replay, _ = _prepare(state, batch, CUDAGraphMode.NONE)
|
||||
context = replay.forward_context
|
||||
assert context.is_padding is None and context.dp_metadata is None
|
||||
count = len(REPLAY_ROWS)
|
||||
assert context.batch_descriptor.num_tokens == count
|
||||
capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
|
||||
capturer.dp_rank, capturer.tp_size = 0, 2
|
||||
capturer.device_buffer = torch.full((401, 1, 2), -1, device=DEVICE)
|
||||
expected = torch.tensor(REPLAY_ROWS, device=DEVICE)[:, None].expand(-1, 2)
|
||||
gathered = torch.cat((expected, torch.full((1, 2), -1, device=DEVICE)))
|
||||
with (
|
||||
override_forward_context(context),
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.routed_experts_capturer.get_tp_group"
|
||||
) as group,
|
||||
):
|
||||
group.return_value.all_gather.return_value = gathered
|
||||
capturer.capture(0, gathered[: (count + 1) // 2])
|
||||
torch.testing.assert_close(capturer.device_buffer[:count, 0], expected)
|
||||
assert (capturer.device_buffer[count:] == -1).all()
|
||||
|
||||
|
||||
def test_replay_batch_keeps_device_decode_boundaries(state):
|
||||
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
|
||||
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
|
||||
@@ -195,6 +224,27 @@ def test_prompt_logprobs_requests_keep_their_rows(state):
|
||||
assert build.batch.max_query_len == 300
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE])
|
||||
def test_aux_output_requests_keep_every_routing_row(state, cg_mode):
|
||||
"""R3 needs every prefill row even when final logits only need the window."""
|
||||
state.vllm_config.aux_output_config.enable_return_routed_experts = True
|
||||
for index in range(3):
|
||||
state.add_request(
|
||||
index,
|
||||
SimpleNamespace(
|
||||
replay_start=0,
|
||||
sampling_params=SimpleNamespace(prompt_logprobs=None),
|
||||
),
|
||||
)
|
||||
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
|
||||
state.decoder_replay_layers.trim_threshold = WINDOW + 1
|
||||
replay, _ = _prepare(state, batch, cg_mode)
|
||||
if cg_mode == CUDAGraphMode.NONE:
|
||||
assert replay is None
|
||||
else:
|
||||
torch.testing.assert_close(replay.rows, torch.arange(401, device=DEVICE))
|
||||
|
||||
|
||||
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
|
||||
"""Adaptive verification bounds the decodes' device-side query lengths
|
||||
above their CPU lengths; the replay batch keeps that bound."""
|
||||
|
||||
@@ -8,10 +8,8 @@ import torch
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
DeepGemmSm100MegaMoEBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
bind_routed_experts_capturer,
|
||||
DeepGemmMegaMoEExperts,
|
||||
MegaMoEActivation,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
@@ -20,17 +18,14 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4ForCausalLM,
|
||||
DeepseekV4MegaMoEExperts,
|
||||
DeepseekV4MoE,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -40,11 +35,12 @@ pytestmark = pytest.mark.skipif(
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def sm100_mega_moe_backend(monkeypatch):
|
||||
def sm100_mega_moe_layout(monkeypatch):
|
||||
"""Build the SM100 kernel layout on any GPU; kernels are gated per test."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.model.get_deep_gemm_mega_moe_backend",
|
||||
lambda *args: DeepGemmSm100MegaMoEBackend(),
|
||||
"vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe."
|
||||
"check_deep_gemm_mega_moe_supported",
|
||||
lambda *args: None,
|
||||
)
|
||||
|
||||
|
||||
@@ -125,7 +121,8 @@ def test_deepseek_v41_moe_routes_without_hash_table(
|
||||
)
|
||||
assert moe.gate.tid2eid is None
|
||||
assert moe.gate.weight.shape == (num_experts, config.hidden_size)
|
||||
assert isinstance(moe.experts, DeepseekV4MegaMoEExperts)
|
||||
assert isinstance(moe.experts, DeepGemmMegaMoEExperts)
|
||||
assert moe.experts.activation == MegaMoEActivation("swiglu", clamp=10.0)
|
||||
assert moe.experts.w13_weight.shape == (num_experts, 256, 128)
|
||||
assert moe.experts.top_k == top_k
|
||||
assert (config.n_routed_experts, config.num_experts_per_tok) == (384, 6)
|
||||
@@ -151,10 +148,9 @@ def test_deepseek_v41_moe_routes_without_hash_table(
|
||||
|
||||
routed = {}
|
||||
|
||||
def check_routing(x, weights, ids, *, activation_clamp):
|
||||
def check_routing(x, weights, ids, shared_experts):
|
||||
routed["ids"], routed["weights"] = ids, weights
|
||||
assert activation_clamp == config.swiglu_limit
|
||||
return x.clone()
|
||||
return x + shared_experts(x)
|
||||
|
||||
monkeypatch.setattr(moe.experts, "forward", check_routing)
|
||||
monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x)
|
||||
@@ -280,10 +276,10 @@ def test_deepseek_v4_mega_gate_hash_routing_correctness(
|
||||
dim=-1, keepdim=True
|
||||
)
|
||||
|
||||
def check_routing(x, weights, ids, *, activation_clamp):
|
||||
def check_routing(x, weights, ids, shared_experts):
|
||||
torch.testing.assert_close(ids, expected_ids)
|
||||
torch.testing.assert_close(weights, expected_weights, rtol=1e-3, atol=1e-4)
|
||||
return x.clone()
|
||||
return x + shared_experts(x)
|
||||
|
||||
monkeypatch.setattr(moe.experts, "forward", check_routing)
|
||||
monkeypatch.setattr(moe.shared_experts, "forward", torch.zeros_like)
|
||||
@@ -308,634 +304,6 @@ def test_deepseek_v4_mega_moe_expert_mapping():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_kimi", [False, True])
|
||||
def test_deep_gemm_mega_moe_capture_precedes_eplb(monkeypatch, use_kimi):
|
||||
experts_cls = DeepseekV4MegaMoEExperts
|
||||
if use_kimi:
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiK3MegaMoEExperts
|
||||
|
||||
experts_cls = KimiK3MegaMoEExperts
|
||||
|
||||
experts = experts_cls.__new__(experts_cls)
|
||||
torch.nn.Module.__init__(experts)
|
||||
if use_kimi:
|
||||
experts.synchronize_first_launch = lambda: None
|
||||
experts.prefix = "model.layers.3.ffn.experts"
|
||||
experts.max_num_tokens = 4
|
||||
experts.capture_fn = None
|
||||
experts.get_symm_buffer = lambda: object()
|
||||
experts.eplb_state = SimpleNamespace(
|
||||
logical_to_physical_map=torch.empty(1),
|
||||
expert_load_view=torch.empty(1),
|
||||
logical_replica_count=torch.empty(1),
|
||||
should_record_tensor=torch.empty(1),
|
||||
num_unpadded_tokens_tensors=None,
|
||||
)
|
||||
|
||||
topk_ids = torch.tensor([[1, 2], [3, 4]])
|
||||
captured: list[tuple[int, torch.Tensor]] = []
|
||||
bind_routed_experts_capturer(
|
||||
SimpleNamespace(modules=lambda: [experts]),
|
||||
SimpleNamespace(capture=lambda layer_id, ids: captured.append((layer_id, ids))),
|
||||
)
|
||||
|
||||
class MappingReached(Exception):
|
||||
pass
|
||||
|
||||
def map_ids(**kwargs):
|
||||
assert captured == [(3, topk_ids)]
|
||||
raise MappingReached
|
||||
|
||||
monkeypatch.setattr(
|
||||
f"{experts_cls.__module__}.eplb_map_to_physical_and_record",
|
||||
map_ids,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm", lambda: SimpleNamespace()
|
||||
)
|
||||
|
||||
with pytest.raises(MappingReached):
|
||||
experts(
|
||||
torch.empty(2, 8),
|
||||
torch.empty(2, 2),
|
||||
topk_ids,
|
||||
activation_clamp=None,
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
)
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=4,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=2,
|
||||
top_k=2,
|
||||
hidden_size=128,
|
||||
intermediate_size=128,
|
||||
)
|
||||
|
||||
nonlocal_weight = torch.ones(128, 64, dtype=torch.uint8)
|
||||
assert (
|
||||
experts.weight_loader(
|
||||
experts.w13_weight,
|
||||
nonlocal_weight,
|
||||
"experts.w13_weight",
|
||||
shard_id="w1",
|
||||
expert_id=1,
|
||||
return_success=True,
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
w1 = torch.full((128, 64), 3, dtype=torch.uint8)
|
||||
w3 = torch.full((128, 64), 7, dtype=torch.uint8)
|
||||
w2 = torch.full((128, 64), 11, dtype=torch.uint8)
|
||||
|
||||
assert experts.weight_loader(
|
||||
experts.w13_weight,
|
||||
w1,
|
||||
"experts.w13_weight",
|
||||
shard_id="w1",
|
||||
expert_id=2,
|
||||
return_success=True,
|
||||
)
|
||||
assert experts.weight_loader(
|
||||
experts.w13_weight,
|
||||
w3,
|
||||
"experts.w13_weight",
|
||||
shard_id="w3",
|
||||
expert_id=2,
|
||||
return_success=True,
|
||||
)
|
||||
assert experts.weight_loader(
|
||||
experts.w2_weight,
|
||||
w2,
|
||||
"experts.w2_weight",
|
||||
shard_id="w2",
|
||||
expert_id=2,
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
interleaved = experts.w13_weight[0].view(torch.uint8).view(16, 2, 8, 64)
|
||||
assert torch.equal(interleaved[:, 0], w1.view(16, 8, 64))
|
||||
assert torch.equal(interleaved[:, 1], w3.view(16, 8, 64))
|
||||
assert torch.equal(experts.w2_weight[0].view(torch.uint8), w2)
|
||||
assert torch.count_nonzero(experts.w13_weight[1]) == 0
|
||||
|
||||
|
||||
def _reference_mega_moe_layout(w13, s13, w2, s2):
|
||||
"""CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales."""
|
||||
|
||||
def interleave(t):
|
||||
g, n, k = t.shape
|
||||
halves = t.view(g, 2, n // 16, 8, k)
|
||||
return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k)
|
||||
|
||||
def utccp(sf):
|
||||
g, mn, k = sf.shape
|
||||
return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k)
|
||||
|
||||
return (
|
||||
interleave(w13),
|
||||
utccp(interleave(s13.view(torch.int32))),
|
||||
w2,
|
||||
utccp(s2.view(torch.int32)),
|
||||
)
|
||||
|
||||
|
||||
def _make_mega_moe_experts(num_local_experts, num_logical_experts):
|
||||
return DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=num_local_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=384,
|
||||
num_logical_experts=num_logical_experts,
|
||||
)
|
||||
|
||||
|
||||
def _load_mega_moe_checkpoint_shuffled(experts, num_logical_experts, seed):
|
||||
hidden, inter = experts.hidden_size, experts.intermediate_size
|
||||
shapes = {
|
||||
"w13_weight": (num_logical_experts, 2 * inter, hidden // 2),
|
||||
"w13_weight_scale": (num_logical_experts, 2 * inter, hidden // 32),
|
||||
"w2_weight": (num_logical_experts, hidden, inter // 2),
|
||||
"w2_weight_scale": (num_logical_experts, hidden, inter // 32),
|
||||
}
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
ckpt = {
|
||||
name: torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
for name, shape in shapes.items()
|
||||
}
|
||||
shards = []
|
||||
for expert_id in range(num_logical_experts):
|
||||
for name, tensor in ckpt.items():
|
||||
if name.startswith("w2_"):
|
||||
shards.append((name, tensor[expert_id], "w2", expert_id))
|
||||
else:
|
||||
shards.append((name, tensor[expert_id, :inter], "w1", expert_id))
|
||||
shards.append((name, tensor[expert_id, inter:], "w3", expert_id))
|
||||
for i in torch.randperm(len(shards), generator=generator).tolist():
|
||||
name, loaded_weight, shard_id, expert_id = shards[i]
|
||||
if name.endswith("scale"):
|
||||
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
|
||||
assert experts.weight_loader(
|
||||
getattr(experts, name),
|
||||
loaded_weight,
|
||||
f"experts.{name}",
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
return ckpt
|
||||
|
||||
|
||||
def _expected_mega_moe_layout(ckpt, num_local_experts, num_logical_experts):
|
||||
physical = torch.arange(num_local_experts) % num_logical_experts
|
||||
ckpt = {name: tensor[physical] for name, tensor in ckpt.items()}
|
||||
expected = _reference_mega_moe_layout(*ckpt.values())
|
||||
if is_deep_gemm_supported():
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
w13, s13, w2, s2 = (t.cuda() for t in ckpt.values())
|
||||
hidden, inter = w2.shape[1], w2.shape[2] * 2
|
||||
scales = [
|
||||
deep_gemm.transform_sf_into_required_layout(
|
||||
(sf.to(torch.int32) << 23).view(torch.float32),
|
||||
mn,
|
||||
k,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
for sf, mn, k in ((s13, 2 * inter, hidden), (s2, hidden, inter))
|
||||
]
|
||||
(l1, l1_sf), (l2, l2_sf) = deep_gemm.transform_weights_for_mega_moe(
|
||||
(w13.view(torch.int8), scales[0]), (w2.view(torch.int8), scales[1])
|
||||
)
|
||||
for actual, reference in zip((l1, l1_sf, l2, l2_sf), expected):
|
||||
assert torch.equal(actual.cpu().view(reference.dtype), reference)
|
||||
return expected
|
||||
|
||||
|
||||
def _assert_mega_moe_layout(experts, expected):
|
||||
actual = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
|
||||
for tensor, reference in zip(actual, expected):
|
||||
assert torch.equal(tensor.view(reference.dtype), reference)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_loads_shards_into_kernel_layout():
|
||||
"""Shards in any order, including EPLB replicas, land in DeepGEMM's layout."""
|
||||
experts = _make_mega_moe_experts(num_local_experts=4, num_logical_experts=3)
|
||||
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 3, seed=0)
|
||||
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 4, 3))
|
||||
# DeepGEMM's MN-major scale strides.
|
||||
assert experts.w13_weight_scale.stride() == (768 * 2, 1, 768)
|
||||
assert experts.w2_weight_scale.stride() == (256 * 3, 1, 256)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_reload_overwrites_kernel_layout_in_place():
|
||||
"""A second load must update the live kernel tensors without reallocating."""
|
||||
experts = _make_mega_moe_experts(num_local_experts=2, num_logical_experts=2)
|
||||
_load_mega_moe_checkpoint_shuffled(experts, 2, seed=0)
|
||||
data_ptrs = [param.data_ptr() for param in experts.parameters()]
|
||||
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 2, seed=1)
|
||||
assert [param.data_ptr() for param in experts.parameters()] == data_ptrs
|
||||
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 2, 2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate_size", [512, 2304])
|
||||
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(intermediate_size):
|
||||
"""Keep native widths so V4.1 routed and shared experts can fuse."""
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=128,
|
||||
intermediate_size=intermediate_size,
|
||||
)
|
||||
|
||||
assert experts.intermediate_size == intermediate_size
|
||||
assert experts.w13_weight.shape == (2, 2 * intermediate_size, 64)
|
||||
assert experts.w13_weight_scale.shape == (2, 2 * intermediate_size, 1)
|
||||
assert experts.w2_weight.shape == (2, 128, intermediate_size // 2)
|
||||
assert experts.w2_weight_scale.shape == (2, 128, intermediate_size // 128)
|
||||
transformed = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
|
||||
for actual, param in zip(transformed, experts.parameters()):
|
||||
assert actual.data_ptr() == param.data_ptr()
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_loads_shared_experts_into_kernel_layout(monkeypatch):
|
||||
"""Shared expert shards land directly in the MegaMoE kernel layout."""
|
||||
monkeypatch.setattr(
|
||||
DeepGemmSm100MegaMoEBackend, "supports_shared_experts", lambda *args: True
|
||||
)
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=384,
|
||||
num_shared_experts=1,
|
||||
)
|
||||
assert experts.has_fused_shared_experts
|
||||
hidden, inter = 256, 384
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
|
||||
def rand(*shape):
|
||||
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
|
||||
w1, w3, w2 = rand(inter, hidden), rand(inter, hidden), rand(hidden, inter)
|
||||
s1, s3, s2 = (
|
||||
rand(inter, hidden // 32),
|
||||
rand(inter, hidden // 32),
|
||||
rand(hidden, inter // 32),
|
||||
)
|
||||
fp8, e8m0 = torch.float8_e4m3fn, torch.float8_e8m0fnu
|
||||
# The down shard arrives as `.w2.` (MTP, DSpark) or `.down_proj.` (mapped).
|
||||
for name, data in [
|
||||
("shared_experts.w1.weight", w1.view(fp8)),
|
||||
("shared_experts.w3.weight", w3.view(fp8)),
|
||||
("shared_experts.down_proj.weight", w2.view(fp8)),
|
||||
("shared_experts.w1.weight_scale", s1.view(e8m0)),
|
||||
("shared_experts.w3.weight_scale", s3.view(e8m0)),
|
||||
("shared_experts.w2.weight_scale", s2.view(e8m0)),
|
||||
]:
|
||||
loaded = experts.load_shared_expert_weight(f"ffn.{name}", data)
|
||||
# Weight tracking checks this name against the registered parameters.
|
||||
assert loaded in {f"ffn.experts.{n}" for n, _ in experts.named_parameters()}
|
||||
|
||||
def l1_halves():
|
||||
return experts.shared_l1_weight.view(torch.uint8).view(-1, 2, 8, hidden)
|
||||
|
||||
# Every tensor matches DeepGEMM's batched transform of the checkpoint.
|
||||
expected = _reference_mega_moe_layout(
|
||||
*(t.unsqueeze(0) for t in (torch.cat([w1, w3]), torch.cat([s1, s3]), w2, s2))
|
||||
)
|
||||
actual = (
|
||||
experts.shared_l1_weight,
|
||||
experts.shared_l1_weight_scale,
|
||||
experts.shared_l2_weight,
|
||||
experts.shared_l2_weight_scale,
|
||||
)
|
||||
for tensor, reference in zip(actual, expected):
|
||||
assert torch.equal(tensor.view(reference.dtype), reference[0])
|
||||
# A reload of one shard overwrites in place and leaves the other half alone.
|
||||
ptr = experts.shared_l1_weight.data_ptr()
|
||||
w1b = rand(inter, hidden)
|
||||
experts.load_shared_expert_weight("shared_experts.w1.weight", w1b.view(fp8))
|
||||
assert experts.shared_l1_weight.data_ptr() == ptr
|
||||
assert torch.equal(l1_halves()[:, 0], w1b.view(-1, 8, hidden))
|
||||
assert torch.equal(l1_halves()[:, 1], w3.view(-1, 8, hidden))
|
||||
with pytest.raises(ValueError):
|
||||
experts.load_shared_expert_weight("shared_experts.gate_up_proj.weight", w1)
|
||||
|
||||
|
||||
def _fake_deep_gemm(with_shared: bool):
|
||||
def old_buffer(group): ...
|
||||
|
||||
def old_kernel(y): ...
|
||||
|
||||
def new_buffer(group, num_shared_experts=0): ...
|
||||
|
||||
def new_kernel(y, shared_l1_weights=None, shared_l2_weights=None): ...
|
||||
|
||||
return SimpleNamespace(
|
||||
get_symm_buffer_for_mega_moe=new_buffer if with_shared else old_buffer,
|
||||
fp8_fp4_mega_moe=new_kernel if with_shared else old_kernel,
|
||||
get_block_m_for_mega_moe=lambda *args: 0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight_dtype,scale_dtype,scale_shape,with_shared,fused",
|
||||
[
|
||||
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), True, True),
|
||||
(torch.float8_e4m3fn, torch.uint8, (256, 8), True, True),
|
||||
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), False, False),
|
||||
(torch.bfloat16, None, None, True, False),
|
||||
(torch.float8_e4m3fn, torch.float32, (2, 2), True, False),
|
||||
(torch.float8_e4m3fn, torch.uint8, (256, 16), True, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_mega_moe_decides_shared_fusion_at_construction(
|
||||
monkeypatch, weight_dtype, scale_dtype, scale_shape, with_shared, fused
|
||||
):
|
||||
"""Fuse only shared experts the installed kernel and the 1x32 layout fit."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm", lambda: _fake_deep_gemm(with_shared)
|
||||
)
|
||||
|
||||
def linear():
|
||||
scale = (
|
||||
None if scale_dtype is None else torch.empty(scale_shape).to(scale_dtype)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
weight=torch.empty(256, 256, dtype=weight_dtype), weight_scale_inv=scale
|
||||
)
|
||||
|
||||
shared = SimpleNamespace(gate_up_proj=linear(), down_proj=linear())
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=128,
|
||||
num_shared_experts=1,
|
||||
shared_experts=shared,
|
||||
)
|
||||
|
||||
assert experts.has_fused_shared_experts == fused
|
||||
assert hasattr(experts, "shared_l1_weight") == fused
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fused", [False, True])
|
||||
def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
monkeypatch, fused
|
||||
):
|
||||
class FakeGate(torch.nn.Module):
|
||||
weight = torch.empty(2, 128)
|
||||
tid2eid = None
|
||||
e_score_correction_bias = None
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return torch.empty(hidden_states.shape[0], 2), None
|
||||
|
||||
class FakeExperts(torch.nn.Module):
|
||||
has_fused_shared_experts = fused
|
||||
backend = DeepGemmSm100MegaMoEBackend()
|
||||
|
||||
def forward(self, hidden_states, *args, **kwargs):
|
||||
return torch.ones_like(hidden_states)
|
||||
|
||||
class FakeSharedExperts(torch.nn.Module):
|
||||
calls = 0
|
||||
|
||||
def forward(self, hidden_states):
|
||||
self.calls += 1
|
||||
return torch.full_like(hidden_states, 2)
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.use_native_mega_moe = True
|
||||
moe.gate = FakeGate()
|
||||
moe.experts = FakeExperts()
|
||||
moe.shared_experts = None if fused else FakeSharedExperts()
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = 1
|
||||
moe.renormalize = True
|
||||
moe.hash_indices_dtype = torch.int64
|
||||
moe.routed_scaling_factor = 1.0
|
||||
moe.swiglu_limit = 10.0
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.model.fused_topk_bias",
|
||||
lambda **kwargs: (
|
||||
torch.ones(kwargs["hidden_states"].shape[0], 1),
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 1, dtype=torch.int64),
|
||||
),
|
||||
)
|
||||
|
||||
output = moe(torch.zeros(2, 128))
|
||||
|
||||
expected = 1 if fused else 3
|
||||
assert torch.all(output == expected)
|
||||
if not fused:
|
||||
assert moe.shared_experts.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [7, 63, 64, 65, 127, 128, 135, 434, 16384])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 5120])
|
||||
def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact(
|
||||
num_tokens, hidden_size
|
||||
):
|
||||
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
|
||||
|
||||
device = torch.device("cuda")
|
||||
top_k = 8
|
||||
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(0)
|
||||
hidden_states = (
|
||||
torch.randn(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
* 17.0
|
||||
).to(torch.bfloat16)
|
||||
hidden_states[0, :32] = 0
|
||||
hidden_states[1, 32:64] = 1.0e-6
|
||||
hidden_states[2, 64:96] = -1.0e-6
|
||||
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_tokens, top_k),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
topk_weights = torch.randn(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
ref_x, ref_x_sf = per_token_cast_to_fp8(
|
||||
hidden_states,
|
||||
use_ue8m0=True,
|
||||
gran_k=32,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
ref_topk_idx = topk_ids.to(torch.int64)
|
||||
ref_topk_weights = topk_weights.clone()
|
||||
|
||||
fused_x = torch.empty_like(ref_x)
|
||||
fused_x_sf = torch.empty_like(ref_x_sf)
|
||||
fused_topk_idx = torch.empty_like(ref_topk_idx)
|
||||
fused_topk_weights = torch.empty_like(ref_topk_weights)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
fused_x,
|
||||
fused_x_sf,
|
||||
fused_topk_idx,
|
||||
fused_topk_weights,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
|
||||
assert torch.equal(fused_x_sf, ref_x_sf)
|
||||
assert torch.equal(fused_topk_idx, ref_topk_idx)
|
||||
assert torch.equal(
|
||||
fused_topk_weights.view(torch.uint8),
|
||||
ref_topk_weights.view(torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared_block_m", [8, 32, 96, 128, 192])
|
||||
def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
|
||||
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_tokens = shared_block_m + 7
|
||||
hidden_size = 256
|
||||
top_k = 8
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(shared_block_m)
|
||||
hidden_states = torch.randn(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=generator,
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_tokens, top_k),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
topk_weights = torch.randn(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
ref_x, ref_x_sf = per_token_cast_to_fp8(
|
||||
hidden_states,
|
||||
use_ue8m0=True,
|
||||
gran_k=32,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
aligned_block_m = ((shared_block_m + 127) // 128) * 128
|
||||
num_shared_rows = ((num_tokens + shared_block_m - 1) // shared_block_m) * (
|
||||
aligned_block_m
|
||||
)
|
||||
ref_shared_x_sf = torch.zeros(
|
||||
num_shared_rows,
|
||||
hidden_size // 128,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
for token_id in range(num_tokens):
|
||||
m_in_block = token_id % shared_block_m
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
shared_row = token_id // shared_block_m * aligned_block_m + transposed_m
|
||||
ref_shared_x_sf[shared_row].copy_(ref_x_sf[token_id])
|
||||
|
||||
fused_x = torch.empty_like(ref_x)
|
||||
fused_x_sf = torch.empty_like(ref_x_sf)
|
||||
fused_shared_storage = torch.full(
|
||||
(hidden_size // 128, num_shared_rows),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
fused_shared_x_sf = fused_shared_storage.t()
|
||||
fused_topk_idx = torch.empty_like(topk_ids, dtype=torch.int64)
|
||||
fused_topk_weights = torch.empty_like(topk_weights)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
fused_x,
|
||||
fused_x_sf,
|
||||
fused_topk_idx,
|
||||
fused_topk_weights,
|
||||
shared_x_sf=fused_shared_x_sf,
|
||||
shared_block_m=shared_block_m,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
populated = ref_shared_x_sf != 0
|
||||
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
|
||||
assert torch.equal(fused_x_sf, ref_x_sf)
|
||||
assert torch.equal(fused_shared_x_sf[populated], ref_shared_x_sf[populated])
|
||||
assert torch.all(fused_shared_x_sf[~populated] == -1)
|
||||
|
||||
|
||||
def test_deepseek_v4_pwal_hook_finalizes_mhc_broadcast():
|
||||
"""The loader invokes the model-level PWAL hook for every load format,
|
||||
so it must finalize mhc broadcast weights to cover dummy load, which
|
||||
@@ -1040,85 +408,3 @@ def test_dspark_draft_registers_mixture_of_experts(
|
||||
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
|
||||
assert draft.num_routed_experts == hf_config.n_routed_experts
|
||||
assert draft.num_redundant_experts == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize("nonfinite_padding", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [7, 65, 135])
|
||||
def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(
|
||||
nonfinite_padding, num_tokens
|
||||
):
|
||||
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
|
||||
|
||||
device = torch.device("cuda")
|
||||
hidden_size = 256
|
||||
top_k = 8
|
||||
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(1)
|
||||
hidden_states = torch.randn(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=generator,
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_tokens, top_k),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
topk_weights = torch.randn(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
is_padding = torch.zeros(num_tokens, device=device, dtype=torch.bool)
|
||||
is_padding[1::3] = True
|
||||
is_padding[-1] = True
|
||||
if nonfinite_padding:
|
||||
topk_weights[is_padding] = float("nan")
|
||||
|
||||
ref_x, ref_x_sf = per_token_cast_to_fp8(
|
||||
hidden_states,
|
||||
use_ue8m0=True,
|
||||
gran_k=32,
|
||||
use_packed_ue8m0=True,
|
||||
)
|
||||
ref_topk_idx = topk_ids.to(torch.int64)
|
||||
ref_topk_idx[is_padding] = -1
|
||||
ref_topk_weights = topk_weights.clone()
|
||||
ref_topk_weights[is_padding] = 0.0
|
||||
|
||||
fused_x = torch.empty_like(ref_x)
|
||||
fused_x_sf = torch.empty_like(ref_x_sf)
|
||||
fused_topk_idx = torch.empty_like(ref_topk_idx)
|
||||
fused_topk_weights = torch.empty_like(ref_topk_weights)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
fused_x,
|
||||
fused_x_sf,
|
||||
fused_topk_idx,
|
||||
fused_topk_weights,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
|
||||
assert torch.equal(fused_x_sf, ref_x_sf)
|
||||
assert torch.equal(fused_topk_idx, ref_topk_idx)
|
||||
assert torch.equal(
|
||||
fused_topk_weights.view(torch.uint8),
|
||||
ref_topk_weights.view(torch.uint8),
|
||||
)
|
||||
|
||||
@@ -127,9 +127,9 @@ def test_registry_is_pp(model_arch, is_pp):
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Qwen4ExpMTP", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForCausalLM", True),
|
||||
("Qwen4ExpForConditionalGeneration", True),
|
||||
("Qwen4ExpMTP", True),
|
||||
("LlamaForCausalLM", False),
|
||||
],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user