[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:03:09 -04:00
committed by GitHub
parent b3ee560ae8
commit a281ad32f0
12 changed files with 665 additions and 76 deletions
+95
View File
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_weight_scale_block,
)
from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
is_mixture_of_experts,
)
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4ForCausalLM,
@@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
assert calls == ["mtp", "dspark"]
@pytest.fixture
def v4_dspark_config(dist_init):
hf_config = DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hidden_act="silu",
swiglu_limit=10.0,
norm_topk_prob=True,
topk_method="noaux_tc",
routed_scaling_factor=1.5,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
compress_ratios=[1, 1],
q_lora_rank=32,
o_lora_rank=32,
qk_rope_head_dim=16,
o_groups=4,
sliding_window=128,
hc_sinkhorn_iters=1,
)
model_config = SimpleNamespace(
dtype=torch.bfloat16,
hf_config=hf_config,
max_model_len=2048,
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(
moe_backend="deep_gemm_mega_moe",
enable_jit_warmup=False,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(draft_model_config=model_config),
use_v2_model_runner=False,
)
def test_dspark_draft_registers_mixture_of_experts(
v4_dspark_config, monkeypatch, dist_init
):
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
if not current_platform.is_device_capability_family(100):
pytest.skip("DeepGEMM MegaMoE requires SM100")
hf_config = v4_dspark_config.model_config.hf_config
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
lambda: v4_dspark_config,
)
with (
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
torch.device("cuda"),
):
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
assert is_mixture_of_experts(draft)
assert get_mixture_of_experts_model(draft) is draft
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
assert draft.num_routed_experts == hf_config.n_routed_experts
assert draft.num_redundant_experts == 4
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
+51
View File
@@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales(
torch.testing.assert_close(param, checkpoint_scale)
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
monkeypatch,
):
from vllm.models.deepseek_v4.nvidia import dspark
captured: dict[str, int] = {}
def capture_expert_mapping(*args, **kwargs):
captured.update(kwargs)
return []
draft = SimpleNamespace(
config=SimpleNamespace(
num_attention_heads=4,
n_routed_experts=8,
expert_dtype="fp4",
),
quant_config=None,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[
SimpleNamespace(
ffn=SimpleNamespace(
use_native_mega_moe=False,
use_mega_moe=False,
n_redundant_experts=4,
)
)
],
confidence_head=None,
),
named_parameters=lambda: [],
process_weights_after_loading=lambda: None,
)
draft._remap_dspark_name = lambda name: (
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
dspark,
"fused_moe_make_expert_params_mapping",
capture_expert_mapping,
)
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
assert captured["num_experts"] == 8
assert captured["num_redundant_experts"] == 4
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
weights = [
("mtp.0.attn.wkv.weight", torch.arange(4)),