[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:03:09 -04:00
committed by GitHub
parent b3ee560ae8
commit a281ad32f0
12 changed files with 665 additions and 76 deletions
+224 -6
View File
@@ -2,11 +2,21 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import lm_eval
import pytest
import torch
from tests.utils import large_gpu_mark
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.v1.worker.gpu.eplb_utils import (
EPLBController,
draft_model_supports_eplb,
)
def get_model_args(
@@ -15,14 +25,18 @@ def get_model_args(
spec_method: str,
tp_size: int,
model_max_len: int,
num_speculative_tokens: int = 1,
use_async: bool = True,
) -> dict:
speculative_config = {
"method": spec_method,
"model": spec_model_name,
"num_speculative_tokens": 1,
"num_speculative_tokens": num_speculative_tokens,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
speculative_config["model"] = model_name
speculative_config["draft_sample_method"] = "probabilistic"
eplb_config = {
"num_redundant_experts": tp_size,
"window_size": 128,
@@ -42,6 +56,8 @@ def get_model_args(
"enable_eplb": True,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
model_args["trust_remote_code"] = True
return model_args
@@ -51,11 +67,190 @@ pytestmark = pytest.mark.skipif(
)
class FakeEplbState:
instances: list[FakeEplbState] = []
def __init__(self, parallel_config, device: torch.device):
self.add_model_calls: list[tuple[object, object, str | None]] = []
FakeEplbState.instances.append(self)
def add_model(
self,
model: object,
model_config: object,
*,
model_name: str | None = None,
) -> None:
self.add_model_calls.append((model, model_config, model_name))
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
return DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
compress_ratios=[1, 1],
)
@pytest.fixture
def dspark_vllm_config():
hf_config = _make_dsv4_dspark_hf_config()
model_config = SimpleNamespace(
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
enable_elastic_ep=False,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(
method="dspark",
draft_model_config=model_config,
),
)
def _make_moe_topology(
*,
num_routed_experts: int = 8,
num_redundant_experts: int = 4,
num_expert_groups: int = 1,
) -> SimpleNamespace:
num_physical_experts = num_routed_experts + num_redundant_experts
return SimpleNamespace(
num_routed_experts=num_routed_experts,
num_redundant_experts=num_redundant_experts,
num_physical_experts=num_physical_experts,
num_logical_experts=num_routed_experts,
num_expert_groups=num_expert_groups,
)
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
draft = _make_moe_topology()
target = _make_moe_topology()
EplbState.assert_confs_equal(draft, target)
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
draft = _make_moe_topology(num_redundant_experts=0)
target = _make_moe_topology(num_redundant_experts=4)
with pytest.raises(RuntimeError, match="mismatch"):
EplbState.assert_confs_equal(draft, target)
@pytest.mark.parametrize(
("model_type", "expected"),
[
("deepseek_v4", True),
("deepseek_v41", False),
],
)
def test_dspark_draft_supports_eplb(
dspark_vllm_config, model_type: str, expected: bool
):
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
assert dspark_draft_supports_eplb(draft_model_config) is expected
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
assert not draft_model_supports_eplb(
dspark_vllm_config.speculative_config,
None,
)
@pytest.mark.parametrize(
"model_type,load_dummy_weights,expected_registered",
[
("deepseek_v4", False, True),
("deepseek_v4", True, False),
("deepseek_v41", False, False),
],
)
def test_eplb_dspark_draft_registration(
dspark_vllm_config,
monkeypatch: pytest.MonkeyPatch,
model_type,
load_dummy_weights: bool,
expected_registered: bool,
):
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
FakeEplbState.instances.clear()
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.EplbState",
FakeEplbState,
)
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
lambda model: model,
)
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
draft = SimpleNamespace()
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
controller.prepare_load()
speculator = SimpleNamespace(model=draft, eplb_state=None)
def set_eplb_state(state) -> None:
speculator.eplb_state = state
speculator.set_eplb_state = set_eplb_state
registered = controller.maybe_register_speculator(
speculator,
dspark_vllm_config.speculative_config,
load_dummy_weights=load_dummy_weights,
)
assert registered is expected_registered
assert controller.state is not None
if expected_registered:
assert controller.state.add_model_calls == [
(draft, draft_model_config, "dspark (draft)")
]
assert speculator.eplb_state is controller.state
else:
assert controller.state.add_model_calls == []
assert speculator.eplb_state is None
@pytest.mark.parametrize(
"model_setup",
[
pytest.param(
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86),
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
marks=large_gpu_mark(min_gb=80),
),
pytest.param(
@@ -65,20 +260,42 @@ pytestmark = pytest.mark.skipif(
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
4,
0.92,
1,
),
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
),
pytest.param(
(
"dspark",
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
None,
4,
0.95,
7,
),
marks=large_gpu_mark(min_gb=80),
),
],
ids=["qwen3_next_mtp", "llama4_eagle"],
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
)
def test_eplb_spec_decode(
monkeypatch: pytest.MonkeyPatch,
model_setup: tuple[str, str, str, int, float],
model_setup: tuple[str, str, str | None, int, float, int],
):
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
Applicable to MoE models with mtp or eagle spec decode.
Applicable to MoE models with mtp, eagle, or dspark spec decode.
"""
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
(
method,
model_name,
spec_model_name,
tp_size,
expected_gsm8k_value,
num_speculative_tokens,
) = model_setup
if method == "dspark" and not current_platform.is_device_capability_family(100):
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
@@ -90,6 +307,7 @@ def test_eplb_spec_decode(
spec_method=method,
tp_size=tp_size,
model_max_len=4096,
num_speculative_tokens=num_speculative_tokens,
)
results = lm_eval.simple_evaluate(
+95
View File
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_weight_scale_block,
)
from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
is_mixture_of_experts,
)
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4ForCausalLM,
@@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
assert calls == ["mtp", "dspark"]
@pytest.fixture
def v4_dspark_config(dist_init):
hf_config = DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hidden_act="silu",
swiglu_limit=10.0,
norm_topk_prob=True,
topk_method="noaux_tc",
routed_scaling_factor=1.5,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
compress_ratios=[1, 1],
q_lora_rank=32,
o_lora_rank=32,
qk_rope_head_dim=16,
o_groups=4,
sliding_window=128,
hc_sinkhorn_iters=1,
)
model_config = SimpleNamespace(
dtype=torch.bfloat16,
hf_config=hf_config,
max_model_len=2048,
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(
moe_backend="deep_gemm_mega_moe",
enable_jit_warmup=False,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(draft_model_config=model_config),
use_v2_model_runner=False,
)
def test_dspark_draft_registers_mixture_of_experts(
v4_dspark_config, monkeypatch, dist_init
):
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
if not current_platform.is_device_capability_family(100):
pytest.skip("DeepGEMM MegaMoE requires SM100")
hf_config = v4_dspark_config.model_config.hf_config
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
lambda: v4_dspark_config,
)
with (
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
torch.device("cuda"),
):
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
assert is_mixture_of_experts(draft)
assert get_mixture_of_experts_model(draft) is draft
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
assert draft.num_routed_experts == hf_config.n_routed_experts
assert draft.num_redundant_experts == 4
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
+51
View File
@@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales(
torch.testing.assert_close(param, checkpoint_scale)
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
monkeypatch,
):
from vllm.models.deepseek_v4.nvidia import dspark
captured: dict[str, int] = {}
def capture_expert_mapping(*args, **kwargs):
captured.update(kwargs)
return []
draft = SimpleNamespace(
config=SimpleNamespace(
num_attention_heads=4,
n_routed_experts=8,
expert_dtype="fp4",
),
quant_config=None,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[
SimpleNamespace(
ffn=SimpleNamespace(
use_native_mega_moe=False,
use_mega_moe=False,
n_redundant_experts=4,
)
)
],
confidence_head=None,
),
named_parameters=lambda: [],
process_weights_after_loading=lambda: None,
)
draft._remap_dspark_name = lambda name: (
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
dspark,
"fused_moe_make_expert_params_mapping",
capture_expert_mapping,
)
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
assert captured["num_experts"] == 8
assert captured["num_redundant_experts"] == 4
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
weights = [
("mtp.0.attn.wkv.weight", torch.arange(4)),
+33 -4
View File
@@ -15,6 +15,16 @@ from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_con
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@dataclass
class _FakeHFConfig:
model_type: str
@dataclass
class _FakeModelConfig:
hf_config: _FakeHFConfig
@dataclass
class _FakeEPLBConfig:
num_redundant_experts: int = 0
@@ -37,12 +47,28 @@ class _FakeParallelConfig:
raise ValueError("elastic EP requires EPLB")
def test_dspark_parallel_config_disables_eplb_atomically():
@pytest.mark.parametrize(
("model_type", "enable_eplb", "num_redundant_experts", "shares_eplb_config"),
[
("deepseek_v4", True, 32, True),
("deepseek_v41", False, 0, False),
],
)
def test_dspark_parallel_config_eplb_by_model_type(
model_type: str,
enable_eplb: bool,
num_redundant_experts: int,
shares_eplb_config: bool,
):
target_config = _FakeParallelConfig()
draft_model_config = _FakeModelConfig(
hf_config=_FakeHFConfig(model_type=model_type)
)
draft_config = _get_dspark_parallel_config(
target_config,
tensor_parallel_size=4,
draft_model_config=draft_model_config,
)
assert target_config.pipeline_parallel_size == 2
@@ -54,10 +80,13 @@ def test_dspark_parallel_config_disables_eplb_atomically():
assert draft_config is not target_config
assert draft_config.pipeline_parallel_size == 1
assert draft_config.tensor_parallel_size == 4
assert not draft_config.enable_eplb
assert draft_config.eplb_config.num_redundant_experts == 0
assert draft_config.enable_eplb is enable_eplb
assert draft_config.eplb_config.num_redundant_experts == num_redundant_experts
assert not draft_config.enable_elastic_ep
assert draft_config.eplb_config is not target_config.eplb_config
if shares_eplb_config:
assert draft_config.eplb_config is target_config.eplb_config
else:
assert draft_config.eplb_config is not target_config.eplb_config
@pytest.mark.parametrize("pcp_size", [1, 4])
@@ -29,15 +29,21 @@ class FakeEplbState:
def __init__(self, parallel_config: Any, device: torch.device):
self.parallel_config = parallel_config
self.device = device
self.add_model_calls: list[tuple[Any, Any]] = []
self.add_model_calls: list[tuple[Any, Any, str | None]] = []
self.step_calls: list[tuple[bool, bool, bool]] = []
self.async_started = False
self.is_async = True
self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = []
FakeEplbState.instances.append(self)
def add_model(self, model: Any, model_config: Any) -> None:
self.add_model_calls.append((model, model_config))
def add_model(
self,
model: Any,
model_config: Any,
*,
model_name: str | None = None,
) -> None:
self.add_model_calls.append((model, model_config, model_name))
def step(self, is_dummy: bool, is_profile: bool, *, log_stats: bool) -> None:
self.step_calls.append((is_dummy, is_profile, log_stats))
@@ -98,6 +104,15 @@ def _make_runner(**overrides: Any) -> Any:
runner.execute_model_state = None
for key, value in overrides.items():
setattr(runner, key, value)
spec_config = runner.speculative_config
runner._draft_workspace_lane = int(
spec_config is not None
and (
spec_config.use_dspark()
if hasattr(spec_config, "use_dspark")
else getattr(spec_config, "method", None) == "dspark"
)
)
return runner
@@ -131,7 +146,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
assert runner.model is model
assert runner.model_state is not None
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
assert runner.eplb_state.async_started is True
@@ -159,7 +174,7 @@ def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch):
assert runner.load_config.load_format == "dummy"
assert runner.eplb_state is not None
# A scaling-up worker registers so it can join the EPLB communicator.
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
assert runner.eplb_state.async_started is False
@@ -174,6 +189,64 @@ def test_v2_setup_eplb_from_mapping_updates_state_in_place():
assert state.update_mapping_calls == [(runner.model_config, mapping)]
def test_v2_load_model_registers_dspark_speculator_with_eplb(monkeypatch):
FakeEplbState.instances.clear()
target_model = SimpleNamespace(is_moe=True)
draft_model = SimpleNamespace(is_moe=True)
draft_model_config = SimpleNamespace(
model="dspark-draft",
hf_config=SimpleNamespace(model_type="deepseek_v4"),
)
class FakeDSparkSpeculator:
def __init__(self):
self.model = draft_model
self.eplb_state = None
def load_model(self, target_model):
return None
def set_eplb_state(self, state) -> None:
self.eplb_state = state
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
monkeypatch.setattr(mrv2, "DraftModelSpeculator", FakeDSparkSpeculator)
monkeypatch.setattr(
mrv2,
"get_model_loader",
lambda load_config: SimpleNamespace(load_model=lambda **_: target_model),
)
monkeypatch.setattr(
mrv2,
"init_model_state",
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
)
monkeypatch.setattr(
eplb,
"get_mixture_of_experts_model",
lambda model: model if getattr(model, "is_moe", False) else None,
)
speculator = FakeDSparkSpeculator()
runner = _make_runner(
is_last_pp_rank=False,
speculative_config=SimpleNamespace(
method="dspark",
draft_model_config=draft_model_config,
),
speculator=speculator,
)
mrv2.GPUModelRunner.load_model(runner)
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == [
(draft_model, draft_model_config, "dspark-draft (draft)"),
(target_model, runner.model_config, None),
]
assert speculator.eplb_state is runner.eplb_state
def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
events = []
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)