[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:03:09 -04:00
committed by GitHub
parent b3ee560ae8
commit a281ad32f0
12 changed files with 665 additions and 76 deletions
+224 -6
View File
@@ -2,11 +2,21 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import lm_eval
import pytest
import torch
from tests.utils import large_gpu_mark
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.v1.worker.gpu.eplb_utils import (
EPLBController,
draft_model_supports_eplb,
)
def get_model_args(
@@ -15,14 +25,18 @@ def get_model_args(
spec_method: str,
tp_size: int,
model_max_len: int,
num_speculative_tokens: int = 1,
use_async: bool = True,
) -> dict:
speculative_config = {
"method": spec_method,
"model": spec_model_name,
"num_speculative_tokens": 1,
"num_speculative_tokens": num_speculative_tokens,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
speculative_config["model"] = model_name
speculative_config["draft_sample_method"] = "probabilistic"
eplb_config = {
"num_redundant_experts": tp_size,
"window_size": 128,
@@ -42,6 +56,8 @@ def get_model_args(
"enable_eplb": True,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
model_args["trust_remote_code"] = True
return model_args
@@ -51,11 +67,190 @@ pytestmark = pytest.mark.skipif(
)
class FakeEplbState:
instances: list[FakeEplbState] = []
def __init__(self, parallel_config, device: torch.device):
self.add_model_calls: list[tuple[object, object, str | None]] = []
FakeEplbState.instances.append(self)
def add_model(
self,
model: object,
model_config: object,
*,
model_name: str | None = None,
) -> None:
self.add_model_calls.append((model, model_config, model_name))
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
return DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
compress_ratios=[1, 1],
)
@pytest.fixture
def dspark_vllm_config():
hf_config = _make_dsv4_dspark_hf_config()
model_config = SimpleNamespace(
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
enable_elastic_ep=False,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(
method="dspark",
draft_model_config=model_config,
),
)
def _make_moe_topology(
*,
num_routed_experts: int = 8,
num_redundant_experts: int = 4,
num_expert_groups: int = 1,
) -> SimpleNamespace:
num_physical_experts = num_routed_experts + num_redundant_experts
return SimpleNamespace(
num_routed_experts=num_routed_experts,
num_redundant_experts=num_redundant_experts,
num_physical_experts=num_physical_experts,
num_logical_experts=num_routed_experts,
num_expert_groups=num_expert_groups,
)
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
draft = _make_moe_topology()
target = _make_moe_topology()
EplbState.assert_confs_equal(draft, target)
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
draft = _make_moe_topology(num_redundant_experts=0)
target = _make_moe_topology(num_redundant_experts=4)
with pytest.raises(RuntimeError, match="mismatch"):
EplbState.assert_confs_equal(draft, target)
@pytest.mark.parametrize(
("model_type", "expected"),
[
("deepseek_v4", True),
("deepseek_v41", False),
],
)
def test_dspark_draft_supports_eplb(
dspark_vllm_config, model_type: str, expected: bool
):
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
assert dspark_draft_supports_eplb(draft_model_config) is expected
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
assert not draft_model_supports_eplb(
dspark_vllm_config.speculative_config,
None,
)
@pytest.mark.parametrize(
"model_type,load_dummy_weights,expected_registered",
[
("deepseek_v4", False, True),
("deepseek_v4", True, False),
("deepseek_v41", False, False),
],
)
def test_eplb_dspark_draft_registration(
dspark_vllm_config,
monkeypatch: pytest.MonkeyPatch,
model_type,
load_dummy_weights: bool,
expected_registered: bool,
):
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
FakeEplbState.instances.clear()
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.EplbState",
FakeEplbState,
)
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
lambda model: model,
)
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
draft = SimpleNamespace()
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
controller.prepare_load()
speculator = SimpleNamespace(model=draft, eplb_state=None)
def set_eplb_state(state) -> None:
speculator.eplb_state = state
speculator.set_eplb_state = set_eplb_state
registered = controller.maybe_register_speculator(
speculator,
dspark_vllm_config.speculative_config,
load_dummy_weights=load_dummy_weights,
)
assert registered is expected_registered
assert controller.state is not None
if expected_registered:
assert controller.state.add_model_calls == [
(draft, draft_model_config, "dspark (draft)")
]
assert speculator.eplb_state is controller.state
else:
assert controller.state.add_model_calls == []
assert speculator.eplb_state is None
@pytest.mark.parametrize(
"model_setup",
[
pytest.param(
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86),
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
marks=large_gpu_mark(min_gb=80),
),
pytest.param(
@@ -65,20 +260,42 @@ pytestmark = pytest.mark.skipif(
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
4,
0.92,
1,
),
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
),
pytest.param(
(
"dspark",
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
None,
4,
0.95,
7,
),
marks=large_gpu_mark(min_gb=80),
),
],
ids=["qwen3_next_mtp", "llama4_eagle"],
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
)
def test_eplb_spec_decode(
monkeypatch: pytest.MonkeyPatch,
model_setup: tuple[str, str, str, int, float],
model_setup: tuple[str, str, str | None, int, float, int],
):
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
Applicable to MoE models with mtp or eagle spec decode.
Applicable to MoE models with mtp, eagle, or dspark spec decode.
"""
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
(
method,
model_name,
spec_model_name,
tp_size,
expected_gsm8k_value,
num_speculative_tokens,
) = model_setup
if method == "dspark" and not current_platform.is_device_capability_family(100):
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
@@ -90,6 +307,7 @@ def test_eplb_spec_decode(
spec_method=method,
tp_size=tp_size,
model_max_len=4096,
num_speculative_tokens=num_speculative_tokens,
)
results = lm_eval.simple_evaluate(
+95
View File
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_weight_scale_block,
)
from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
is_mixture_of_experts,
)
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4ForCausalLM,
@@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
assert calls == ["mtp", "dspark"]
@pytest.fixture
def v4_dspark_config(dist_init):
hf_config = DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hidden_act="silu",
swiglu_limit=10.0,
norm_topk_prob=True,
topk_method="noaux_tc",
routed_scaling_factor=1.5,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
compress_ratios=[1, 1],
q_lora_rank=32,
o_lora_rank=32,
qk_rope_head_dim=16,
o_groups=4,
sliding_window=128,
hc_sinkhorn_iters=1,
)
model_config = SimpleNamespace(
dtype=torch.bfloat16,
hf_config=hf_config,
max_model_len=2048,
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(
moe_backend="deep_gemm_mega_moe",
enable_jit_warmup=False,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(draft_model_config=model_config),
use_v2_model_runner=False,
)
def test_dspark_draft_registers_mixture_of_experts(
v4_dspark_config, monkeypatch, dist_init
):
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
if not current_platform.is_device_capability_family(100):
pytest.skip("DeepGEMM MegaMoE requires SM100")
hf_config = v4_dspark_config.model_config.hf_config
monkeypatch.setattr(
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
lambda: v4_dspark_config,
)
with (
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
torch.device("cuda"),
):
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
assert is_mixture_of_experts(draft)
assert get_mixture_of_experts_model(draft) is draft
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
assert draft.num_routed_experts == hf_config.n_routed_experts
assert draft.num_redundant_experts == 4
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
+51
View File
@@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales(
torch.testing.assert_close(param, checkpoint_scale)
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
monkeypatch,
):
from vllm.models.deepseek_v4.nvidia import dspark
captured: dict[str, int] = {}
def capture_expert_mapping(*args, **kwargs):
captured.update(kwargs)
return []
draft = SimpleNamespace(
config=SimpleNamespace(
num_attention_heads=4,
n_routed_experts=8,
expert_dtype="fp4",
),
quant_config=None,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[
SimpleNamespace(
ffn=SimpleNamespace(
use_native_mega_moe=False,
use_mega_moe=False,
n_redundant_experts=4,
)
)
],
confidence_head=None,
),
named_parameters=lambda: [],
process_weights_after_loading=lambda: None,
)
draft._remap_dspark_name = lambda name: (
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
dspark,
"fused_moe_make_expert_params_mapping",
capture_expert_mapping,
)
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
assert captured["num_experts"] == 8
assert captured["num_redundant_experts"] == 4
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
weights = [
("mtp.0.attn.wkv.weight", torch.arange(4)),
+33 -4
View File
@@ -15,6 +15,16 @@ from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_con
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@dataclass
class _FakeHFConfig:
model_type: str
@dataclass
class _FakeModelConfig:
hf_config: _FakeHFConfig
@dataclass
class _FakeEPLBConfig:
num_redundant_experts: int = 0
@@ -37,12 +47,28 @@ class _FakeParallelConfig:
raise ValueError("elastic EP requires EPLB")
def test_dspark_parallel_config_disables_eplb_atomically():
@pytest.mark.parametrize(
("model_type", "enable_eplb", "num_redundant_experts", "shares_eplb_config"),
[
("deepseek_v4", True, 32, True),
("deepseek_v41", False, 0, False),
],
)
def test_dspark_parallel_config_eplb_by_model_type(
model_type: str,
enable_eplb: bool,
num_redundant_experts: int,
shares_eplb_config: bool,
):
target_config = _FakeParallelConfig()
draft_model_config = _FakeModelConfig(
hf_config=_FakeHFConfig(model_type=model_type)
)
draft_config = _get_dspark_parallel_config(
target_config,
tensor_parallel_size=4,
draft_model_config=draft_model_config,
)
assert target_config.pipeline_parallel_size == 2
@@ -54,10 +80,13 @@ def test_dspark_parallel_config_disables_eplb_atomically():
assert draft_config is not target_config
assert draft_config.pipeline_parallel_size == 1
assert draft_config.tensor_parallel_size == 4
assert not draft_config.enable_eplb
assert draft_config.eplb_config.num_redundant_experts == 0
assert draft_config.enable_eplb is enable_eplb
assert draft_config.eplb_config.num_redundant_experts == num_redundant_experts
assert not draft_config.enable_elastic_ep
assert draft_config.eplb_config is not target_config.eplb_config
if shares_eplb_config:
assert draft_config.eplb_config is target_config.eplb_config
else:
assert draft_config.eplb_config is not target_config.eplb_config
@pytest.mark.parametrize("pcp_size", [1, 4])
@@ -29,15 +29,21 @@ class FakeEplbState:
def __init__(self, parallel_config: Any, device: torch.device):
self.parallel_config = parallel_config
self.device = device
self.add_model_calls: list[tuple[Any, Any]] = []
self.add_model_calls: list[tuple[Any, Any, str | None]] = []
self.step_calls: list[tuple[bool, bool, bool]] = []
self.async_started = False
self.is_async = True
self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = []
FakeEplbState.instances.append(self)
def add_model(self, model: Any, model_config: Any) -> None:
self.add_model_calls.append((model, model_config))
def add_model(
self,
model: Any,
model_config: Any,
*,
model_name: str | None = None,
) -> None:
self.add_model_calls.append((model, model_config, model_name))
def step(self, is_dummy: bool, is_profile: bool, *, log_stats: bool) -> None:
self.step_calls.append((is_dummy, is_profile, log_stats))
@@ -98,6 +104,15 @@ def _make_runner(**overrides: Any) -> Any:
runner.execute_model_state = None
for key, value in overrides.items():
setattr(runner, key, value)
spec_config = runner.speculative_config
runner._draft_workspace_lane = int(
spec_config is not None
and (
spec_config.use_dspark()
if hasattr(spec_config, "use_dspark")
else getattr(spec_config, "method", None) == "dspark"
)
)
return runner
@@ -131,7 +146,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
assert runner.model is model
assert runner.model_state is not None
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
assert runner.eplb_state.async_started is True
@@ -159,7 +174,7 @@ def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch):
assert runner.load_config.load_format == "dummy"
assert runner.eplb_state is not None
# A scaling-up worker registers so it can join the EPLB communicator.
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
assert runner.eplb_state.async_started is False
@@ -174,6 +189,64 @@ def test_v2_setup_eplb_from_mapping_updates_state_in_place():
assert state.update_mapping_calls == [(runner.model_config, mapping)]
def test_v2_load_model_registers_dspark_speculator_with_eplb(monkeypatch):
FakeEplbState.instances.clear()
target_model = SimpleNamespace(is_moe=True)
draft_model = SimpleNamespace(is_moe=True)
draft_model_config = SimpleNamespace(
model="dspark-draft",
hf_config=SimpleNamespace(model_type="deepseek_v4"),
)
class FakeDSparkSpeculator:
def __init__(self):
self.model = draft_model
self.eplb_state = None
def load_model(self, target_model):
return None
def set_eplb_state(self, state) -> None:
self.eplb_state = state
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
monkeypatch.setattr(mrv2, "DraftModelSpeculator", FakeDSparkSpeculator)
monkeypatch.setattr(
mrv2,
"get_model_loader",
lambda load_config: SimpleNamespace(load_model=lambda **_: target_model),
)
monkeypatch.setattr(
mrv2,
"init_model_state",
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
)
monkeypatch.setattr(
eplb,
"get_mixture_of_experts_model",
lambda model: model if getattr(model, "is_moe", False) else None,
)
speculator = FakeDSparkSpeculator()
runner = _make_runner(
is_last_pp_rank=False,
speculative_config=SimpleNamespace(
method="dspark",
draft_model_config=draft_model_config,
),
speculator=speculator,
)
mrv2.GPUModelRunner.load_model(runner)
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == [
(draft_model, draft_model_config, "dspark-draft (draft)"),
(target_model, runner.model_config, None),
]
assert speculator.eplb_state is runner.eplb_state
def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
events = []
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
+36 -28
View File
@@ -323,39 +323,47 @@ class EplbState:
"""
if len(self.model_states) > 0:
model = next(iter(self.model_states.values())).model
if (
model.num_routed_experts != new_model.num_routed_experts
or model.num_redundant_experts != new_model.num_redundant_experts
or model.num_physical_experts != new_model.num_physical_experts
or model.num_logical_experts != new_model.num_logical_experts
or model.num_expert_groups != new_model.num_expert_groups
):
raise RuntimeError(
"Model: {} "
"with config {} "
"{} {} {} {} "
"mismatch with new model {} "
"with config {} "
"{} {} {} {}".format(
type(model),
model.num_routed_experts,
model.num_redundant_experts,
model.num_physical_experts,
model.num_logical_experts,
model.num_expert_groups,
type(new_model),
new_model.num_routed_experts,
new_model.num_redundant_experts,
new_model.num_physical_experts,
new_model.num_logical_experts,
new_model.num_expert_groups,
)
self.assert_confs_equal(model, new_model)
@staticmethod
def assert_confs_equal(
model: MixtureOfExperts, new_model: MixtureOfExperts
) -> None:
"""Raise if the models have different expert parallel configurations."""
if (
model.num_routed_experts != new_model.num_routed_experts
or model.num_redundant_experts != new_model.num_redundant_experts
or model.num_physical_experts != new_model.num_physical_experts
or model.num_logical_experts != new_model.num_logical_experts
or model.num_expert_groups != new_model.num_expert_groups
):
raise RuntimeError(
"Model: {} "
"with config {} "
"{} {} {} {} "
"mismatch with new model {} "
"with config {} "
"{} {} {} {}".format(
type(model),
model.num_routed_experts,
model.num_redundant_experts,
model.num_physical_experts,
model.num_logical_experts,
model.num_expert_groups,
type(new_model),
new_model.num_routed_experts,
new_model.num_redundant_experts,
new_model.num_physical_experts,
new_model.num_logical_experts,
new_model.num_expert_groups,
)
)
def add_model(
self,
model: MixtureOfExperts,
model_config: ModelConfig,
model_name: str | None = None,
):
"""Build the initial EPLB state."""
self.validate_ep_configuration(model)
@@ -506,7 +514,7 @@ class EplbState:
dtype=torch.int64,
device=self.device,
),
model_name=model_config.model,
model_name=model_name or model_config.model,
model=model,
expert_buffer=expert_buffer,
rebalanced=False,
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared EPLB MoE registration helpers for DeepSeek V4 family models."""
from collections.abc import Iterable
from typing import Any
import torch.nn as nn
from vllm.config import ModelConfig
def dspark_draft_supports_eplb(draft_model_config: ModelConfig) -> bool:
"""Return whether a DSpark draft can share EPLB state with the target.
Only DeepSeek-V4 DSpark drafts reuse the target expert layout. V4.1 drafts
use a smaller routed-expert count and cannot share EPLB state.
"""
return getattr(draft_model_config.hf_config, "model_type", None) == "deepseek_v4"
def collect_moe_layers(
moe_model: Any,
layers: Iterable[nn.Module],
config: object,
*,
decoder_layer_type: type[nn.Module],
moe_type: type[nn.Module],
) -> None:
"""Populate ``MixtureOfExperts`` fields from decoder layers for EPLB.
Works for both target backbones and DSpark draft stacks. Draft models pass
their shorter ``model.layers`` list; pipeline-parallel placeholder layers
are skipped via ``decoder_layer_type``.
"""
moe_model.num_expert_groups = getattr(config, "n_group", 1)
moe_model.moe_layers = []
moe_model.moe_mlp_layers = []
example_moe = None
for layer in layers:
if not isinstance(layer, decoder_layer_type):
continue
if not isinstance(layer.ffn, moe_type):
continue
example_moe = layer.ffn
moe_model.moe_mlp_layers.append(layer.ffn)
moe_model.moe_layers.append(layer.ffn.experts)
moe_model.num_moe_layers = len(moe_model.moe_layers)
moe_model.extract_moe_parameters(example_moe)
+15 -1
View File
@@ -52,11 +52,14 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4MixtureOfExperts,
DeepseekV4Model,
DeepseekV4MoE,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
@@ -346,7 +349,7 @@ def _insert_context_kv(
)
class DSparkDeepseekV4ForCausalLM(nn.Module):
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
# load_dspark_model always aliases the target's.
has_own_embed_tokens = False
@@ -373,6 +376,16 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(self.config.vocab_size)
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
collect_moe_layers(
self,
self.model.layers,
self.config,
decoder_layer_type=DeepseekV4DecoderLayer,
moe_type=DeepseekV4MoE,
)
# --- Hooks used by the speculator -------------------------------------
@@ -451,6 +464,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
num_redundant_experts=first_layer.ffn.n_redundant_experts,
)
expert_scale_suffix = (
".weight_scale"
+8 -17
View File
@@ -81,6 +81,7 @@ from vllm.models.common.ops.sequence_parallel import (
sp_shard,
)
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
DeepseekV4FlashInferMLAAttention,
DeepseekV4FlashInferSM120Attention,
@@ -1841,23 +1842,13 @@ class DeepseekV4ForCausalLM(
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
self.num_expert_groups = getattr(self.config, "n_group", 1)
self.num_moe_layers = self.config.num_hidden_layers
self.moe_layers: list[nn.Module] = []
self.moe_mlp_layers: list[DeepseekV4MoE] = []
example_moe: DeepseekV4MoE | None = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
if not isinstance(layer, DeepseekV4DecoderLayer):
continue
if isinstance(layer.ffn, DeepseekV4MoE):
example_moe = layer.ffn
self.moe_mlp_layers.append(layer.ffn)
self.moe_layers.append(layer.ffn.experts)
self.num_moe_layers = len(self.moe_layers)
self.extract_moe_parameters(example_moe)
collect_moe_layers(
self,
self.model.layers,
self.config,
decoder_layer_type=DeepseekV4DecoderLayer,
moe_type=DeepseekV4MoE,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
+34 -3
View File
@@ -13,8 +13,10 @@ from vllm.config import ModelConfig
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MixtureOfExperts,
get_mixture_of_experts_model,
)
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
@@ -22,6 +24,25 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def eplb_draft_model_name(model_config: ModelConfig) -> str:
"""Return the EPLB step-log display name for a draft model."""
return f"{model_config.model} (draft)"
def draft_model_supports_eplb(
speculative_config: Any | None,
draft_moe_model: MixtureOfExperts | None,
) -> bool:
"""Return whether a draft MoE model should register with EPLB."""
if draft_moe_model is None:
return False
if speculative_config is None or speculative_config.draft_model_config is None:
return False
if getattr(speculative_config, "method", None) == "dspark":
return dspark_draft_supports_eplb(speculative_config.draft_model_config)
return True
def step_eplb_after(*, is_dummy: bool = False) -> Callable:
"""Step EPLB after a model runner method completes successfully."""
@@ -93,16 +114,26 @@ class EPLBController:
draft_model = speculator.model
draft_moe_model = get_mixture_of_experts_model(draft_model)
if draft_moe_model is None:
if not draft_model_supports_eplb(speculative_config, draft_moe_model):
return False
assert draft_moe_model is not None
assert not self.parallel_config.enable_elastic_ep, (
"Elastic EP is not supported with draft model."
)
assert speculative_config is not None
assert speculative_config.draft_model_config is not None
draft_model_config = speculative_config.draft_model_config
assert draft_model_config is not None
logger.info_once(
"EPLB is enabled for MoE part of drafter model %s.",
draft_model_config.model,
)
assert self.state is not None
self.state.add_model(draft_moe_model, speculative_config.draft_model_config)
self.state.add_model(
draft_moe_model,
draft_model_config,
model_name=eplb_draft_model_name(draft_model_config),
)
speculator.set_eplb_state(self.state)
self._has_registered_models = True
return True
+18 -3
View File
@@ -6,6 +6,7 @@ import torch.nn as nn
from vllm.config import ModelConfig, ParallelConfig, VllmConfig, replace
from vllm.logger import init_logger
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
@@ -35,11 +36,24 @@ def _resolve_dspark_attention_backend(
def _get_dspark_parallel_config(
parallel_config: ParallelConfig,
tensor_parallel_size: int,
draft_model_config: ModelConfig,
) -> ParallelConfig:
if parallel_config.enable_eplb:
enable_draft_eplb = parallel_config.enable_eplb and dspark_draft_supports_eplb(
draft_model_config
)
if parallel_config.enable_eplb and not enable_draft_eplb:
logger.warning_once(
"EPLB is disabled for the DSpark draft model. EPLB remains enabled "
"for the target model."
"EPLB is disabled for the DSpark draft model (%s). EPLB remains "
"enabled for the target model.",
draft_model_config.hf_config.model_type,
)
if enable_draft_eplb:
return replace(
parallel_config,
pipeline_parallel_size=1,
tensor_parallel_size=tensor_parallel_size,
enable_elastic_ep=False,
)
return replace(
@@ -81,6 +95,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
parallel_config=_get_dspark_parallel_config(
vllm_config.parallel_config,
speculative_config.draft_parallel_config.tensor_parallel_size,
draft_model_config,
),
attention_config=replace(
vllm_config.attention_config,
+23 -9
View File
@@ -206,6 +206,10 @@ from vllm.v1.worker.cp_utils import (
)
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
from vllm.v1.worker.gpu.eplb_utils import (
draft_model_supports_eplb,
eplb_draft_model_name,
)
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin
@@ -5237,25 +5241,34 @@ class GPUModelRunner(
logger.info_once("Loading drafter model...")
if hasattr(self.drafter, "load_model"):
self.drafter.load_model(self.model)
drafter_moe_model = None
if self.parallel_config.enable_eplb and hasattr(
self.drafter, "model"
):
drafter_moe_model = get_mixture_of_experts_model(
self.drafter.model
)
if not draft_model_supports_eplb(
self.vllm_config.speculative_config,
drafter_moe_model,
):
drafter_moe_model = None
if (
self.parallel_config.enable_eplb
and hasattr(self.drafter, "model")
and (
drafter_moe_model := get_mixture_of_experts_model(
self.drafter.model
)
)
is not None
and drafter_moe_model is not None
):
assert not self.parallel_config.enable_elastic_ep, (
"Elastic EP is not supported with drafter model."
)
spec_config = self.vllm_config.speculative_config
assert spec_config is not None
assert spec_config.draft_model_config is not None
draft_model_config = spec_config.draft_model_config
assert draft_model_config is not None
logger.info_once(
"EPLB is enabled for MoE part of drafter model %s.",
spec_config.draft_model_config.model,
draft_model_config.model,
)
if self.eplb_state is None:
self.eplb_state = EplbState(
@@ -5263,7 +5276,8 @@ class GPUModelRunner(
)
self.eplb_state.add_model(
drafter_moe_model,
spec_config.draft_model_config,
draft_model_config,
model_name=eplb_draft_model_name(draft_model_config),
)
assert hasattr(self.drafter, "set_eplb_state")
self.drafter.set_eplb_state(self.eplb_state)