From a281ad32f0368caa2c6ebe38a0e59a75b3f22392 Mon Sep 17 00:00:00 2001 From: Ilya Markov Date: Mon, 12 Oct 2026 16:03:09 +0200 Subject: [PATCH] [EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549) Signed-off-by: Markov Ilya --- tests/distributed/test_eplb_spec_decode.py | 230 +++++++++++++++++- tests/models/test_deepseek_v4_mega_moe.py | 95 ++++++++ tests/models/test_dspark_mla.py | 51 ++++ tests/v1/worker/test_dspark_utils.py | 37 ++- .../worker/test_gpu_model_runner_v2_eplb.py | 83 ++++++- vllm/distributed/eplb/eplb_state.py | 64 ++--- vllm/models/deepseek_v4/common/eplb_util.py | 50 ++++ vllm/models/deepseek_v4/nvidia/dspark.py | 16 +- vllm/models/deepseek_v4/nvidia/model.py | 25 +- vllm/v1/worker/gpu/eplb_utils.py | 37 ++- .../v1/worker/gpu/spec_decode/dspark/utils.py | 21 +- vllm/v1/worker/gpu_model_runner.py | 32 ++- 12 files changed, 665 insertions(+), 76 deletions(-) create mode 100644 vllm/models/deepseek_v4/common/eplb_util.py diff --git a/tests/distributed/test_eplb_spec_decode.py b/tests/distributed/test_eplb_spec_decode.py index ced206a305..b49d06cd76 100644 --- a/tests/distributed/test_eplb_spec_decode.py +++ b/tests/distributed/test_eplb_spec_decode.py @@ -2,11 +2,21 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from __future__ import annotations +from types import SimpleNamespace + import lm_eval import pytest +import torch from tests.utils import large_gpu_mark +from vllm.distributed.eplb.eplb_state import EplbState +from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb from vllm.platforms import current_platform +from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config +from vllm.v1.worker.gpu.eplb_utils import ( + EPLBController, + draft_model_supports_eplb, +) def get_model_args( @@ -15,14 +25,18 @@ def get_model_args( spec_method: str, tp_size: int, model_max_len: int, + num_speculative_tokens: int = 1, use_async: bool = True, ) -> dict: speculative_config = { "method": spec_method, "model": spec_model_name, - "num_speculative_tokens": 1, + "num_speculative_tokens": num_speculative_tokens, "max_model_len": model_max_len, } + if spec_method == "dspark": + speculative_config["model"] = model_name + speculative_config["draft_sample_method"] = "probabilistic" eplb_config = { "num_redundant_experts": tp_size, "window_size": 128, @@ -42,6 +56,8 @@ def get_model_args( "enable_eplb": True, "max_model_len": model_max_len, } + if spec_method == "dspark": + model_args["trust_remote_code"] = True return model_args @@ -51,11 +67,190 @@ pytestmark = pytest.mark.skipif( ) +class FakeEplbState: + instances: list[FakeEplbState] = [] + + def __init__(self, parallel_config, device: torch.device): + self.add_model_calls: list[tuple[object, object, str | None]] = [] + FakeEplbState.instances.append(self) + + def add_model( + self, + model: object, + model_config: object, + *, + model_name: str | None = None, + ) -> None: + self.add_model_calls.append((model, model_config, model_name)) + + +def _make_dsv4_dspark_hf_config() -> DeepseekV4Config: + return DeepseekV4Config( + architectures=["DeepseekV4ForCausalLM"], + hidden_size=128, + num_hidden_layers=2, + n_routed_experts=8, + num_experts_per_tok=2, + num_hash_layers=0, + n_shared_experts=1, + moe_intermediate_size=128, + hc_mult=1, + hc_eps=1e-5, + rms_norm_eps=1e-5, + dspark_target_layer_ids=[0], + dspark_markov_rank=8, + index_topk=4, + head_dim=64, + num_attention_heads=4, + vocab_size=256, + n_mtp_layers=2, + enable_confidence_head=False, + compress_ratios=[1, 1], + ) + + +@pytest.fixture +def dspark_vllm_config(): + hf_config = _make_dsv4_dspark_hf_config() + model_config = SimpleNamespace( + dtype=torch.bfloat16, hf_config=hf_config, model="dspark" + ) + return SimpleNamespace( + model_config=model_config, + quant_config=None, + kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"), + parallel_config=SimpleNamespace( + pipeline_parallel_size=1, + tensor_parallel_size=1, + data_parallel_size=1, + enable_expert_parallel=True, + enable_eplb=True, + enable_elastic_ep=False, + eplb_config=SimpleNamespace(num_redundant_experts=4), + ), + scheduler_config=SimpleNamespace(max_num_batched_tokens=4), + compilation_config=SimpleNamespace(static_forward_context={}), + speculative_config=SimpleNamespace( + method="dspark", + draft_model_config=model_config, + ), + ) + + +def _make_moe_topology( + *, + num_routed_experts: int = 8, + num_redundant_experts: int = 4, + num_expert_groups: int = 1, +) -> SimpleNamespace: + num_physical_experts = num_routed_experts + num_redundant_experts + return SimpleNamespace( + num_routed_experts=num_routed_experts, + num_redundant_experts=num_redundant_experts, + num_physical_experts=num_physical_experts, + num_logical_experts=num_routed_experts, + num_expert_groups=num_expert_groups, + ) + + +def test_eplb_state_accepts_matching_dsv4_draft_and_target(): + draft = _make_moe_topology() + target = _make_moe_topology() + + EplbState.assert_confs_equal(draft, target) + + +def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts(): + draft = _make_moe_topology(num_redundant_experts=0) + target = _make_moe_topology(num_redundant_experts=4) + + with pytest.raises(RuntimeError, match="mismatch"): + EplbState.assert_confs_equal(draft, target) + + +@pytest.mark.parametrize( + ("model_type", "expected"), + [ + ("deepseek_v4", True), + ("deepseek_v41", False), + ], +) +def test_dspark_draft_supports_eplb( + dspark_vllm_config, model_type: str, expected: bool +): + draft_model_config = dspark_vllm_config.speculative_config.draft_model_config + draft_model_config.hf_config.model_type = model_type + assert dspark_draft_supports_eplb(draft_model_config) is expected + + +def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config): + assert not draft_model_supports_eplb( + dspark_vllm_config.speculative_config, + None, + ) + + +@pytest.mark.parametrize( + "model_type,load_dummy_weights,expected_registered", + [ + ("deepseek_v4", False, True), + ("deepseek_v4", True, False), + ("deepseek_v41", False, False), + ], +) +def test_eplb_dspark_draft_registration( + dspark_vllm_config, + monkeypatch: pytest.MonkeyPatch, + model_type, + load_dummy_weights: bool, + expected_registered: bool, +): + """Register V4 drafts; skip incompatible V4.1 drafts and dummy weights.""" + FakeEplbState.instances.clear() + monkeypatch.setattr( + "vllm.v1.worker.gpu.eplb_utils.EplbState", + FakeEplbState, + ) + monkeypatch.setattr( + "vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model", + lambda model: model, + ) + + draft_model_config = dspark_vllm_config.speculative_config.draft_model_config + draft_model_config.hf_config.model_type = model_type + draft = SimpleNamespace() + controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu")) + controller.prepare_load() + speculator = SimpleNamespace(model=draft, eplb_state=None) + + def set_eplb_state(state) -> None: + speculator.eplb_state = state + + speculator.set_eplb_state = set_eplb_state + + registered = controller.maybe_register_speculator( + speculator, + dspark_vllm_config.speculative_config, + load_dummy_weights=load_dummy_weights, + ) + + assert registered is expected_registered + assert controller.state is not None + if expected_registered: + assert controller.state.add_model_calls == [ + (draft, draft_model_config, "dspark (draft)") + ] + assert speculator.eplb_state is controller.state + else: + assert controller.state.add_model_calls == [] + assert speculator.eplb_state is None + + @pytest.mark.parametrize( "model_setup", [ pytest.param( - ("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86), + ("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1), marks=large_gpu_mark(min_gb=80), ), pytest.param( @@ -65,20 +260,42 @@ pytestmark = pytest.mark.skipif( "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", 4, 0.92, + 1, ), marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"), ), + pytest.param( + ( + "dspark", + "deepseek-ai/DeepSeek-V4-Flash-DSpark", + None, + 4, + 0.95, + 7, + ), + marks=large_gpu_mark(min_gb=80), + ), ], - ids=["qwen3_next_mtp", "llama4_eagle"], + ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"], ) def test_eplb_spec_decode( monkeypatch: pytest.MonkeyPatch, - model_setup: tuple[str, str, str, int, float], + model_setup: tuple[str, str, str | None, int, float, int], ): """Test the correctness of EPLB speculative decoding with GSM8K dataset. - Applicable to MoE models with mtp or eagle spec decode. + + Applicable to MoE models with mtp, eagle, or dspark spec decode. """ - method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup + ( + method, + model_name, + spec_model_name, + tp_size, + expected_gsm8k_value, + num_speculative_tokens, + ) = model_setup + if method == "dspark" and not current_platform.is_device_capability_family(100): + pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE") TASK = "gsm8k" FILTER = "exact_match,strict-match" @@ -90,6 +307,7 @@ def test_eplb_spec_decode( spec_method=method, tp_size=tp_size, model_max_len=4096, + num_speculative_tokens=num_speculative_tokens, ) results = lm_eval.simple_evaluate( diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index 76915b94a3..250a9d7291 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( from vllm.model_executor.layers.quantization.utils.fp8_utils import ( deepgemm_post_process_weight_scale_block, ) +from vllm.model_executor.models.interfaces import ( + get_mixture_of_experts_model, + is_mixture_of_experts, +) from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM from vllm.models.deepseek_v4.nvidia.model import ( DeepseekV4ForCausalLM, @@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE from vllm.platforms import current_platform +from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config from vllm.utils.deep_gemm import is_deep_gemm_supported from vllm.utils.torch_utils import set_default_torch_dtype @@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe(): assert calls == ["mtp", "dspark"] +@pytest.fixture +def v4_dspark_config(dist_init): + hf_config = DeepseekV4Config( + architectures=["DeepseekV4ForCausalLM"], + hidden_size=128, + num_hidden_layers=2, + n_routed_experts=8, + num_experts_per_tok=2, + num_hash_layers=0, + n_shared_experts=1, + moe_intermediate_size=128, + hidden_act="silu", + swiglu_limit=10.0, + norm_topk_prob=True, + topk_method="noaux_tc", + routed_scaling_factor=1.5, + hc_mult=1, + hc_eps=1e-5, + rms_norm_eps=1e-5, + dspark_target_layer_ids=[0], + dspark_markov_rank=8, + index_topk=4, + head_dim=64, + num_attention_heads=4, + vocab_size=256, + n_mtp_layers=2, + enable_confidence_head=False, + compress_ratios=[1, 1], + q_lora_rank=32, + o_lora_rank=32, + qk_rope_head_dim=16, + o_groups=4, + sliding_window=128, + hc_sinkhorn_iters=1, + ) + model_config = SimpleNamespace( + dtype=torch.bfloat16, + hf_config=hf_config, + max_model_len=2048, + ) + return SimpleNamespace( + model_config=model_config, + quant_config=None, + kernel_config=SimpleNamespace( + moe_backend="deep_gemm_mega_moe", + enable_jit_warmup=False, + ), + parallel_config=SimpleNamespace( + pipeline_parallel_size=1, + tensor_parallel_size=1, + data_parallel_size=1, + enable_expert_parallel=True, + enable_eplb=True, + eplb_config=SimpleNamespace(num_redundant_experts=4), + ), + attention_config=SimpleNamespace(backend=None), + cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"), + scheduler_config=SimpleNamespace(max_num_batched_tokens=4), + compilation_config=SimpleNamespace(static_forward_context={}), + speculative_config=SimpleNamespace(draft_model_config=model_config), + use_v2_model_runner=False, + ) + + +def test_dspark_draft_registers_mixture_of_experts( + v4_dspark_config, monkeypatch, dist_init +): + """DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration.""" + if not current_platform.is_device_capability_family(100): + pytest.skip("DeepGEMM MegaMoE requires SM100") + + hf_config = v4_dspark_config.model_config.hf_config + monkeypatch.setattr( + "vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config", + lambda: v4_dspark_config, + ) + + with ( + set_default_torch_dtype(v4_dspark_config.model_config.dtype), + torch.device("cuda"), + ): + draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config) + + assert is_mixture_of_experts(draft) + assert get_mixture_of_experts_model(draft) is draft + assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2 + assert draft.num_routed_experts == hf_config.n_routed_experts + assert draft.num_redundant_experts == 4 + + @pytest.mark.skipif( not torch.cuda.is_available(), reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.", diff --git a/tests/models/test_dspark_mla.py b/tests/models/test_dspark_mla.py index 53a93b940c..b2156af346 100644 --- a/tests/models/test_dspark_mla.py +++ b/tests/models/test_dspark_mla.py @@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales( torch.testing.assert_close(param, checkpoint_scale) +def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts( + monkeypatch, +): + from vllm.models.deepseek_v4.nvidia import dspark + + captured: dict[str, int] = {} + + def capture_expert_mapping(*args, **kwargs): + captured.update(kwargs) + return [] + + draft = SimpleNamespace( + config=SimpleNamespace( + num_attention_heads=4, + n_routed_experts=8, + expert_dtype="fp4", + ), + quant_config=None, + pad_shared_expert=False, + model=SimpleNamespace( + layers=[ + SimpleNamespace( + ffn=SimpleNamespace( + use_native_mega_moe=False, + use_mega_moe=False, + n_redundant_experts=4, + ) + ) + ], + confidence_head=None, + ), + named_parameters=lambda: [], + process_weights_after_loading=lambda: None, + ) + draft._remap_dspark_name = lambda name: ( + dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name) + ) + monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1) + monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + dspark, + "fused_moe_make_expert_params_mapping", + capture_expert_mapping, + ) + + dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, []) + + assert captured["num_experts"] == 8 + assert captured["num_redundant_experts"] == 4 + + def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer(): weights = [ ("mtp.0.attn.wkv.weight", torch.arange(4)), diff --git a/tests/v1/worker/test_dspark_utils.py b/tests/v1/worker/test_dspark_utils.py index 1014051d64..891ca56020 100644 --- a/tests/v1/worker/test_dspark_utils.py +++ b/tests/v1/worker/test_dspark_utils.py @@ -15,6 +15,16 @@ from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_con from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +@dataclass +class _FakeHFConfig: + model_type: str + + +@dataclass +class _FakeModelConfig: + hf_config: _FakeHFConfig + + @dataclass class _FakeEPLBConfig: num_redundant_experts: int = 0 @@ -37,12 +47,28 @@ class _FakeParallelConfig: raise ValueError("elastic EP requires EPLB") -def test_dspark_parallel_config_disables_eplb_atomically(): +@pytest.mark.parametrize( + ("model_type", "enable_eplb", "num_redundant_experts", "shares_eplb_config"), + [ + ("deepseek_v4", True, 32, True), + ("deepseek_v41", False, 0, False), + ], +) +def test_dspark_parallel_config_eplb_by_model_type( + model_type: str, + enable_eplb: bool, + num_redundant_experts: int, + shares_eplb_config: bool, +): target_config = _FakeParallelConfig() + draft_model_config = _FakeModelConfig( + hf_config=_FakeHFConfig(model_type=model_type) + ) draft_config = _get_dspark_parallel_config( target_config, tensor_parallel_size=4, + draft_model_config=draft_model_config, ) assert target_config.pipeline_parallel_size == 2 @@ -54,10 +80,13 @@ def test_dspark_parallel_config_disables_eplb_atomically(): assert draft_config is not target_config assert draft_config.pipeline_parallel_size == 1 assert draft_config.tensor_parallel_size == 4 - assert not draft_config.enable_eplb - assert draft_config.eplb_config.num_redundant_experts == 0 + assert draft_config.enable_eplb is enable_eplb + assert draft_config.eplb_config.num_redundant_experts == num_redundant_experts assert not draft_config.enable_elastic_ep - assert draft_config.eplb_config is not target_config.eplb_config + if shares_eplb_config: + assert draft_config.eplb_config is target_config.eplb_config + else: + assert draft_config.eplb_config is not target_config.eplb_config @pytest.mark.parametrize("pcp_size", [1, 4]) diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 6b38bfef96..b0acf69160 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -29,15 +29,21 @@ class FakeEplbState: def __init__(self, parallel_config: Any, device: torch.device): self.parallel_config = parallel_config self.device = device - self.add_model_calls: list[tuple[Any, Any]] = [] + self.add_model_calls: list[tuple[Any, Any, str | None]] = [] self.step_calls: list[tuple[bool, bool, bool]] = [] self.async_started = False self.is_async = True self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = [] FakeEplbState.instances.append(self) - def add_model(self, model: Any, model_config: Any) -> None: - self.add_model_calls.append((model, model_config)) + def add_model( + self, + model: Any, + model_config: Any, + *, + model_name: str | None = None, + ) -> None: + self.add_model_calls.append((model, model_config, model_name)) def step(self, is_dummy: bool, is_profile: bool, *, log_stats: bool) -> None: self.step_calls.append((is_dummy, is_profile, log_stats)) @@ -98,6 +104,15 @@ def _make_runner(**overrides: Any) -> Any: runner.execute_model_state = None for key, value in overrides.items(): setattr(runner, key, value) + spec_config = runner.speculative_config + runner._draft_workspace_lane = int( + spec_config is not None + and ( + spec_config.use_dspark() + if hasattr(spec_config, "use_dspark") + else getattr(spec_config, "method", None) == "dspark" + ) + ) return runner @@ -131,7 +146,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch): assert runner.model is model assert runner.model_state is not None assert runner.eplb_state is not None - assert runner.eplb_state.add_model_calls == [(model, runner.model_config)] + assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)] assert runner.eplb_state.async_started is True @@ -159,7 +174,7 @@ def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch): assert runner.load_config.load_format == "dummy" assert runner.eplb_state is not None # A scaling-up worker registers so it can join the EPLB communicator. - assert runner.eplb_state.add_model_calls == [(model, runner.model_config)] + assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)] assert runner.eplb_state.async_started is False @@ -174,6 +189,64 @@ def test_v2_setup_eplb_from_mapping_updates_state_in_place(): assert state.update_mapping_calls == [(runner.model_config, mapping)] +def test_v2_load_model_registers_dspark_speculator_with_eplb(monkeypatch): + FakeEplbState.instances.clear() + target_model = SimpleNamespace(is_moe=True) + draft_model = SimpleNamespace(is_moe=True) + draft_model_config = SimpleNamespace( + model="dspark-draft", + hf_config=SimpleNamespace(model_type="deepseek_v4"), + ) + + class FakeDSparkSpeculator: + def __init__(self): + self.model = draft_model + self.eplb_state = None + + def load_model(self, target_model): + return None + + def set_eplb_state(self, state) -> None: + self.eplb_state = state + + monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler) + monkeypatch.setattr(eplb, "EplbState", FakeEplbState) + monkeypatch.setattr(mrv2, "DraftModelSpeculator", FakeDSparkSpeculator) + monkeypatch.setattr( + mrv2, + "get_model_loader", + lambda load_config: SimpleNamespace(load_model=lambda **_: target_model), + ) + monkeypatch.setattr( + mrv2, + "init_model_state", + lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1), + ) + monkeypatch.setattr( + eplb, + "get_mixture_of_experts_model", + lambda model: model if getattr(model, "is_moe", False) else None, + ) + + speculator = FakeDSparkSpeculator() + runner = _make_runner( + is_last_pp_rank=False, + speculative_config=SimpleNamespace( + method="dspark", + draft_model_config=draft_model_config, + ), + speculator=speculator, + ) + mrv2.GPUModelRunner.load_model(runner) + + assert runner.eplb_state is not None + assert runner.eplb_state.add_model_calls == [ + (draft_model, draft_model_config, "dspark-draft (draft)"), + (target_model, runner.model_config, None), + ] + assert speculator.eplb_state is runner.eplb_state + + def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): events = [] runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 205226cde4..2d782c4179 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -323,39 +323,47 @@ class EplbState: """ if len(self.model_states) > 0: model = next(iter(self.model_states.values())).model - if ( - model.num_routed_experts != new_model.num_routed_experts - or model.num_redundant_experts != new_model.num_redundant_experts - or model.num_physical_experts != new_model.num_physical_experts - or model.num_logical_experts != new_model.num_logical_experts - or model.num_expert_groups != new_model.num_expert_groups - ): - raise RuntimeError( - "Model: {} " - "with config {} " - "{} {} {} {} " - "mismatch with new model {} " - "with config {} " - "{} {} {} {}".format( - type(model), - model.num_routed_experts, - model.num_redundant_experts, - model.num_physical_experts, - model.num_logical_experts, - model.num_expert_groups, - type(new_model), - new_model.num_routed_experts, - new_model.num_redundant_experts, - new_model.num_physical_experts, - new_model.num_logical_experts, - new_model.num_expert_groups, - ) + self.assert_confs_equal(model, new_model) + + @staticmethod + def assert_confs_equal( + model: MixtureOfExperts, new_model: MixtureOfExperts + ) -> None: + """Raise if the models have different expert parallel configurations.""" + if ( + model.num_routed_experts != new_model.num_routed_experts + or model.num_redundant_experts != new_model.num_redundant_experts + or model.num_physical_experts != new_model.num_physical_experts + or model.num_logical_experts != new_model.num_logical_experts + or model.num_expert_groups != new_model.num_expert_groups + ): + raise RuntimeError( + "Model: {} " + "with config {} " + "{} {} {} {} " + "mismatch with new model {} " + "with config {} " + "{} {} {} {}".format( + type(model), + model.num_routed_experts, + model.num_redundant_experts, + model.num_physical_experts, + model.num_logical_experts, + model.num_expert_groups, + type(new_model), + new_model.num_routed_experts, + new_model.num_redundant_experts, + new_model.num_physical_experts, + new_model.num_logical_experts, + new_model.num_expert_groups, ) + ) def add_model( self, model: MixtureOfExperts, model_config: ModelConfig, + model_name: str | None = None, ): """Build the initial EPLB state.""" self.validate_ep_configuration(model) @@ -506,7 +514,7 @@ class EplbState: dtype=torch.int64, device=self.device, ), - model_name=model_config.model, + model_name=model_name or model_config.model, model=model, expert_buffer=expert_buffer, rebalanced=False, diff --git a/vllm/models/deepseek_v4/common/eplb_util.py b/vllm/models/deepseek_v4/common/eplb_util.py new file mode 100644 index 0000000000..53c6d10024 --- /dev/null +++ b/vllm/models/deepseek_v4/common/eplb_util.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared EPLB MoE registration helpers for DeepSeek V4 family models.""" + +from collections.abc import Iterable +from typing import Any + +import torch.nn as nn + +from vllm.config import ModelConfig + + +def dspark_draft_supports_eplb(draft_model_config: ModelConfig) -> bool: + """Return whether a DSpark draft can share EPLB state with the target. + + Only DeepSeek-V4 DSpark drafts reuse the target expert layout. V4.1 drafts + use a smaller routed-expert count and cannot share EPLB state. + """ + return getattr(draft_model_config.hf_config, "model_type", None) == "deepseek_v4" + + +def collect_moe_layers( + moe_model: Any, + layers: Iterable[nn.Module], + config: object, + *, + decoder_layer_type: type[nn.Module], + moe_type: type[nn.Module], +) -> None: + """Populate ``MixtureOfExperts`` fields from decoder layers for EPLB. + + Works for both target backbones and DSpark draft stacks. Draft models pass + their shorter ``model.layers`` list; pipeline-parallel placeholder layers + are skipped via ``decoder_layer_type``. + """ + moe_model.num_expert_groups = getattr(config, "n_group", 1) + moe_model.moe_layers = [] + moe_model.moe_mlp_layers = [] + example_moe = None + for layer in layers: + if not isinstance(layer, decoder_layer_type): + continue + if not isinstance(layer.ffn, moe_type): + continue + example_moe = layer.ffn + moe_model.moe_mlp_layers.append(layer.ffn) + moe_model.moe_layers.append(layer.ffn.experts) + + moe_model.num_moe_layers = len(moe_model.moe_layers) + moe_model.extract_moe_parameters(example_moe) diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 54250185c8..2a40412e93 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -52,11 +52,14 @@ from vllm.models.common.ops.sequence_parallel import ( sp_padding_mask, sp_shard, ) +from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from .model import ( DeepseekV4DecoderLayer, + DeepseekV4MixtureOfExperts, DeepseekV4Model, + DeepseekV4MoE, _use_sequence_parallel, make_deepseek_v4_expert_params_mapping, prepare_mega_gate_routing_metadata, @@ -346,7 +349,7 @@ def _insert_context_kv( ) -class DSparkDeepseekV4ForCausalLM(nn.Module): +class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts): # Draft weights ship in the target checkpoint (mtp.*) without embed/head, so # load_dspark_model always aliases the target's. has_own_embed_tokens = False @@ -373,6 +376,16 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): prefix=maybe_prefix(prefix, "lm_head"), ) self.logits_processor = LogitsProcessor(self.config.vocab_size) + self.set_moe_parameters() + + def set_moe_parameters(self) -> None: + collect_moe_layers( + self, + self.model.layers, + self.config, + decoder_layer_type=DeepseekV4DecoderLayer, + moe_type=DeepseekV4MoE, + ) # --- Hooks used by the speculator ------------------------------------- @@ -451,6 +464,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): ckpt_down_proj_name="w2", ckpt_up_proj_name="w3", num_experts=self.config.n_routed_experts, + num_redundant_experts=first_layer.ffn.n_redundant_experts, ) expert_scale_suffix = ( ".weight_scale" diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index fc10cd07c1..ff4470dc4c 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -81,6 +81,7 @@ from vllm.models.common.ops.sequence_parallel import ( sp_shard, ) from vllm.models.deepseek_v4.attention import DeepseekV4Attention +from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import ( DeepseekV4FlashInferMLAAttention, DeepseekV4FlashInferSM120Attention, @@ -1841,23 +1842,13 @@ class DeepseekV4ForCausalLM( self.set_moe_parameters() def set_moe_parameters(self) -> None: - self.num_expert_groups = getattr(self.config, "n_group", 1) - self.num_moe_layers = self.config.num_hidden_layers - self.moe_layers: list[nn.Module] = [] - self.moe_mlp_layers: list[DeepseekV4MoE] = [] - example_moe: DeepseekV4MoE | None = None - for layer in self.model.layers: - if isinstance(layer, PPMissingLayer): - continue - if not isinstance(layer, DeepseekV4DecoderLayer): - continue - if isinstance(layer.ffn, DeepseekV4MoE): - example_moe = layer.ffn - self.moe_mlp_layers.append(layer.ffn) - self.moe_layers.append(layer.ffn.experts) - - self.num_moe_layers = len(self.moe_layers) - self.extract_moe_parameters(example_moe) + collect_moe_layers( + self, + self.model.layers, + self.config, + decoder_layer_type=DeepseekV4DecoderLayer, + moe_type=DeepseekV4MoE, + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) diff --git a/vllm/v1/worker/gpu/eplb_utils.py b/vllm/v1/worker/gpu/eplb_utils.py index e5eb31ad28..5c2c5d1412 100644 --- a/vllm/v1/worker/gpu/eplb_utils.py +++ b/vllm/v1/worker/gpu/eplb_utils.py @@ -13,8 +13,10 @@ from vllm.config import ModelConfig from vllm.distributed.eplb.eplb_state import EplbState from vllm.logger import init_logger from vllm.model_executor.models.interfaces import ( + MixtureOfExperts, get_mixture_of_experts_model, ) +from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb if TYPE_CHECKING: from vllm.v1.worker.gpu.model_runner import GPUModelRunner @@ -22,6 +24,25 @@ if TYPE_CHECKING: logger = init_logger(__name__) +def eplb_draft_model_name(model_config: ModelConfig) -> str: + """Return the EPLB step-log display name for a draft model.""" + return f"{model_config.model} (draft)" + + +def draft_model_supports_eplb( + speculative_config: Any | None, + draft_moe_model: MixtureOfExperts | None, +) -> bool: + """Return whether a draft MoE model should register with EPLB.""" + if draft_moe_model is None: + return False + if speculative_config is None or speculative_config.draft_model_config is None: + return False + if getattr(speculative_config, "method", None) == "dspark": + return dspark_draft_supports_eplb(speculative_config.draft_model_config) + return True + + def step_eplb_after(*, is_dummy: bool = False) -> Callable: """Step EPLB after a model runner method completes successfully.""" @@ -93,16 +114,26 @@ class EPLBController: draft_model = speculator.model draft_moe_model = get_mixture_of_experts_model(draft_model) - if draft_moe_model is None: + if not draft_model_supports_eplb(speculative_config, draft_moe_model): return False + assert draft_moe_model is not None assert not self.parallel_config.enable_elastic_ep, ( "Elastic EP is not supported with draft model." ) assert speculative_config is not None - assert speculative_config.draft_model_config is not None + draft_model_config = speculative_config.draft_model_config + assert draft_model_config is not None + logger.info_once( + "EPLB is enabled for MoE part of drafter model %s.", + draft_model_config.model, + ) assert self.state is not None - self.state.add_model(draft_moe_model, speculative_config.draft_model_config) + self.state.add_model( + draft_moe_model, + draft_model_config, + model_name=eplb_draft_model_name(draft_model_config), + ) speculator.set_eplb_state(self.state) self._has_registered_models = True return True diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/utils.py b/vllm/v1/worker/gpu/spec_decode/dspark/utils.py index 489d0da554..38108d4e04 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/utils.py @@ -6,6 +6,7 @@ import torch.nn as nn from vllm.config import ModelConfig, ParallelConfig, VllmConfig, replace from vllm.logger import init_logger from vllm.model_executor.model_loader.utils import get_draft_load_config +from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config @@ -35,11 +36,24 @@ def _resolve_dspark_attention_backend( def _get_dspark_parallel_config( parallel_config: ParallelConfig, tensor_parallel_size: int, + draft_model_config: ModelConfig, ) -> ParallelConfig: - if parallel_config.enable_eplb: + enable_draft_eplb = parallel_config.enable_eplb and dspark_draft_supports_eplb( + draft_model_config + ) + if parallel_config.enable_eplb and not enable_draft_eplb: logger.warning_once( - "EPLB is disabled for the DSpark draft model. EPLB remains enabled " - "for the target model." + "EPLB is disabled for the DSpark draft model (%s). EPLB remains " + "enabled for the target model.", + draft_model_config.hf_config.model_type, + ) + + if enable_draft_eplb: + return replace( + parallel_config, + pipeline_parallel_size=1, + tensor_parallel_size=tensor_parallel_size, + enable_elastic_ep=False, ) return replace( @@ -81,6 +95,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo parallel_config=_get_dspark_parallel_config( vllm_config.parallel_config, speculative_config.draft_parallel_config.tensor_parallel_size, + draft_model_config, ), attention_config=replace( vllm_config.attention_config, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 4db3f00b91..669b314d4c 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -206,6 +206,10 @@ from vllm.v1.worker.cp_utils import ( ) from vllm.v1.worker.dp_utils import coordinate_batch_across_dp from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin +from vllm.v1.worker.gpu.eplb_utils import ( + draft_model_supports_eplb, + eplb_draft_model_name, +) from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin @@ -5237,25 +5241,34 @@ class GPUModelRunner( logger.info_once("Loading drafter model...") if hasattr(self.drafter, "load_model"): self.drafter.load_model(self.model) + drafter_moe_model = None + if self.parallel_config.enable_eplb and hasattr( + self.drafter, "model" + ): + drafter_moe_model = get_mixture_of_experts_model( + self.drafter.model + ) + if not draft_model_supports_eplb( + self.vllm_config.speculative_config, + drafter_moe_model, + ): + drafter_moe_model = None + if ( self.parallel_config.enable_eplb and hasattr(self.drafter, "model") - and ( - drafter_moe_model := get_mixture_of_experts_model( - self.drafter.model - ) - ) - is not None + and drafter_moe_model is not None ): assert not self.parallel_config.enable_elastic_ep, ( "Elastic EP is not supported with drafter model." ) spec_config = self.vllm_config.speculative_config assert spec_config is not None - assert spec_config.draft_model_config is not None + draft_model_config = spec_config.draft_model_config + assert draft_model_config is not None logger.info_once( "EPLB is enabled for MoE part of drafter model %s.", - spec_config.draft_model_config.model, + draft_model_config.model, ) if self.eplb_state is None: self.eplb_state = EplbState( @@ -5263,7 +5276,8 @@ class GPUModelRunner( ) self.eplb_state.add_model( drafter_moe_model, - spec_config.draft_model_config, + draft_model_config, + model_name=eplb_draft_model_name(draft_model_config), ) assert hasattr(self.drafter, "set_eplb_state") self.drafter.set_eplb_state(self.eplb_state)