[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -2,11 +2,21 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.v1.worker.gpu.eplb_utils import (
|
||||
EPLBController,
|
||||
draft_model_supports_eplb,
|
||||
)
|
||||
|
||||
|
||||
def get_model_args(
|
||||
@@ -15,14 +25,18 @@ def get_model_args(
|
||||
spec_method: str,
|
||||
tp_size: int,
|
||||
model_max_len: int,
|
||||
num_speculative_tokens: int = 1,
|
||||
use_async: bool = True,
|
||||
) -> dict:
|
||||
speculative_config = {
|
||||
"method": spec_method,
|
||||
"model": spec_model_name,
|
||||
"num_speculative_tokens": 1,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
speculative_config["model"] = model_name
|
||||
speculative_config["draft_sample_method"] = "probabilistic"
|
||||
eplb_config = {
|
||||
"num_redundant_experts": tp_size,
|
||||
"window_size": 128,
|
||||
@@ -42,6 +56,8 @@ def get_model_args(
|
||||
"enable_eplb": True,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
model_args["trust_remote_code"] = True
|
||||
return model_args
|
||||
|
||||
|
||||
@@ -51,11 +67,190 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
class FakeEplbState:
|
||||
instances: list[FakeEplbState] = []
|
||||
|
||||
def __init__(self, parallel_config, device: torch.device):
|
||||
self.add_model_calls: list[tuple[object, object, str | None]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(
|
||||
self,
|
||||
model: object,
|
||||
model_config: object,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
|
||||
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
|
||||
return DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
compress_ratios=[1, 1],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dspark_vllm_config():
|
||||
hf_config = _make_dsv4_dspark_hf_config()
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
enable_elastic_ep=False,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=model_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_moe_topology(
|
||||
*,
|
||||
num_routed_experts: int = 8,
|
||||
num_redundant_experts: int = 4,
|
||||
num_expert_groups: int = 1,
|
||||
) -> SimpleNamespace:
|
||||
num_physical_experts = num_routed_experts + num_redundant_experts
|
||||
return SimpleNamespace(
|
||||
num_routed_experts=num_routed_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_logical_experts=num_routed_experts,
|
||||
num_expert_groups=num_expert_groups,
|
||||
)
|
||||
|
||||
|
||||
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
|
||||
draft = _make_moe_topology()
|
||||
target = _make_moe_topology()
|
||||
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
|
||||
draft = _make_moe_topology(num_redundant_experts=0)
|
||||
target = _make_moe_topology(num_redundant_experts=4)
|
||||
|
||||
with pytest.raises(RuntimeError, match="mismatch"):
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "expected"),
|
||||
[
|
||||
("deepseek_v4", True),
|
||||
("deepseek_v41", False),
|
||||
],
|
||||
)
|
||||
def test_dspark_draft_supports_eplb(
|
||||
dspark_vllm_config, model_type: str, expected: bool
|
||||
):
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
assert dspark_draft_supports_eplb(draft_model_config) is expected
|
||||
|
||||
|
||||
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
|
||||
assert not draft_model_supports_eplb(
|
||||
dspark_vllm_config.speculative_config,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type,load_dummy_weights,expected_registered",
|
||||
[
|
||||
("deepseek_v4", False, True),
|
||||
("deepseek_v4", True, False),
|
||||
("deepseek_v41", False, False),
|
||||
],
|
||||
)
|
||||
def test_eplb_dspark_draft_registration(
|
||||
dspark_vllm_config,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type,
|
||||
load_dummy_weights: bool,
|
||||
expected_registered: bool,
|
||||
):
|
||||
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
|
||||
FakeEplbState.instances.clear()
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.EplbState",
|
||||
FakeEplbState,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
|
||||
lambda model: model,
|
||||
)
|
||||
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
draft = SimpleNamespace()
|
||||
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
|
||||
controller.prepare_load()
|
||||
speculator = SimpleNamespace(model=draft, eplb_state=None)
|
||||
|
||||
def set_eplb_state(state) -> None:
|
||||
speculator.eplb_state = state
|
||||
|
||||
speculator.set_eplb_state = set_eplb_state
|
||||
|
||||
registered = controller.maybe_register_speculator(
|
||||
speculator,
|
||||
dspark_vllm_config.speculative_config,
|
||||
load_dummy_weights=load_dummy_weights,
|
||||
)
|
||||
|
||||
assert registered is expected_registered
|
||||
assert controller.state is not None
|
||||
if expected_registered:
|
||||
assert controller.state.add_model_calls == [
|
||||
(draft, draft_model_config, "dspark (draft)")
|
||||
]
|
||||
assert speculator.eplb_state is controller.state
|
||||
else:
|
||||
assert controller.state.add_model_calls == []
|
||||
assert speculator.eplb_state is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_setup",
|
||||
[
|
||||
pytest.param(
|
||||
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86),
|
||||
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
pytest.param(
|
||||
@@ -65,20 +260,42 @@ pytestmark = pytest.mark.skipif(
|
||||
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
|
||||
4,
|
||||
0.92,
|
||||
1,
|
||||
),
|
||||
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
"dspark",
|
||||
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
|
||||
None,
|
||||
4,
|
||||
0.95,
|
||||
7,
|
||||
),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
],
|
||||
ids=["qwen3_next_mtp", "llama4_eagle"],
|
||||
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
|
||||
)
|
||||
def test_eplb_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str, int, float],
|
||||
model_setup: tuple[str, str, str | None, int, float, int],
|
||||
):
|
||||
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
Applicable to MoE models with mtp or eagle spec decode.
|
||||
|
||||
Applicable to MoE models with mtp, eagle, or dspark spec decode.
|
||||
"""
|
||||
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
|
||||
(
|
||||
method,
|
||||
model_name,
|
||||
spec_model_name,
|
||||
tp_size,
|
||||
expected_gsm8k_value,
|
||||
num_speculative_tokens,
|
||||
) = model_setup
|
||||
if method == "dspark" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
|
||||
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
@@ -90,6 +307,7 @@ def test_eplb_spec_decode(
|
||||
spec_method=method,
|
||||
tp_size=tp_size,
|
||||
model_max_len=4096,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
|
||||
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_weight_scale_block,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
is_mixture_of_experts,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4ForCausalLM,
|
||||
@@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
@@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
|
||||
assert calls == ["mtp", "dspark"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def v4_dspark_config(dist_init):
|
||||
hf_config = DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hidden_act="silu",
|
||||
swiglu_limit=10.0,
|
||||
norm_topk_prob=True,
|
||||
topk_method="noaux_tc",
|
||||
routed_scaling_factor=1.5,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
compress_ratios=[1, 1],
|
||||
q_lora_rank=32,
|
||||
o_lora_rank=32,
|
||||
qk_rope_head_dim=16,
|
||||
o_groups=4,
|
||||
sliding_window=128,
|
||||
hc_sinkhorn_iters=1,
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
hf_config=hf_config,
|
||||
max_model_len=2048,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(
|
||||
moe_backend="deep_gemm_mega_moe",
|
||||
enable_jit_warmup=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(draft_model_config=model_config),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
|
||||
def test_dspark_draft_registers_mixture_of_experts(
|
||||
v4_dspark_config, monkeypatch, dist_init
|
||||
):
|
||||
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DeepGEMM MegaMoE requires SM100")
|
||||
|
||||
hf_config = v4_dspark_config.model_config.hf_config
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
|
||||
lambda: v4_dspark_config,
|
||||
)
|
||||
|
||||
with (
|
||||
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
|
||||
torch.device("cuda"),
|
||||
):
|
||||
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
|
||||
|
||||
assert is_mixture_of_experts(draft)
|
||||
assert get_mixture_of_experts_model(draft) is draft
|
||||
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
|
||||
assert draft.num_routed_experts == hf_config.n_routed_experts
|
||||
assert draft.num_redundant_experts == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
|
||||
@@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales(
|
||||
torch.testing.assert_close(param, checkpoint_scale)
|
||||
|
||||
|
||||
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
|
||||
monkeypatch,
|
||||
):
|
||||
from vllm.models.deepseek_v4.nvidia import dspark
|
||||
|
||||
captured: dict[str, int] = {}
|
||||
|
||||
def capture_expert_mapping(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return []
|
||||
|
||||
draft = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
num_attention_heads=4,
|
||||
n_routed_experts=8,
|
||||
expert_dtype="fp4",
|
||||
),
|
||||
quant_config=None,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[
|
||||
SimpleNamespace(
|
||||
ffn=SimpleNamespace(
|
||||
use_native_mega_moe=False,
|
||||
use_mega_moe=False,
|
||||
n_redundant_experts=4,
|
||||
)
|
||||
)
|
||||
],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [],
|
||||
process_weights_after_loading=lambda: None,
|
||||
)
|
||||
draft._remap_dspark_name = lambda name: (
|
||||
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
|
||||
)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
dspark,
|
||||
"fused_moe_make_expert_params_mapping",
|
||||
capture_expert_mapping,
|
||||
)
|
||||
|
||||
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
|
||||
|
||||
assert captured["num_experts"] == 8
|
||||
assert captured["num_redundant_experts"] == 4
|
||||
|
||||
|
||||
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
|
||||
weights = [
|
||||
("mtp.0.attn.wkv.weight", torch.arange(4)),
|
||||
|
||||
@@ -15,6 +15,16 @@ from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_con
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeHFConfig:
|
||||
model_type: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeModelConfig:
|
||||
hf_config: _FakeHFConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeEPLBConfig:
|
||||
num_redundant_experts: int = 0
|
||||
@@ -37,12 +47,28 @@ class _FakeParallelConfig:
|
||||
raise ValueError("elastic EP requires EPLB")
|
||||
|
||||
|
||||
def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "enable_eplb", "num_redundant_experts", "shares_eplb_config"),
|
||||
[
|
||||
("deepseek_v4", True, 32, True),
|
||||
("deepseek_v41", False, 0, False),
|
||||
],
|
||||
)
|
||||
def test_dspark_parallel_config_eplb_by_model_type(
|
||||
model_type: str,
|
||||
enable_eplb: bool,
|
||||
num_redundant_experts: int,
|
||||
shares_eplb_config: bool,
|
||||
):
|
||||
target_config = _FakeParallelConfig()
|
||||
draft_model_config = _FakeModelConfig(
|
||||
hf_config=_FakeHFConfig(model_type=model_type)
|
||||
)
|
||||
|
||||
draft_config = _get_dspark_parallel_config(
|
||||
target_config,
|
||||
tensor_parallel_size=4,
|
||||
draft_model_config=draft_model_config,
|
||||
)
|
||||
|
||||
assert target_config.pipeline_parallel_size == 2
|
||||
@@ -54,10 +80,13 @@ def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
assert draft_config is not target_config
|
||||
assert draft_config.pipeline_parallel_size == 1
|
||||
assert draft_config.tensor_parallel_size == 4
|
||||
assert not draft_config.enable_eplb
|
||||
assert draft_config.eplb_config.num_redundant_experts == 0
|
||||
assert draft_config.enable_eplb is enable_eplb
|
||||
assert draft_config.eplb_config.num_redundant_experts == num_redundant_experts
|
||||
assert not draft_config.enable_elastic_ep
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
if shares_eplb_config:
|
||||
assert draft_config.eplb_config is target_config.eplb_config
|
||||
else:
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
|
||||
@@ -29,15 +29,21 @@ class FakeEplbState:
|
||||
def __init__(self, parallel_config: Any, device: torch.device):
|
||||
self.parallel_config = parallel_config
|
||||
self.device = device
|
||||
self.add_model_calls: list[tuple[Any, Any]] = []
|
||||
self.add_model_calls: list[tuple[Any, Any, str | None]] = []
|
||||
self.step_calls: list[tuple[bool, bool, bool]] = []
|
||||
self.async_started = False
|
||||
self.is_async = True
|
||||
self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(self, model: Any, model_config: Any) -> None:
|
||||
self.add_model_calls.append((model, model_config))
|
||||
def add_model(
|
||||
self,
|
||||
model: Any,
|
||||
model_config: Any,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
def step(self, is_dummy: bool, is_profile: bool, *, log_stats: bool) -> None:
|
||||
self.step_calls.append((is_dummy, is_profile, log_stats))
|
||||
@@ -98,6 +104,15 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
runner.execute_model_state = None
|
||||
for key, value in overrides.items():
|
||||
setattr(runner, key, value)
|
||||
spec_config = runner.speculative_config
|
||||
runner._draft_workspace_lane = int(
|
||||
spec_config is not None
|
||||
and (
|
||||
spec_config.use_dspark()
|
||||
if hasattr(spec_config, "use_dspark")
|
||||
else getattr(spec_config, "method", None) == "dspark"
|
||||
)
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
@@ -131,7 +146,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
assert runner.model is model
|
||||
assert runner.model_state is not None
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
|
||||
assert runner.eplb_state.async_started is True
|
||||
|
||||
|
||||
@@ -159,7 +174,7 @@ def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch):
|
||||
assert runner.load_config.load_format == "dummy"
|
||||
assert runner.eplb_state is not None
|
||||
# A scaling-up worker registers so it can join the EPLB communicator.
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
|
||||
assert runner.eplb_state.async_started is False
|
||||
|
||||
|
||||
@@ -174,6 +189,64 @@ def test_v2_setup_eplb_from_mapping_updates_state_in_place():
|
||||
assert state.update_mapping_calls == [(runner.model_config, mapping)]
|
||||
|
||||
|
||||
def test_v2_load_model_registers_dspark_speculator_with_eplb(monkeypatch):
|
||||
FakeEplbState.instances.clear()
|
||||
target_model = SimpleNamespace(is_moe=True)
|
||||
draft_model = SimpleNamespace(is_moe=True)
|
||||
draft_model_config = SimpleNamespace(
|
||||
model="dspark-draft",
|
||||
hf_config=SimpleNamespace(model_type="deepseek_v4"),
|
||||
)
|
||||
|
||||
class FakeDSparkSpeculator:
|
||||
def __init__(self):
|
||||
self.model = draft_model
|
||||
self.eplb_state = None
|
||||
|
||||
def load_model(self, target_model):
|
||||
return None
|
||||
|
||||
def set_eplb_state(self, state) -> None:
|
||||
self.eplb_state = state
|
||||
|
||||
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
|
||||
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
|
||||
monkeypatch.setattr(mrv2, "DraftModelSpeculator", FakeDSparkSpeculator)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"get_model_loader",
|
||||
lambda load_config: SimpleNamespace(load_model=lambda **_: target_model),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"init_model_state",
|
||||
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eplb,
|
||||
"get_mixture_of_experts_model",
|
||||
lambda model: model if getattr(model, "is_moe", False) else None,
|
||||
)
|
||||
|
||||
speculator = FakeDSparkSpeculator()
|
||||
runner = _make_runner(
|
||||
is_last_pp_rank=False,
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=draft_model_config,
|
||||
),
|
||||
speculator=speculator,
|
||||
)
|
||||
mrv2.GPUModelRunner.load_model(runner)
|
||||
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [
|
||||
(draft_model, draft_model_config, "dspark-draft (draft)"),
|
||||
(target_model, runner.model_config, None),
|
||||
]
|
||||
assert speculator.eplb_state is runner.eplb_state
|
||||
|
||||
|
||||
def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
events = []
|
||||
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
|
||||
|
||||
@@ -323,39 +323,47 @@ class EplbState:
|
||||
"""
|
||||
if len(self.model_states) > 0:
|
||||
model = next(iter(self.model_states.values())).model
|
||||
if (
|
||||
model.num_routed_experts != new_model.num_routed_experts
|
||||
or model.num_redundant_experts != new_model.num_redundant_experts
|
||||
or model.num_physical_experts != new_model.num_physical_experts
|
||||
or model.num_logical_experts != new_model.num_logical_experts
|
||||
or model.num_expert_groups != new_model.num_expert_groups
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Model: {} "
|
||||
"with config {} "
|
||||
"{} {} {} {} "
|
||||
"mismatch with new model {} "
|
||||
"with config {} "
|
||||
"{} {} {} {}".format(
|
||||
type(model),
|
||||
model.num_routed_experts,
|
||||
model.num_redundant_experts,
|
||||
model.num_physical_experts,
|
||||
model.num_logical_experts,
|
||||
model.num_expert_groups,
|
||||
type(new_model),
|
||||
new_model.num_routed_experts,
|
||||
new_model.num_redundant_experts,
|
||||
new_model.num_physical_experts,
|
||||
new_model.num_logical_experts,
|
||||
new_model.num_expert_groups,
|
||||
)
|
||||
self.assert_confs_equal(model, new_model)
|
||||
|
||||
@staticmethod
|
||||
def assert_confs_equal(
|
||||
model: MixtureOfExperts, new_model: MixtureOfExperts
|
||||
) -> None:
|
||||
"""Raise if the models have different expert parallel configurations."""
|
||||
if (
|
||||
model.num_routed_experts != new_model.num_routed_experts
|
||||
or model.num_redundant_experts != new_model.num_redundant_experts
|
||||
or model.num_physical_experts != new_model.num_physical_experts
|
||||
or model.num_logical_experts != new_model.num_logical_experts
|
||||
or model.num_expert_groups != new_model.num_expert_groups
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Model: {} "
|
||||
"with config {} "
|
||||
"{} {} {} {} "
|
||||
"mismatch with new model {} "
|
||||
"with config {} "
|
||||
"{} {} {} {}".format(
|
||||
type(model),
|
||||
model.num_routed_experts,
|
||||
model.num_redundant_experts,
|
||||
model.num_physical_experts,
|
||||
model.num_logical_experts,
|
||||
model.num_expert_groups,
|
||||
type(new_model),
|
||||
new_model.num_routed_experts,
|
||||
new_model.num_redundant_experts,
|
||||
new_model.num_physical_experts,
|
||||
new_model.num_logical_experts,
|
||||
new_model.num_expert_groups,
|
||||
)
|
||||
)
|
||||
|
||||
def add_model(
|
||||
self,
|
||||
model: MixtureOfExperts,
|
||||
model_config: ModelConfig,
|
||||
model_name: str | None = None,
|
||||
):
|
||||
"""Build the initial EPLB state."""
|
||||
self.validate_ep_configuration(model)
|
||||
@@ -506,7 +514,7 @@ class EplbState:
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
),
|
||||
model_name=model_config.model,
|
||||
model_name=model_name or model_config.model,
|
||||
model=model,
|
||||
expert_buffer=expert_buffer,
|
||||
rebalanced=False,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared EPLB MoE registration helpers for DeepSeek V4 family models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
def dspark_draft_supports_eplb(draft_model_config: ModelConfig) -> bool:
|
||||
"""Return whether a DSpark draft can share EPLB state with the target.
|
||||
|
||||
Only DeepSeek-V4 DSpark drafts reuse the target expert layout. V4.1 drafts
|
||||
use a smaller routed-expert count and cannot share EPLB state.
|
||||
"""
|
||||
return getattr(draft_model_config.hf_config, "model_type", None) == "deepseek_v4"
|
||||
|
||||
|
||||
def collect_moe_layers(
|
||||
moe_model: Any,
|
||||
layers: Iterable[nn.Module],
|
||||
config: object,
|
||||
*,
|
||||
decoder_layer_type: type[nn.Module],
|
||||
moe_type: type[nn.Module],
|
||||
) -> None:
|
||||
"""Populate ``MixtureOfExperts`` fields from decoder layers for EPLB.
|
||||
|
||||
Works for both target backbones and DSpark draft stacks. Draft models pass
|
||||
their shorter ``model.layers`` list; pipeline-parallel placeholder layers
|
||||
are skipped via ``decoder_layer_type``.
|
||||
"""
|
||||
moe_model.num_expert_groups = getattr(config, "n_group", 1)
|
||||
moe_model.moe_layers = []
|
||||
moe_model.moe_mlp_layers = []
|
||||
example_moe = None
|
||||
for layer in layers:
|
||||
if not isinstance(layer, decoder_layer_type):
|
||||
continue
|
||||
if not isinstance(layer.ffn, moe_type):
|
||||
continue
|
||||
example_moe = layer.ffn
|
||||
moe_model.moe_mlp_layers.append(layer.ffn)
|
||||
moe_model.moe_layers.append(layer.ffn.experts)
|
||||
|
||||
moe_model.num_moe_layers = len(moe_model.moe_layers)
|
||||
moe_model.extract_moe_parameters(example_moe)
|
||||
@@ -52,11 +52,14 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4MixtureOfExperts,
|
||||
DeepseekV4Model,
|
||||
DeepseekV4MoE,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
@@ -346,7 +349,7 @@ def _insert_context_kv(
|
||||
)
|
||||
|
||||
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
class DSparkDeepseekV4ForCausalLM(nn.Module, DeepseekV4MixtureOfExperts):
|
||||
# Draft weights ship in the target checkpoint (mtp.*) without embed/head, so
|
||||
# load_dspark_model always aliases the target's.
|
||||
has_own_embed_tokens = False
|
||||
@@ -373,6 +376,16 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
# --- Hooks used by the speculator -------------------------------------
|
||||
|
||||
@@ -451,6 +464,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_redundant_experts=first_layer.ffn.n_redundant_experts,
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
|
||||
@@ -81,6 +81,7 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.eplb_util import collect_moe_layers
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
@@ -1841,23 +1842,13 @@ class DeepseekV4ForCausalLM(
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
self.num_expert_groups = getattr(self.config, "n_group", 1)
|
||||
self.num_moe_layers = self.config.num_hidden_layers
|
||||
self.moe_layers: list[nn.Module] = []
|
||||
self.moe_mlp_layers: list[DeepseekV4MoE] = []
|
||||
example_moe: DeepseekV4MoE | None = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
if not isinstance(layer, DeepseekV4DecoderLayer):
|
||||
continue
|
||||
if isinstance(layer.ffn, DeepseekV4MoE):
|
||||
example_moe = layer.ffn
|
||||
self.moe_mlp_layers.append(layer.ffn)
|
||||
self.moe_layers.append(layer.ffn.experts)
|
||||
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
self.extract_moe_parameters(example_moe)
|
||||
collect_moe_layers(
|
||||
self,
|
||||
self.model.layers,
|
||||
self.config,
|
||||
decoder_layer_type=DeepseekV4DecoderLayer,
|
||||
moe_type=DeepseekV4MoE,
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
@@ -13,8 +13,10 @@ from vllm.config import ModelConfig
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MixtureOfExperts,
|
||||
get_mixture_of_experts_model,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
@@ -22,6 +24,25 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def eplb_draft_model_name(model_config: ModelConfig) -> str:
|
||||
"""Return the EPLB step-log display name for a draft model."""
|
||||
return f"{model_config.model} (draft)"
|
||||
|
||||
|
||||
def draft_model_supports_eplb(
|
||||
speculative_config: Any | None,
|
||||
draft_moe_model: MixtureOfExperts | None,
|
||||
) -> bool:
|
||||
"""Return whether a draft MoE model should register with EPLB."""
|
||||
if draft_moe_model is None:
|
||||
return False
|
||||
if speculative_config is None or speculative_config.draft_model_config is None:
|
||||
return False
|
||||
if getattr(speculative_config, "method", None) == "dspark":
|
||||
return dspark_draft_supports_eplb(speculative_config.draft_model_config)
|
||||
return True
|
||||
|
||||
|
||||
def step_eplb_after(*, is_dummy: bool = False) -> Callable:
|
||||
"""Step EPLB after a model runner method completes successfully."""
|
||||
|
||||
@@ -93,16 +114,26 @@ class EPLBController:
|
||||
|
||||
draft_model = speculator.model
|
||||
draft_moe_model = get_mixture_of_experts_model(draft_model)
|
||||
if draft_moe_model is None:
|
||||
if not draft_model_supports_eplb(speculative_config, draft_moe_model):
|
||||
return False
|
||||
assert draft_moe_model is not None
|
||||
|
||||
assert not self.parallel_config.enable_elastic_ep, (
|
||||
"Elastic EP is not supported with draft model."
|
||||
)
|
||||
assert speculative_config is not None
|
||||
assert speculative_config.draft_model_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
logger.info_once(
|
||||
"EPLB is enabled for MoE part of drafter model %s.",
|
||||
draft_model_config.model,
|
||||
)
|
||||
assert self.state is not None
|
||||
self.state.add_model(draft_moe_model, speculative_config.draft_model_config)
|
||||
self.state.add_model(
|
||||
draft_moe_model,
|
||||
draft_model_config,
|
||||
model_name=eplb_draft_model_name(draft_model_config),
|
||||
)
|
||||
speculator.set_eplb_state(self.state)
|
||||
self._has_registered_models = True
|
||||
return True
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch.nn as nn
|
||||
from vllm.config import ModelConfig, ParallelConfig, VllmConfig, replace
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.utils import get_draft_load_config
|
||||
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
@@ -35,11 +36,24 @@ def _resolve_dspark_attention_backend(
|
||||
def _get_dspark_parallel_config(
|
||||
parallel_config: ParallelConfig,
|
||||
tensor_parallel_size: int,
|
||||
draft_model_config: ModelConfig,
|
||||
) -> ParallelConfig:
|
||||
if parallel_config.enable_eplb:
|
||||
enable_draft_eplb = parallel_config.enable_eplb and dspark_draft_supports_eplb(
|
||||
draft_model_config
|
||||
)
|
||||
if parallel_config.enable_eplb and not enable_draft_eplb:
|
||||
logger.warning_once(
|
||||
"EPLB is disabled for the DSpark draft model. EPLB remains enabled "
|
||||
"for the target model."
|
||||
"EPLB is disabled for the DSpark draft model (%s). EPLB remains "
|
||||
"enabled for the target model.",
|
||||
draft_model_config.hf_config.model_type,
|
||||
)
|
||||
|
||||
if enable_draft_eplb:
|
||||
return replace(
|
||||
parallel_config,
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enable_elastic_ep=False,
|
||||
)
|
||||
|
||||
return replace(
|
||||
@@ -81,6 +95,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
parallel_config=_get_dspark_parallel_config(
|
||||
vllm_config.parallel_config,
|
||||
speculative_config.draft_parallel_config.tensor_parallel_size,
|
||||
draft_model_config,
|
||||
),
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
|
||||
@@ -206,6 +206,10 @@ from vllm.v1.worker.cp_utils import (
|
||||
)
|
||||
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
|
||||
from vllm.v1.worker.gpu.eplb_utils import (
|
||||
draft_model_supports_eplb,
|
||||
eplb_draft_model_name,
|
||||
)
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin
|
||||
@@ -5237,25 +5241,34 @@ class GPUModelRunner(
|
||||
logger.info_once("Loading drafter model...")
|
||||
if hasattr(self.drafter, "load_model"):
|
||||
self.drafter.load_model(self.model)
|
||||
drafter_moe_model = None
|
||||
if self.parallel_config.enable_eplb and hasattr(
|
||||
self.drafter, "model"
|
||||
):
|
||||
drafter_moe_model = get_mixture_of_experts_model(
|
||||
self.drafter.model
|
||||
)
|
||||
if not draft_model_supports_eplb(
|
||||
self.vllm_config.speculative_config,
|
||||
drafter_moe_model,
|
||||
):
|
||||
drafter_moe_model = None
|
||||
|
||||
if (
|
||||
self.parallel_config.enable_eplb
|
||||
and hasattr(self.drafter, "model")
|
||||
and (
|
||||
drafter_moe_model := get_mixture_of_experts_model(
|
||||
self.drafter.model
|
||||
)
|
||||
)
|
||||
is not None
|
||||
and drafter_moe_model is not None
|
||||
):
|
||||
assert not self.parallel_config.enable_elastic_ep, (
|
||||
"Elastic EP is not supported with drafter model."
|
||||
)
|
||||
spec_config = self.vllm_config.speculative_config
|
||||
assert spec_config is not None
|
||||
assert spec_config.draft_model_config is not None
|
||||
draft_model_config = spec_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
logger.info_once(
|
||||
"EPLB is enabled for MoE part of drafter model %s.",
|
||||
spec_config.draft_model_config.model,
|
||||
draft_model_config.model,
|
||||
)
|
||||
if self.eplb_state is None:
|
||||
self.eplb_state = EplbState(
|
||||
@@ -5263,7 +5276,8 @@ class GPUModelRunner(
|
||||
)
|
||||
self.eplb_state.add_model(
|
||||
drafter_moe_model,
|
||||
spec_config.draft_model_config,
|
||||
draft_model_config,
|
||||
model_name=eplb_draft_model_name(draft_model_config),
|
||||
)
|
||||
assert hasattr(self.drafter, "set_eplb_state")
|
||||
self.drafter.set_eplb_state(self.eplb_state)
|
||||
|
||||
Reference in New Issue
Block a user