[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_weight_scale_block,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
is_mixture_of_experts,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4ForCausalLM,
|
||||
@@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
@@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
|
||||
assert calls == ["mtp", "dspark"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def v4_dspark_config(dist_init):
|
||||
hf_config = DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hidden_act="silu",
|
||||
swiglu_limit=10.0,
|
||||
norm_topk_prob=True,
|
||||
topk_method="noaux_tc",
|
||||
routed_scaling_factor=1.5,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
compress_ratios=[1, 1],
|
||||
q_lora_rank=32,
|
||||
o_lora_rank=32,
|
||||
qk_rope_head_dim=16,
|
||||
o_groups=4,
|
||||
sliding_window=128,
|
||||
hc_sinkhorn_iters=1,
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
hf_config=hf_config,
|
||||
max_model_len=2048,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(
|
||||
moe_backend="deep_gemm_mega_moe",
|
||||
enable_jit_warmup=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(draft_model_config=model_config),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
|
||||
def test_dspark_draft_registers_mixture_of_experts(
|
||||
v4_dspark_config, monkeypatch, dist_init
|
||||
):
|
||||
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DeepGEMM MegaMoE requires SM100")
|
||||
|
||||
hf_config = v4_dspark_config.model_config.hf_config
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
|
||||
lambda: v4_dspark_config,
|
||||
)
|
||||
|
||||
with (
|
||||
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
|
||||
torch.device("cuda"),
|
||||
):
|
||||
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
|
||||
|
||||
assert is_mixture_of_experts(draft)
|
||||
assert get_mixture_of_experts_model(draft) is draft
|
||||
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
|
||||
assert draft.num_routed_experts == hf_config.n_routed_experts
|
||||
assert draft.num_redundant_experts == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
|
||||
@@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales(
|
||||
torch.testing.assert_close(param, checkpoint_scale)
|
||||
|
||||
|
||||
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
|
||||
monkeypatch,
|
||||
):
|
||||
from vllm.models.deepseek_v4.nvidia import dspark
|
||||
|
||||
captured: dict[str, int] = {}
|
||||
|
||||
def capture_expert_mapping(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return []
|
||||
|
||||
draft = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
num_attention_heads=4,
|
||||
n_routed_experts=8,
|
||||
expert_dtype="fp4",
|
||||
),
|
||||
quant_config=None,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[
|
||||
SimpleNamespace(
|
||||
ffn=SimpleNamespace(
|
||||
use_native_mega_moe=False,
|
||||
use_mega_moe=False,
|
||||
n_redundant_experts=4,
|
||||
)
|
||||
)
|
||||
],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [],
|
||||
process_weights_after_loading=lambda: None,
|
||||
)
|
||||
draft._remap_dspark_name = lambda name: (
|
||||
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
|
||||
)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
dspark,
|
||||
"fused_moe_make_expert_params_mapping",
|
||||
capture_expert_mapping,
|
||||
)
|
||||
|
||||
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
|
||||
|
||||
assert captured["num_experts"] == 8
|
||||
assert captured["num_redundant_experts"] == 4
|
||||
|
||||
|
||||
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
|
||||
weights = [
|
||||
("mtp.0.attn.wkv.weight", torch.arange(4)),
|
||||
|
||||
Reference in New Issue
Block a user