[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -2,11 +2,21 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.v1.worker.gpu.eplb_utils import (
|
||||
EPLBController,
|
||||
draft_model_supports_eplb,
|
||||
)
|
||||
|
||||
|
||||
def get_model_args(
|
||||
@@ -15,14 +25,18 @@ def get_model_args(
|
||||
spec_method: str,
|
||||
tp_size: int,
|
||||
model_max_len: int,
|
||||
num_speculative_tokens: int = 1,
|
||||
use_async: bool = True,
|
||||
) -> dict:
|
||||
speculative_config = {
|
||||
"method": spec_method,
|
||||
"model": spec_model_name,
|
||||
"num_speculative_tokens": 1,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
speculative_config["model"] = model_name
|
||||
speculative_config["draft_sample_method"] = "probabilistic"
|
||||
eplb_config = {
|
||||
"num_redundant_experts": tp_size,
|
||||
"window_size": 128,
|
||||
@@ -42,6 +56,8 @@ def get_model_args(
|
||||
"enable_eplb": True,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
model_args["trust_remote_code"] = True
|
||||
return model_args
|
||||
|
||||
|
||||
@@ -51,11 +67,190 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
class FakeEplbState:
|
||||
instances: list[FakeEplbState] = []
|
||||
|
||||
def __init__(self, parallel_config, device: torch.device):
|
||||
self.add_model_calls: list[tuple[object, object, str | None]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(
|
||||
self,
|
||||
model: object,
|
||||
model_config: object,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
|
||||
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
|
||||
return DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
compress_ratios=[1, 1],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dspark_vllm_config():
|
||||
hf_config = _make_dsv4_dspark_hf_config()
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
enable_elastic_ep=False,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=model_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_moe_topology(
|
||||
*,
|
||||
num_routed_experts: int = 8,
|
||||
num_redundant_experts: int = 4,
|
||||
num_expert_groups: int = 1,
|
||||
) -> SimpleNamespace:
|
||||
num_physical_experts = num_routed_experts + num_redundant_experts
|
||||
return SimpleNamespace(
|
||||
num_routed_experts=num_routed_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_logical_experts=num_routed_experts,
|
||||
num_expert_groups=num_expert_groups,
|
||||
)
|
||||
|
||||
|
||||
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
|
||||
draft = _make_moe_topology()
|
||||
target = _make_moe_topology()
|
||||
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
|
||||
draft = _make_moe_topology(num_redundant_experts=0)
|
||||
target = _make_moe_topology(num_redundant_experts=4)
|
||||
|
||||
with pytest.raises(RuntimeError, match="mismatch"):
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "expected"),
|
||||
[
|
||||
("deepseek_v4", True),
|
||||
("deepseek_v41", False),
|
||||
],
|
||||
)
|
||||
def test_dspark_draft_supports_eplb(
|
||||
dspark_vllm_config, model_type: str, expected: bool
|
||||
):
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
assert dspark_draft_supports_eplb(draft_model_config) is expected
|
||||
|
||||
|
||||
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
|
||||
assert not draft_model_supports_eplb(
|
||||
dspark_vllm_config.speculative_config,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type,load_dummy_weights,expected_registered",
|
||||
[
|
||||
("deepseek_v4", False, True),
|
||||
("deepseek_v4", True, False),
|
||||
("deepseek_v41", False, False),
|
||||
],
|
||||
)
|
||||
def test_eplb_dspark_draft_registration(
|
||||
dspark_vllm_config,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type,
|
||||
load_dummy_weights: bool,
|
||||
expected_registered: bool,
|
||||
):
|
||||
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
|
||||
FakeEplbState.instances.clear()
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.EplbState",
|
||||
FakeEplbState,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
|
||||
lambda model: model,
|
||||
)
|
||||
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
draft = SimpleNamespace()
|
||||
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
|
||||
controller.prepare_load()
|
||||
speculator = SimpleNamespace(model=draft, eplb_state=None)
|
||||
|
||||
def set_eplb_state(state) -> None:
|
||||
speculator.eplb_state = state
|
||||
|
||||
speculator.set_eplb_state = set_eplb_state
|
||||
|
||||
registered = controller.maybe_register_speculator(
|
||||
speculator,
|
||||
dspark_vllm_config.speculative_config,
|
||||
load_dummy_weights=load_dummy_weights,
|
||||
)
|
||||
|
||||
assert registered is expected_registered
|
||||
assert controller.state is not None
|
||||
if expected_registered:
|
||||
assert controller.state.add_model_calls == [
|
||||
(draft, draft_model_config, "dspark (draft)")
|
||||
]
|
||||
assert speculator.eplb_state is controller.state
|
||||
else:
|
||||
assert controller.state.add_model_calls == []
|
||||
assert speculator.eplb_state is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_setup",
|
||||
[
|
||||
pytest.param(
|
||||
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86),
|
||||
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
pytest.param(
|
||||
@@ -65,20 +260,42 @@ pytestmark = pytest.mark.skipif(
|
||||
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
|
||||
4,
|
||||
0.92,
|
||||
1,
|
||||
),
|
||||
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
"dspark",
|
||||
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
|
||||
None,
|
||||
4,
|
||||
0.95,
|
||||
7,
|
||||
),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
],
|
||||
ids=["qwen3_next_mtp", "llama4_eagle"],
|
||||
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
|
||||
)
|
||||
def test_eplb_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str, int, float],
|
||||
model_setup: tuple[str, str, str | None, int, float, int],
|
||||
):
|
||||
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
Applicable to MoE models with mtp or eagle spec decode.
|
||||
|
||||
Applicable to MoE models with mtp, eagle, or dspark spec decode.
|
||||
"""
|
||||
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
|
||||
(
|
||||
method,
|
||||
model_name,
|
||||
spec_model_name,
|
||||
tp_size,
|
||||
expected_gsm8k_value,
|
||||
num_speculative_tokens,
|
||||
) = model_setup
|
||||
if method == "dspark" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
|
||||
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
@@ -90,6 +307,7 @@ def test_eplb_spec_decode(
|
||||
spec_method=method,
|
||||
tp_size=tp_size,
|
||||
model_max_len=4096,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
|
||||
@@ -15,6 +15,10 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_weight_scale_block,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
is_mixture_of_experts,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4ForCausalLM,
|
||||
@@ -28,6 +32,7 @@ from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_i
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
@@ -913,6 +918,96 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
|
||||
assert calls == ["mtp", "dspark"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def v4_dspark_config(dist_init):
|
||||
hf_config = DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hidden_act="silu",
|
||||
swiglu_limit=10.0,
|
||||
norm_topk_prob=True,
|
||||
topk_method="noaux_tc",
|
||||
routed_scaling_factor=1.5,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
compress_ratios=[1, 1],
|
||||
q_lora_rank=32,
|
||||
o_lora_rank=32,
|
||||
qk_rope_head_dim=16,
|
||||
o_groups=4,
|
||||
sliding_window=128,
|
||||
hc_sinkhorn_iters=1,
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
hf_config=hf_config,
|
||||
max_model_len=2048,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(
|
||||
moe_backend="deep_gemm_mega_moe",
|
||||
enable_jit_warmup=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(draft_model_config=model_config),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
|
||||
def test_dspark_draft_registers_mixture_of_experts(
|
||||
v4_dspark_config, monkeypatch, dist_init
|
||||
):
|
||||
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DeepGEMM MegaMoE requires SM100")
|
||||
|
||||
hf_config = v4_dspark_config.model_config.hf_config
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
|
||||
lambda: v4_dspark_config,
|
||||
)
|
||||
|
||||
with (
|
||||
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
|
||||
torch.device("cuda"),
|
||||
):
|
||||
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
|
||||
|
||||
assert is_mixture_of_experts(draft)
|
||||
assert get_mixture_of_experts_model(draft) is draft
|
||||
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
|
||||
assert draft.num_routed_experts == hf_config.n_routed_experts
|
||||
assert draft.num_redundant_experts == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
|
||||
|
||||
@@ -266,6 +266,57 @@ def test_v41_dspark_loads_linear_scales(
|
||||
torch.testing.assert_close(param, checkpoint_scale)
|
||||
|
||||
|
||||
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
|
||||
monkeypatch,
|
||||
):
|
||||
from vllm.models.deepseek_v4.nvidia import dspark
|
||||
|
||||
captured: dict[str, int] = {}
|
||||
|
||||
def capture_expert_mapping(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return []
|
||||
|
||||
draft = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
num_attention_heads=4,
|
||||
n_routed_experts=8,
|
||||
expert_dtype="fp4",
|
||||
),
|
||||
quant_config=None,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[
|
||||
SimpleNamespace(
|
||||
ffn=SimpleNamespace(
|
||||
use_native_mega_moe=False,
|
||||
use_mega_moe=False,
|
||||
n_redundant_experts=4,
|
||||
)
|
||||
)
|
||||
],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [],
|
||||
process_weights_after_loading=lambda: None,
|
||||
)
|
||||
draft._remap_dspark_name = lambda name: (
|
||||
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
|
||||
)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
dspark,
|
||||
"fused_moe_make_expert_params_mapping",
|
||||
capture_expert_mapping,
|
||||
)
|
||||
|
||||
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
|
||||
|
||||
assert captured["num_experts"] == 8
|
||||
assert captured["num_redundant_experts"] == 4
|
||||
|
||||
|
||||
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
|
||||
weights = [
|
||||
("mtp.0.attn.wkv.weight", torch.arange(4)),
|
||||
|
||||
@@ -15,6 +15,16 @@ from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_con
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeHFConfig:
|
||||
model_type: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeModelConfig:
|
||||
hf_config: _FakeHFConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeEPLBConfig:
|
||||
num_redundant_experts: int = 0
|
||||
@@ -37,12 +47,28 @@ class _FakeParallelConfig:
|
||||
raise ValueError("elastic EP requires EPLB")
|
||||
|
||||
|
||||
def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "enable_eplb", "num_redundant_experts", "shares_eplb_config"),
|
||||
[
|
||||
("deepseek_v4", True, 32, True),
|
||||
("deepseek_v41", False, 0, False),
|
||||
],
|
||||
)
|
||||
def test_dspark_parallel_config_eplb_by_model_type(
|
||||
model_type: str,
|
||||
enable_eplb: bool,
|
||||
num_redundant_experts: int,
|
||||
shares_eplb_config: bool,
|
||||
):
|
||||
target_config = _FakeParallelConfig()
|
||||
draft_model_config = _FakeModelConfig(
|
||||
hf_config=_FakeHFConfig(model_type=model_type)
|
||||
)
|
||||
|
||||
draft_config = _get_dspark_parallel_config(
|
||||
target_config,
|
||||
tensor_parallel_size=4,
|
||||
draft_model_config=draft_model_config,
|
||||
)
|
||||
|
||||
assert target_config.pipeline_parallel_size == 2
|
||||
@@ -54,10 +80,13 @@ def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
assert draft_config is not target_config
|
||||
assert draft_config.pipeline_parallel_size == 1
|
||||
assert draft_config.tensor_parallel_size == 4
|
||||
assert not draft_config.enable_eplb
|
||||
assert draft_config.eplb_config.num_redundant_experts == 0
|
||||
assert draft_config.enable_eplb is enable_eplb
|
||||
assert draft_config.eplb_config.num_redundant_experts == num_redundant_experts
|
||||
assert not draft_config.enable_elastic_ep
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
if shares_eplb_config:
|
||||
assert draft_config.eplb_config is target_config.eplb_config
|
||||
else:
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
|
||||
@@ -29,15 +29,21 @@ class FakeEplbState:
|
||||
def __init__(self, parallel_config: Any, device: torch.device):
|
||||
self.parallel_config = parallel_config
|
||||
self.device = device
|
||||
self.add_model_calls: list[tuple[Any, Any]] = []
|
||||
self.add_model_calls: list[tuple[Any, Any, str | None]] = []
|
||||
self.step_calls: list[tuple[bool, bool, bool]] = []
|
||||
self.async_started = False
|
||||
self.is_async = True
|
||||
self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(self, model: Any, model_config: Any) -> None:
|
||||
self.add_model_calls.append((model, model_config))
|
||||
def add_model(
|
||||
self,
|
||||
model: Any,
|
||||
model_config: Any,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
def step(self, is_dummy: bool, is_profile: bool, *, log_stats: bool) -> None:
|
||||
self.step_calls.append((is_dummy, is_profile, log_stats))
|
||||
@@ -98,6 +104,15 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
runner.execute_model_state = None
|
||||
for key, value in overrides.items():
|
||||
setattr(runner, key, value)
|
||||
spec_config = runner.speculative_config
|
||||
runner._draft_workspace_lane = int(
|
||||
spec_config is not None
|
||||
and (
|
||||
spec_config.use_dspark()
|
||||
if hasattr(spec_config, "use_dspark")
|
||||
else getattr(spec_config, "method", None) == "dspark"
|
||||
)
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
@@ -131,7 +146,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
assert runner.model is model
|
||||
assert runner.model_state is not None
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
|
||||
assert runner.eplb_state.async_started is True
|
||||
|
||||
|
||||
@@ -159,7 +174,7 @@ def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch):
|
||||
assert runner.load_config.load_format == "dummy"
|
||||
assert runner.eplb_state is not None
|
||||
# A scaling-up worker registers so it can join the EPLB communicator.
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
|
||||
assert runner.eplb_state.async_started is False
|
||||
|
||||
|
||||
@@ -174,6 +189,64 @@ def test_v2_setup_eplb_from_mapping_updates_state_in_place():
|
||||
assert state.update_mapping_calls == [(runner.model_config, mapping)]
|
||||
|
||||
|
||||
def test_v2_load_model_registers_dspark_speculator_with_eplb(monkeypatch):
|
||||
FakeEplbState.instances.clear()
|
||||
target_model = SimpleNamespace(is_moe=True)
|
||||
draft_model = SimpleNamespace(is_moe=True)
|
||||
draft_model_config = SimpleNamespace(
|
||||
model="dspark-draft",
|
||||
hf_config=SimpleNamespace(model_type="deepseek_v4"),
|
||||
)
|
||||
|
||||
class FakeDSparkSpeculator:
|
||||
def __init__(self):
|
||||
self.model = draft_model
|
||||
self.eplb_state = None
|
||||
|
||||
def load_model(self, target_model):
|
||||
return None
|
||||
|
||||
def set_eplb_state(self, state) -> None:
|
||||
self.eplb_state = state
|
||||
|
||||
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
|
||||
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
|
||||
monkeypatch.setattr(mrv2, "DraftModelSpeculator", FakeDSparkSpeculator)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"get_model_loader",
|
||||
lambda load_config: SimpleNamespace(load_model=lambda **_: target_model),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"init_model_state",
|
||||
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eplb,
|
||||
"get_mixture_of_experts_model",
|
||||
lambda model: model if getattr(model, "is_moe", False) else None,
|
||||
)
|
||||
|
||||
speculator = FakeDSparkSpeculator()
|
||||
runner = _make_runner(
|
||||
is_last_pp_rank=False,
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=draft_model_config,
|
||||
),
|
||||
speculator=speculator,
|
||||
)
|
||||
mrv2.GPUModelRunner.load_model(runner)
|
||||
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [
|
||||
(draft_model, draft_model_config, "dspark-draft (draft)"),
|
||||
(target_model, runner.model_config, None),
|
||||
]
|
||||
assert speculator.eplb_state is runner.eplb_state
|
||||
|
||||
|
||||
def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
events = []
|
||||
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
|
||||
|
||||
Reference in New Issue
Block a user