[EPLB] Add EPLB support to dspark draft models (DeepSeekV4) (#57549)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -15,6 +15,16 @@ from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_con
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeHFConfig:
|
||||
model_type: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeModelConfig:
|
||||
hf_config: _FakeHFConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeEPLBConfig:
|
||||
num_redundant_experts: int = 0
|
||||
@@ -37,12 +47,28 @@ class _FakeParallelConfig:
|
||||
raise ValueError("elastic EP requires EPLB")
|
||||
|
||||
|
||||
def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "enable_eplb", "num_redundant_experts", "shares_eplb_config"),
|
||||
[
|
||||
("deepseek_v4", True, 32, True),
|
||||
("deepseek_v41", False, 0, False),
|
||||
],
|
||||
)
|
||||
def test_dspark_parallel_config_eplb_by_model_type(
|
||||
model_type: str,
|
||||
enable_eplb: bool,
|
||||
num_redundant_experts: int,
|
||||
shares_eplb_config: bool,
|
||||
):
|
||||
target_config = _FakeParallelConfig()
|
||||
draft_model_config = _FakeModelConfig(
|
||||
hf_config=_FakeHFConfig(model_type=model_type)
|
||||
)
|
||||
|
||||
draft_config = _get_dspark_parallel_config(
|
||||
target_config,
|
||||
tensor_parallel_size=4,
|
||||
draft_model_config=draft_model_config,
|
||||
)
|
||||
|
||||
assert target_config.pipeline_parallel_size == 2
|
||||
@@ -54,10 +80,13 @@ def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
assert draft_config is not target_config
|
||||
assert draft_config.pipeline_parallel_size == 1
|
||||
assert draft_config.tensor_parallel_size == 4
|
||||
assert not draft_config.enable_eplb
|
||||
assert draft_config.eplb_config.num_redundant_experts == 0
|
||||
assert draft_config.enable_eplb is enable_eplb
|
||||
assert draft_config.eplb_config.num_redundant_experts == num_redundant_experts
|
||||
assert not draft_config.enable_elastic_ep
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
if shares_eplb_config:
|
||||
assert draft_config.eplb_config is target_config.eplb_config
|
||||
else:
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
|
||||
@@ -29,15 +29,21 @@ class FakeEplbState:
|
||||
def __init__(self, parallel_config: Any, device: torch.device):
|
||||
self.parallel_config = parallel_config
|
||||
self.device = device
|
||||
self.add_model_calls: list[tuple[Any, Any]] = []
|
||||
self.add_model_calls: list[tuple[Any, Any, str | None]] = []
|
||||
self.step_calls: list[tuple[bool, bool, bool]] = []
|
||||
self.async_started = False
|
||||
self.is_async = True
|
||||
self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(self, model: Any, model_config: Any) -> None:
|
||||
self.add_model_calls.append((model, model_config))
|
||||
def add_model(
|
||||
self,
|
||||
model: Any,
|
||||
model_config: Any,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
def step(self, is_dummy: bool, is_profile: bool, *, log_stats: bool) -> None:
|
||||
self.step_calls.append((is_dummy, is_profile, log_stats))
|
||||
@@ -98,6 +104,15 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
runner.execute_model_state = None
|
||||
for key, value in overrides.items():
|
||||
setattr(runner, key, value)
|
||||
spec_config = runner.speculative_config
|
||||
runner._draft_workspace_lane = int(
|
||||
spec_config is not None
|
||||
and (
|
||||
spec_config.use_dspark()
|
||||
if hasattr(spec_config, "use_dspark")
|
||||
else getattr(spec_config, "method", None) == "dspark"
|
||||
)
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
@@ -131,7 +146,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
assert runner.model is model
|
||||
assert runner.model_state is not None
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
|
||||
assert runner.eplb_state.async_started is True
|
||||
|
||||
|
||||
@@ -159,7 +174,7 @@ def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch):
|
||||
assert runner.load_config.load_format == "dummy"
|
||||
assert runner.eplb_state is not None
|
||||
# A scaling-up worker registers so it can join the EPLB communicator.
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config, None)]
|
||||
assert runner.eplb_state.async_started is False
|
||||
|
||||
|
||||
@@ -174,6 +189,64 @@ def test_v2_setup_eplb_from_mapping_updates_state_in_place():
|
||||
assert state.update_mapping_calls == [(runner.model_config, mapping)]
|
||||
|
||||
|
||||
def test_v2_load_model_registers_dspark_speculator_with_eplb(monkeypatch):
|
||||
FakeEplbState.instances.clear()
|
||||
target_model = SimpleNamespace(is_moe=True)
|
||||
draft_model = SimpleNamespace(is_moe=True)
|
||||
draft_model_config = SimpleNamespace(
|
||||
model="dspark-draft",
|
||||
hf_config=SimpleNamespace(model_type="deepseek_v4"),
|
||||
)
|
||||
|
||||
class FakeDSparkSpeculator:
|
||||
def __init__(self):
|
||||
self.model = draft_model
|
||||
self.eplb_state = None
|
||||
|
||||
def load_model(self, target_model):
|
||||
return None
|
||||
|
||||
def set_eplb_state(self, state) -> None:
|
||||
self.eplb_state = state
|
||||
|
||||
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
|
||||
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
|
||||
monkeypatch.setattr(mrv2, "DraftModelSpeculator", FakeDSparkSpeculator)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"get_model_loader",
|
||||
lambda load_config: SimpleNamespace(load_model=lambda **_: target_model),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"init_model_state",
|
||||
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eplb,
|
||||
"get_mixture_of_experts_model",
|
||||
lambda model: model if getattr(model, "is_moe", False) else None,
|
||||
)
|
||||
|
||||
speculator = FakeDSparkSpeculator()
|
||||
runner = _make_runner(
|
||||
is_last_pp_rank=False,
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=draft_model_config,
|
||||
),
|
||||
speculator=speculator,
|
||||
)
|
||||
mrv2.GPUModelRunner.load_model(runner)
|
||||
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [
|
||||
(draft_model, draft_model_config, "dspark-draft (draft)"),
|
||||
(target_model, runner.model_config, None),
|
||||
]
|
||||
assert speculator.eplb_state is runner.eplb_state
|
||||
|
||||
|
||||
def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
events = []
|
||||
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
|
||||
|
||||
Reference in New Issue
Block a user