[BugFix] Support EP/DP + EPLB with MTP (#25311)

Signed-off-by: ilmarkov <markovilya197@gmail.com>
Signed-off-by: Sage Moore <sage@neuralmagic.com>
Co-authored-by: Sage Moore <sage@neuralmagic.com>
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Ilya Markov
2025-11-05 15:22:17 +00:00
committed by GitHub
co-authored by Sage Moore Tyler Michael Smith Lucas Wilkinson
parent 5d16d0fa62
commit e50c454672
27 changed files with 957 additions and 529 deletions
+55 -33
View File
@@ -2046,7 +2046,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
model = self.get_model()
assert is_mixture_of_experts(model)
self.eplb_state.step(
model,
is_dummy,
is_profile,
log_stats=self.parallel_config.eplb_config.log_balancedness,
@@ -2803,7 +2802,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
else:
indices = []
offset = 0
assert spec_decode_metadata is not None
assert spec_decode_metadata is not None, (
"No spec decode metadata for medusa"
)
for num_draft, tokens in zip(
spec_decode_metadata.num_draft_tokens, sampled_token_ids
):
@@ -2934,32 +2935,15 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.model_config.model,
scope="global",
)
if eep_scale_up:
from vllm.distributed.parallel_state import get_ep_group
num_local_physical_experts = torch.empty(1, dtype=torch.int32, device="cpu")
torch.distributed.broadcast(
num_local_physical_experts, group=get_ep_group().cpu_group, group_src=0
)
num_local_physical_experts = int(num_local_physical_experts.item())
new_ep_size = get_ep_group().world_size
global_expert_load, old_global_expert_indices = EplbState.recv_state()
num_logical_experts = global_expert_load.shape[1]
self.parallel_config.eplb_config.num_redundant_experts = (
num_local_physical_experts * new_ep_size - num_logical_experts
)
assert old_global_expert_indices.shape[1] % num_local_physical_experts == 0
old_ep_size = (
old_global_expert_indices.shape[1] // num_local_physical_experts
)
rank_mapping = {
old_ep_rank: old_ep_rank for old_ep_rank in range(old_ep_size)
}
else:
global_expert_load = None
old_global_expert_indices = None
rank_mapping = None
global_expert_loads, old_global_expert_indices_per_model, rank_mapping = (
EplbState.get_eep_state(self.parallel_config)
if eep_scale_up
else (None, None, None)
)
if self.parallel_config.enable_eplb:
self.eplb_state = EplbState(self.parallel_config, self.device)
eplb_models = 0
with DeviceMemoryProfiler() as m:
time_before_load = time.perf_counter()
model_loader = get_model_loader(self.load_config)
@@ -2971,8 +2955,39 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.model, self.vllm_config, self.device
)
if hasattr(self, "drafter"):
logger.info("Loading drafter model...")
logger.info_once("Loading drafter model...")
self.drafter.load_model(self.model)
if (
hasattr(self.drafter, "model")
and is_mixture_of_experts(self.drafter.model)
and self.parallel_config.enable_eplb
):
logger.info_once(
"EPLB is enabled for drafter model %s.",
self.vllm_config.speculative_config.draft_model_config.model,
)
global_expert_load = (
global_expert_loads[eplb_models]
if global_expert_loads
else None
)
old_global_expert_indices = (
old_global_expert_indices_per_model[eplb_models]
if old_global_expert_indices_per_model
else None
)
if self.eplb_state is None:
self.eplb_state = EplbState(self.parallel_config, self.device)
self.eplb_state.add_model(
self.drafter.model,
self.vllm_config.speculative_config.draft_model_config,
global_expert_load,
old_global_expert_indices,
rank_mapping,
)
eplb_models += 1
if self.use_aux_hidden_state_outputs:
if not supports_eagle3(self.get_model()):
raise RuntimeError(
@@ -3001,18 +3016,25 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
scope="local",
)
prepare_communication_buffer_for_model(self.model)
self.is_multimodal_pruning_enabled = (
supports_multimodal_pruning(self.get_model())
and self.model_config.multimodal_config.is_multimodal_pruning_enabled()
)
if is_mixture_of_experts(self.model) and self.parallel_config.enable_eplb:
logger.info("EPLB is enabled for model %s.", self.model_config.model)
self.eplb_state = EplbState.build(
logger.info_once("EPLB is enabled for model %s.", self.model_config.model)
global_expert_load = (
global_expert_loads[eplb_models] if global_expert_loads else None
)
old_global_expert_indices = (
old_global_expert_indices_per_model[eplb_models]
if old_global_expert_indices_per_model
else None
)
assert self.eplb_state is not None
self.eplb_state.add_model(
self.model,
self.device,
self.parallel_config,
self.model_config,
global_expert_load,
old_global_expert_indices,
rank_mapping,