[EPLB] Mask padding in EPLB load recording (#38128)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
311ad689ad
commit
a2abce646f
@@ -9,6 +9,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
@@ -43,6 +44,8 @@ class ExtractHiddenStatesProposer:
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
|
||||
self.eplb_state: EplbState | None = None
|
||||
|
||||
# Model and attention layer tracking (initialized in load_model)
|
||||
self.model: nn.Module | None = None
|
||||
self.attn_layer_names: list[str] = []
|
||||
@@ -83,6 +86,10 @@ class ExtractHiddenStatesProposer:
|
||||
self.max_num_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
def set_eplb_state(self, eplb_state: EplbState) -> None:
|
||||
"""Inject EPLB state after construction."""
|
||||
self.eplb_state = eplb_state
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
@@ -145,6 +152,12 @@ class ExtractHiddenStatesProposer:
|
||||
if num_tokens_across_dp is not None:
|
||||
num_tokens_across_dp[self.dp_rank] = num_input_tokens
|
||||
|
||||
if self.eplb_state is not None:
|
||||
assert self.vllm_config.speculative_config is not None
|
||||
self.eplb_state.prepare_forward(
|
||||
self.vllm_config.speculative_config.draft_model_config,
|
||||
num_tokens,
|
||||
)
|
||||
with set_forward_context(
|
||||
per_layer_attn_metadata,
|
||||
self.vllm_config,
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.config import (
|
||||
get_layers_from_vllm_config,
|
||||
replace,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -79,6 +80,7 @@ class SpecDecodeBaseProposer:
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
self.eplb_state: EplbState | None = None
|
||||
self.num_speculative_tokens = self.speculative_config.num_speculative_tokens
|
||||
|
||||
# We need to get the hidden size from the draft model config because
|
||||
@@ -328,6 +330,10 @@ class SpecDecodeBaseProposer:
|
||||
"does not support M-RoPE yet"
|
||||
)
|
||||
|
||||
def set_eplb_state(self, eplb_state: EplbState) -> None:
|
||||
"""Inject EPLB state after construction."""
|
||||
self.eplb_state = eplb_state
|
||||
|
||||
def _init_parallel_drafting_params(self):
|
||||
# For parallel drafting, we need the token ID to use for masked slots
|
||||
# And for EAGLE + parallel drafting, we need the hidden state tensor to use
|
||||
@@ -527,6 +533,12 @@ class SpecDecodeBaseProposer:
|
||||
if self._share_mtp_indices and hasattr(self.model.model, "set_skip_topk"):
|
||||
self.model.model.set_skip_topk(False)
|
||||
|
||||
if self.eplb_state is not None:
|
||||
self.eplb_state.prepare_forward(
|
||||
self.draft_model_config,
|
||||
num_tokens,
|
||||
)
|
||||
|
||||
with set_forward_context(
|
||||
per_layer_attn_metadata,
|
||||
self.vllm_config,
|
||||
@@ -672,6 +684,12 @@ class SpecDecodeBaseProposer:
|
||||
if self.pass_hidden_states_to_model:
|
||||
model_kwargs["hidden_states"] = self.hidden_states[:input_batch_size]
|
||||
|
||||
if self.eplb_state is not None:
|
||||
self.eplb_state.prepare_forward(
|
||||
self.draft_model_config,
|
||||
batch_size,
|
||||
)
|
||||
|
||||
with set_forward_context(
|
||||
per_layer_attn_metadata,
|
||||
self.vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user