[EPLB] Mask padding in EPLB load recording (#38128)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
311ad689ad
commit
a2abce646f
@@ -213,6 +213,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
self._prepare_eplb_forward(input_batch.num_tokens)
|
||||
|
||||
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Replay the full graph for draft prefill.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
@@ -424,6 +426,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
self._prepare_eplb_forward(num_reqs)
|
||||
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
# Run the draft model forward pass.
|
||||
|
||||
@@ -278,6 +278,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.hidden_states[:num_target_tokens],
|
||||
self.context_positions[:num_target_tokens],
|
||||
)
|
||||
# DFlash processes all speculative tokens in one forward pass,
|
||||
# so the real token count is num_query_tokens.
|
||||
self._prepare_eplb_forward(num_query_tokens)
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
@@ -354,6 +357,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.kv_cache_config,
|
||||
)
|
||||
|
||||
# DFlash processes all speculative tokens in one forward pass,
|
||||
# so the real token count is num_query_tokens.
|
||||
self._prepare_eplb_forward(num_query_tokens)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -106,6 +107,8 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
|
||||
self.eplb_state: EplbState | None = None
|
||||
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
@@ -165,6 +168,18 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
)
|
||||
self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names
|
||||
|
||||
def set_eplb_state(self, eplb_state: EplbState) -> None:
|
||||
"""Inject EPLB state after construction."""
|
||||
self.eplb_state = eplb_state
|
||||
|
||||
def _prepare_eplb_forward(self, num_unpadded_tokens: int) -> None:
|
||||
"""Call EPLB prepare_forward if EPLB is active for the draft model."""
|
||||
if self.eplb_state is not None:
|
||||
self.eplb_state.prepare_forward(
|
||||
self.speculative_config.draft_model_config,
|
||||
num_unpadded_tokens,
|
||||
)
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
|
||||
Reference in New Issue
Block a user