[Model Runner V2][MTP] Share topk index buffer between draft steps (#47352)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-08-10 15:46:13 -07:00
committed by GitHub
co-authored by Woosuk Kwon
parent c3cac8c63d
commit 3e174bb73c
2 changed files with 68 additions and 6 deletions
@@ -54,6 +54,18 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
device=self.device,
)
# Lifecycle hooks for model-specific optimizations. Subclasses override
# the ones they need. These fire in both `capture` and `propose` so that
# any state they toggle (e.g. attention flags baked into a CUDA graph) is
# identical at capture time and replay time.
def on_prefill_begin(self, num_reqs: int) -> None: ...
def on_prefill_end(self, num_reqs: int) -> None: ...
def on_multi_step_decode_begin(self, num_reqs: int) -> None: ...
def on_multi_step_decode_end(self, num_reqs: int) -> None: ...
@property
def advance_draft_positions(self) -> bool:
"""
@@ -104,6 +116,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.on_prefill_begin(self.max_num_reqs)
self.prefill_cudagraph_manager.capture(
self._prefill,
self.model_state,
@@ -113,10 +127,12 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.kv_cache_config,
progress_bar_desc="Capturing prefill CUDA graphs",
)
self.on_prefill_end(self.max_num_reqs)
if self.num_speculative_steps == 1:
return
self.on_multi_step_decode_begin(self.max_num_reqs)
# Capture the decode draft generation routine (model forward +
# sample + update_draft_inputs) for a single
# step.
@@ -130,6 +146,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.kv_cache_config,
progress_bar_desc="Capturing decode CUDA graphs",
)
self.on_multi_step_decode_end(self.max_num_reqs)
@torch.inference_mode()
def propose(
@@ -159,7 +176,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_tokens = input_batch.num_tokens_after_padding
num_tokens = input_batch.num_tokens
num_tokens_padded = input_batch.num_tokens_after_padding
num_reqs = input_batch.num_reqs
max_query_len = input_batch.num_scheduled_tokens.max()
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
@@ -180,7 +198,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_tokens].copy_(hidden_states)
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
self._copy_request_inputs(
num_reqs,
@@ -208,21 +226,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
num_reqs,
# Use the actual number of tokens without padding added by
# the target model during FULL cudagraph.
input_batch.num_tokens,
num_tokens,
max_query_len,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
num_reqs,
num_tokens,
num_tokens_padded,
uniform_token_count,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
self._prepare_eplb_forward(input_batch.num_tokens)
self._prepare_eplb_forward(num_tokens)
self.on_prefill_begin(num_reqs)
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
# Replay the full graph for draft prefill.
assert self.prefill_cudagraph_manager is not None
@@ -240,6 +259,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
cudagraph_runtime_mode=prefill_batch_desc.cg_mode,
mm_inputs=mm_inputs,
)
self.on_prefill_end(num_reqs)
if self.num_speculative_steps == 1:
# Early exit.
@@ -268,6 +288,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
need_eager=is_profile,
)
self.on_multi_step_decode_begin(num_reqs)
# Generate the remaining num_speculative_steps - 1 draft tokens.
self._multi_step_decode(
num_reqs,
@@ -276,6 +297,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
num_tokens_across_dp,
input_batch.seq_lens_cpu_upper_bound,
)
self.on_multi_step_decode_end(num_reqs)
return self.draft_tokens[:num_reqs]
@@ -10,9 +10,49 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
class MTPSpeculator(AutoRegressiveSpeculator):
share_mtp_topk_indices: bool = False
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
return load_eagle_model(target_model, self.vllm_config)
draft_model = load_eagle_model(target_model, self.vllm_config)
spec_config = self.vllm_config.speculative_config
draft_hf_config = (
spec_config.draft_model_config.hf_config
if spec_config is not None
else None
)
# Detect index_share_for_mtp_iteration. When True, the proposer
# toggles skip_topk so step 0 computes MTP's own indices and
# steps 1+ reuse them.
self.share_mtp_topk_indices = (
getattr(draft_hf_config, "index_share_for_mtp_iteration", False)
and hasattr(draft_model.model, "set_skip_topk")
and hasattr(draft_model.model, "compact_topk_indices")
)
return draft_model
def on_prefill_begin(self, num_reqs: int) -> None:
# Step 0 computes its own top-k. Unconditional, so a step that died
# midway cannot leave reuse mode on.
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(False)
def on_prefill_end(self, num_reqs: int) -> None:
# Step 0 (prefill) wrote topk indices for every query token in the
# multi-token batch. Compact them down to each request's last token so
# steps 1+ can reuse them from the shared buffer.
if self.share_mtp_topk_indices and self.num_speculative_steps > 1:
self.model.model.compact_topk_indices(self.last_token_indices[:num_reqs])
def on_multi_step_decode_begin(self, num_reqs: int) -> None:
# Switch to reuse mode so draft steps 1+ skip the indexer op and read
# the indices that step 0 wrote into the shared buffer.
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(True)
def on_multi_step_decode_end(self, num_reqs: int) -> None:
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(False)