[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-30 15:54:15 -07:00
committed by GitHub
parent 7fe5312332
commit dec13a33b7
9 changed files with 1227 additions and 32 deletions
+13 -1
View File
@@ -223,6 +223,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.is_pooling_model = self.model_config.runner_type == "pooling"
self.pooling_runner: PoolingRunner | None = None
# Multi-module MTP feeds its modules the next num_speculative_steps prefill
# tokens during chunked prefill. Other speculators only read the immediate
# next one.
num_prefill_lookahead = (
self.num_speculative_steps
if self.speculative_config is not None
and self.speculative_config.use_multi_module_mtp()
else 1
)
# General request states.
self.req_states = RequestState(
max_num_reqs=self.max_num_reqs,
@@ -231,6 +240,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_speculative_steps=self.num_speculative_steps,
vocab_size=self.vocab_size,
device=self.device,
num_prefill_lookahead=num_prefill_lookahead,
)
self.input_buffers = InputBuffers(
max_num_reqs=self.max_num_reqs,
@@ -628,7 +638,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
torch.zeros(
input_batch.num_tokens,
dtype=torch.bool,
device=self.device,
device="cpu",
),
)
@@ -1521,6 +1531,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# NOTE: This is done here because postprocess updates
# num_computed_prefill_tokens.
# The EAGLE/MTP drafter reads one position ahead of the target.
# TODO(TheEpicDolphin): Gather MM embeddings for all speculative
# steps during multi-module MTP.
mm_inputs = self.model_state.gather_mm_embeddings(
input_batch, draft_lookahead=1
)