[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-30 15:54:15 -07:00
committed by GitHub
parent 7fe5312332
commit dec13a33b7
9 changed files with 1227 additions and 32 deletions
+21 -4
View File
@@ -191,6 +191,8 @@ class InputBatch:
def _prepare_prefill_inputs_kernel(
input_ids_ptr,
next_prefill_tokens_ptr,
next_prefill_tokens_stride,
num_lookahead,
idx_mapping_ptr,
query_start_loc_ptr,
all_token_ids_ptr,
@@ -198,6 +200,7 @@ def _prepare_prefill_inputs_kernel(
prefill_lens_ptr,
num_computed_tokens_ptr,
BLOCK_SIZE: tl.constexpr,
LOOKAHEAD_BLOCK: tl.constexpr,
):
batch_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
@@ -218,10 +221,20 @@ def _prepare_prefill_inputs_kernel(
tokens = tl.load(request_ptr + num_computed + block, mask=mask)
tl.store(input_ids_ptr + query_start + block, tokens, mask=mask)
next_pos = num_computed + query_len
if next_pos < prefill_len:
next_token = tl.load(request_ptr + next_pos)
tl.store(next_prefill_tokens_ptr + req_state_idx, next_token)
# Store the next num_lookahead prefill tokens.
lookahead = tl.arange(0, LOOKAHEAD_BLOCK)
pos = num_computed + query_len + lookahead
in_lookahead = lookahead < num_lookahead
tokens = tl.load(
request_ptr + pos, mask=in_lookahead & (pos < prefill_len), other=0
)
tl.store(
next_prefill_tokens_ptr
+ lookahead * next_prefill_tokens_stride
+ req_state_idx,
tokens,
mask=in_lookahead,
)
def prepare_prefill_inputs(
@@ -234,9 +247,12 @@ def prepare_prefill_inputs(
num_computed_tokens: torch.Tensor,
) -> None:
num_reqs = idx_mapping.shape[0]
num_lookahead = next_prefill_tokens.shape[0]
_prepare_prefill_inputs_kernel[(num_reqs,)](
input_ids,
next_prefill_tokens,
next_prefill_tokens.stride(0),
num_lookahead,
idx_mapping,
query_start_loc,
all_token_ids,
@@ -244,6 +260,7 @@ def prepare_prefill_inputs(
prefill_len,
num_computed_tokens,
BLOCK_SIZE=1024,
LOOKAHEAD_BLOCK=triton.next_power_of_2(num_lookahead),
)
+13 -1
View File
@@ -223,6 +223,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.is_pooling_model = self.model_config.runner_type == "pooling"
self.pooling_runner: PoolingRunner | None = None
# Multi-module MTP feeds its modules the next num_speculative_steps prefill
# tokens during chunked prefill. Other speculators only read the immediate
# next one.
num_prefill_lookahead = (
self.num_speculative_steps
if self.speculative_config is not None
and self.speculative_config.use_multi_module_mtp()
else 1
)
# General request states.
self.req_states = RequestState(
max_num_reqs=self.max_num_reqs,
@@ -231,6 +240,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_speculative_steps=self.num_speculative_steps,
vocab_size=self.vocab_size,
device=self.device,
num_prefill_lookahead=num_prefill_lookahead,
)
self.input_buffers = InputBuffers(
max_num_reqs=self.max_num_reqs,
@@ -628,7 +638,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
torch.zeros(
input_batch.num_tokens,
dtype=torch.bool,
device=self.device,
device="cpu",
),
)
@@ -1521,6 +1531,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# NOTE: This is done here because postprocess updates
# num_computed_prefill_tokens.
# The EAGLE/MTP drafter reads one position ahead of the target.
# TODO(TheEpicDolphin): Gather MM embeddings for all speculative
# steps during multi-module MTP.
mm_inputs = self.model_state.gather_mm_embeddings(
input_batch, draft_lookahead=1
)
@@ -3,6 +3,7 @@
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -282,6 +283,7 @@ class DFlashSpeculator(DraftModelSpeculator):
step: int,
num_query_per_req: int | None = None,
causal: bool | Mapping[int, bool] = False,
query_start_loc_np: np.ndarray | None = None,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
@@ -294,6 +296,7 @@ class DFlashSpeculator(DraftModelSpeculator):
step=step,
num_query_per_req=self.num_query_per_req,
causal=causal,
query_start_loc_np=query_start_loc_np,
)
@torch.inference_mode()
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
File diff suppressed because it is too large Load Diff
+27 -13
View File
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -51,7 +52,7 @@ class BaseSpeculator(ABC):
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
# [num_prefill_lookahead, max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
@@ -214,14 +215,28 @@ class DraftModelSpeculator(BaseSpeculator):
step: int,
num_query_per_req: int = 1,
causal: bool | Mapping[int, bool] = True,
query_start_loc_np: np.ndarray | None = None,
) -> dict[str, Any] | None:
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some
# attention backends require.
query_start_loc_cpu = (
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
* num_query_per_req
)
if query_start_loc_np is not None:
# Non-uniform query layout (e.g. multi-module MTP's mixed
# prefill/decode queries); num_query_per_req is ignored.
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
query_start_loc_np[: num_reqs + 1]
)
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
max_query_len = int(
(query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()
)
else:
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some
# attention backends require.
query_start_loc_cpu = (
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
* num_query_per_req
)
max_query_len = num_query_per_req
block_tables = [
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
]
@@ -243,7 +258,7 @@ class DraftModelSpeculator(BaseSpeculator):
: num_reqs_padded + 1
],
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=num_query_per_req,
max_query_len=max_query_len,
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
max_seq_len=self.draft_max_seq_len,
block_tables=block_tables,
@@ -325,7 +340,6 @@ class DraftModelSpeculator(BaseSpeculator):
self.temperature.copy_(temperature)
self.seeds.copy_(seeds)
self.idx_mapping[:num_reqs].copy_(idx_mapping)
if self.draft_logits is not None:
# idx_mapping for CG padded requests points to -1, which is ignored
# during sampling to prevent writing stale values to draft logits.
self.idx_mapping[num_reqs:].fill_(-1)
# idx_mapping for CG padded requests points to -1, which is ignored
# during sampling to prevent writing stale values to draft logits.
self.idx_mapping[num_reqs:].fill_(-1)
+5 -1
View File
@@ -15,6 +15,7 @@ class RequestState:
num_speculative_steps: int,
vocab_size: int,
device: torch.device,
num_prefill_lookahead: int = 1,
):
self.max_num_reqs = max_num_reqs
self.max_model_len = max_model_len
@@ -77,7 +78,10 @@ class RequestState:
)
self.next_prefill_tokens = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=device
num_prefill_lookahead,
self.max_num_reqs,
dtype=torch.int32,
device=device,
)
@property