[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-30 15:54:15 -07:00
committed by GitHub
parent 7fe5312332
commit dec13a33b7
9 changed files with 1227 additions and 32 deletions
@@ -3,6 +3,7 @@
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -282,6 +283,7 @@ class DFlashSpeculator(DraftModelSpeculator):
step: int,
num_query_per_req: int | None = None,
causal: bool | Mapping[int, bool] = False,
query_start_loc_np: np.ndarray | None = None,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
@@ -294,6 +296,7 @@ class DFlashSpeculator(DraftModelSpeculator):
step=step,
num_query_per_req=self.num_query_per_req,
causal=causal,
query_start_loc_np=query_start_loc_np,
)
@torch.inference_mode()
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
File diff suppressed because it is too large Load Diff
+27 -13
View File
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -51,7 +52,7 @@ class BaseSpeculator(ABC):
num_rejected: torch.Tensor,
# [max_num_reqs]
last_sampled: torch.Tensor,
# [max_num_reqs]
# [num_prefill_lookahead, max_num_reqs]
next_prefill_tokens: torch.Tensor,
# [max_num_reqs]
temperature: torch.Tensor,
@@ -214,14 +215,28 @@ class DraftModelSpeculator(BaseSpeculator):
step: int,
num_query_per_req: int = 1,
causal: bool | Mapping[int, bool] = True,
query_start_loc_np: np.ndarray | None = None,
) -> dict[str, Any] | None:
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some
# attention backends require.
query_start_loc_cpu = (
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
* num_query_per_req
)
if query_start_loc_np is not None:
# Non-uniform query layout (e.g. multi-module MTP's mixed
# prefill/decode queries); num_query_per_req is ignored.
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
query_start_loc_np[: num_reqs + 1]
)
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
max_query_len = int(
(query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()
)
else:
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some
# attention backends require.
query_start_loc_cpu = (
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
* num_query_per_req
)
max_query_len = num_query_per_req
block_tables = [
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
]
@@ -243,7 +258,7 @@ class DraftModelSpeculator(BaseSpeculator):
: num_reqs_padded + 1
],
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=num_query_per_req,
max_query_len=max_query_len,
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
max_seq_len=self.draft_max_seq_len,
block_tables=block_tables,
@@ -325,7 +340,6 @@ class DraftModelSpeculator(BaseSpeculator):
self.temperature.copy_(temperature)
self.seeds.copy_(seeds)
self.idx_mapping[:num_reqs].copy_(idx_mapping)
if self.draft_logits is not None:
# idx_mapping for CG padded requests points to -1, which is ignored
# during sampling to prevent writing stale values to draft logits.
self.idx_mapping[num_reqs:].fill_(-1)
# idx_mapping for CG padded requests points to -1, which is ignored
# during sampling to prevent writing stale values to draft logits.
self.idx_mapping[num_reqs:].fill_(-1)