[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-30 15:54:15 -07:00
committed by GitHub
parent 7fe5312332
commit dec13a33b7
9 changed files with 1227 additions and 32 deletions
@@ -3,6 +3,7 @@
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn as nn
@@ -282,6 +283,7 @@ class DFlashSpeculator(DraftModelSpeculator):
step: int,
num_query_per_req: int | None = None,
causal: bool | Mapping[int, bool] = False,
query_start_loc_np: np.ndarray | None = None,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
@@ -294,6 +296,7 @@ class DFlashSpeculator(DraftModelSpeculator):
step=step,
num_query_per_req=self.num_query_per_req,
causal=causal,
query_start_loc_np=query_start_loc_np,
)
@torch.inference_mode()