[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-30 15:54:15 -07:00
committed by GitHub
parent 7fe5312332
commit dec13a33b7
9 changed files with 1227 additions and 32 deletions
+8
View File
@@ -1410,6 +1410,14 @@ class SpeculativeConfig:
def use_ngram_gpu(self) -> bool:
return self.method == "ngram_gpu"
def use_multi_module_mtp(self) -> bool:
if self.method != "mtp" or self.draft_model_config is None:
return False
num_mtp_layers = getattr(
self.draft_model_config.hf_config, "num_nextn_predict_layers", 1
)
return min(num_mtp_layers, self.num_speculative_tokens) > 1
def __repr__(self) -> str:
method = self.method
model = (