[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-07-30 15:54:15 -07:00
committed by GitHub
parent 7fe5312332
commit dec13a33b7
9 changed files with 1227 additions and 32 deletions
+21 -4
View File
@@ -191,6 +191,8 @@ class InputBatch:
def _prepare_prefill_inputs_kernel(
input_ids_ptr,
next_prefill_tokens_ptr,
next_prefill_tokens_stride,
num_lookahead,
idx_mapping_ptr,
query_start_loc_ptr,
all_token_ids_ptr,
@@ -198,6 +200,7 @@ def _prepare_prefill_inputs_kernel(
prefill_lens_ptr,
num_computed_tokens_ptr,
BLOCK_SIZE: tl.constexpr,
LOOKAHEAD_BLOCK: tl.constexpr,
):
batch_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
@@ -218,10 +221,20 @@ def _prepare_prefill_inputs_kernel(
tokens = tl.load(request_ptr + num_computed + block, mask=mask)
tl.store(input_ids_ptr + query_start + block, tokens, mask=mask)
next_pos = num_computed + query_len
if next_pos < prefill_len:
next_token = tl.load(request_ptr + next_pos)
tl.store(next_prefill_tokens_ptr + req_state_idx, next_token)
# Store the next num_lookahead prefill tokens.
lookahead = tl.arange(0, LOOKAHEAD_BLOCK)
pos = num_computed + query_len + lookahead
in_lookahead = lookahead < num_lookahead
tokens = tl.load(
request_ptr + pos, mask=in_lookahead & (pos < prefill_len), other=0
)
tl.store(
next_prefill_tokens_ptr
+ lookahead * next_prefill_tokens_stride
+ req_state_idx,
tokens,
mask=in_lookahead,
)
def prepare_prefill_inputs(
@@ -234,9 +247,12 @@ def prepare_prefill_inputs(
num_computed_tokens: torch.Tensor,
) -> None:
num_reqs = idx_mapping.shape[0]
num_lookahead = next_prefill_tokens.shape[0]
_prepare_prefill_inputs_kernel[(num_reqs,)](
input_ids,
next_prefill_tokens,
next_prefill_tokens.stride(0),
num_lookahead,
idx_mapping,
query_start_loc,
all_token_ids,
@@ -244,6 +260,7 @@ def prepare_prefill_inputs(
prefill_len,
num_computed_tokens,
BLOCK_SIZE=1024,
LOOKAHEAD_BLOCK=triton.next_power_of_2(num_lookahead),
)